sydney/IPB
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 5.0.6

Browse Source
This commit is contained in:
Neo committed 2025-12-19 18:55:49 -08:00
1 parent a3a4a9ccbd
commit b89858b2ff
1214 files changed
+92828 -14859
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

system/Content/Search/Elastic/Index.php
+91 -72
View File
@@ -18,6 +18,7 @@ use IPS\Content\Item;
use IPS\Content\Search\Index as SearchIndex;
use IPS\Content\Search\Elastic\Exception as ElasticException;
use IPS\Content;
use IPS\Content\Taggable;
use IPS\DateTime;
use IPS\Db;
use IPS\Http\Request\Exception as RequestException;
@@ -58,8 +59,8 @@ class Index extends SearchIndex
/**
* @brief Elasticsearch version requirements
*/
const MINIMUM_VERSION = '7.2.0';
const UNSUPPORTED_VERSION = '8.0.0';
const MINIMUM_VERSION = '8.0.0';
const UNSUPPORTED_VERSION = '9.0.0';
/**
* @brief The server URL
@@ -101,65 +102,63 @@ class Index extends SearchIndex
$definition = array(
'settings' => $settings,
'mappings' => array(
'_doc' => array(
'properties' => array(
'index_class' => array( 'type' => 'keyword' ),
'index_object_id' => array( 'type' => 'long' ),
'index_item_id' => array( 'type' => 'long' ),
'index_container_class' => array( 'type' => 'keyword' ),
'index_container_id' => array( 'type' => 'long' ),
'index_title' => array(
'type' => 'text',
'analyzer' => $analyzer,
),
'index_content' => array(
'type' => 'text',
'analyzer' => $analyzer,
),
'index_permissions' => array( 'type' => 'keyword' ),
'index_date_created' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_date_updated' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_date_commented' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_author' => array( 'type' => 'long' ),
'index_tags' => array( 'type' => 'keyword' ),
'index_prefix' => array( 'type' => 'keyword' ),
'index_hidden' => array( 'type' => 'byte' ),
'index_item_index_id' => array( 'type' => 'keyword' ),
'index_item_author' => array( 'type' => 'long' ),
'index_is_last_comment' => array( 'type' => 'boolean' ),
'index_club_id' => array( 'type' => 'long' ),
'index_class_type_id_hash' => array( 'type' => 'keyword' ),
'index_comments' => array( 'type' => 'long' ),
'index_reviews' => array( 'type' => 'long' ),
'index_participants' => array( 'type' => 'long' ),
'index_is_anon' => array( 'type' => 'byte' ),
'index_item_solved' => array( 'type' => 'byte' )
)
'properties' => array(
'index_class' => array( 'type' => 'keyword' ),
'index_object_id' => array( 'type' => 'long' ),
'index_item_id' => array( 'type' => 'long' ),
'index_container_class' => array( 'type' => 'keyword' ),
'index_container_id' => array( 'type' => 'long' ),
'index_title' => array(
'type' => 'text',
'analyzer' => $analyzer,
),
'index_content' => array(
'type' => 'text',
'analyzer' => $analyzer,
),
'index_permissions' => array( 'type' => 'keyword' ),
'index_date_created' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_date_updated' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_date_commented' => array(
'type' => 'date',
'format' => 'epoch_second',
),
'index_author' => array( 'type' => 'long' ),
'index_tags' => array( 'type' => 'keyword' ),
'index_prefix' => array( 'type' => 'keyword' ),
'index_hidden' => array( 'type' => 'byte' ),
'index_item_index_id' => array( 'type' => 'keyword' ),
'index_item_author' => array( 'type' => 'long' ),
'index_is_last_comment' => array( 'type' => 'boolean' ),
'index_club_id' => array( 'type' => 'long' ),
'index_class_type_id_hash' => array( 'type' => 'keyword' ),
'index_comments' => array( 'type' => 'long' ),
'index_reviews' => array( 'type' => 'long' ),
'index_participants' => array( 'type' => 'long' ),
'index_is_anon' => array( 'type' => 'byte' ),
'index_item_solved' => array( 'type' => 'byte' )
)
)
);
try
{
$response = Index::request( $this->url->setQueryString( 'include_type_name', 'true' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
$response = Index::request( $this->url )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
if( $response->httpResponseCode != 200 )
{
throw new RuntimeException;
}
}
catch( Exception )
catch( Exception $e )
{
$response = Index::request( $this->url )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
Log::log( $e, 'elasticsearch' );
}
}
catch ( Exception $e )
@@ -223,7 +222,7 @@ class Index extends SearchIndex
try
{
$r = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
$r = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/' . $this->getIndexId( $object ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
if ( $r['found'] )
{
$existingData = $r['_source'];
@@ -239,12 +238,12 @@ class Index extends SearchIndex
$resetLastComment = TRUE;
}
else if ( $indexData['index_is_last_comment'] and $indexData['index_item_id'] )
else if ( $indexData['index_hidden'] !== 0 or ( $indexData['index_is_last_comment'] and $indexData['index_item_id'] ) )
{
/* We have a new "last comment" */
/* We have a new "last comment" or we just hid a comment in the feed so reset the last visible comment */
$resetLastComment = TRUE;
}
/* Insert into index */
$r = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ), LONG_REQUEST_TIMEOUT )
->setHeaders( array( 'Content-Type' => 'application/json' ) )
@@ -298,7 +297,7 @@ class Index extends SearchIndex
if( $error = $this->getResponseError( $response ) )
{
throw new ElasticException( $error['type'] . ' ' . $error['type'] );
throw new ElasticException( $error['type'] . ' ' . $error['reason'] ?? '' );
}
$response = $response->decodeJson();
@@ -374,8 +373,8 @@ class Index extends SearchIndex
'index_comments' => $item->mapped('num_comments'),
'index_reviews' => $item->mapped('num_reviews'),
'index_participants' => $participants,
'index_tags' => $item->tags(),
'index_prefix' => $item->prefix(),
'index_tags' => ( IPS::classUsesTrait( $item, Taggable::class ) ? $item->tags() : [] ),
'index_prefix' => ( IPS::classUsesTrait( $item, Taggable::class ) ? $item->prefix() : null ),
'index_item_solved' => $isSolved
);
}
@@ -444,6 +443,11 @@ class Index extends SearchIndex
)
)
) ) );
if( $error = $this->getResponseError( $r ) )
{
throw new ElasticException( $error['type'] . ' ' . $error['reason'] ?? '' );
}
}
catch ( RequestException $e )
{
@@ -455,9 +459,9 @@ class Index extends SearchIndex
* Retrieve the search ID for an item
*
* @param Content $object Item to add
* @return mixed
* @return string
*/
public function getIndexId( Content $object ): mixed
public function getIndexId( Content $object ): string
{
$databaseColumnId = $object::$databaseColumnId;
return strtolower( str_replace( '\\', '_', substr( get_class( $object ), 4 ) ) ) . '-' . $object->$databaseColumnId;
@@ -517,14 +521,14 @@ class Index extends SearchIndex
try
{
$indexId = strtolower( str_replace( '\\', '_', substr( $class, 4 ) ) ) . '-' . $id;
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/' . $indexId ) )->delete();
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/' . $indexId ) )->delete();
if ( is_subclass_of( $class, 'IPS\Content\Item' ) )
{
if ( isset( $class::$commentClass ) )
{
$commentClass = $class::$commentClass;
$response = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$response = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -547,7 +551,7 @@ class Index extends SearchIndex
if ( isset( $class::$reviewClass ) )
{
$reviewClass = $class::$reviewClass;
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -619,13 +623,13 @@ class Index extends SearchIndex
);
}
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => $query
) ) );
}
else
{
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'term' => array(
'index_class' => $class
@@ -730,7 +734,7 @@ class Index extends SearchIndex
if ( count( $updates ) )
{
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => implode( ' ', $updates ),
'lang' => 'painless',
@@ -751,7 +755,7 @@ class Index extends SearchIndex
)
);
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_permissions.add( 'm' + ctx._source.index_author );",
'lang' => 'painless'
@@ -781,7 +785,7 @@ class Index extends SearchIndex
);
}
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_hidden = params.newHiddenStatus;",
'lang' => 'painless',
@@ -847,7 +851,7 @@ class Index extends SearchIndex
try
{
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $item ) . '/_update' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/' . $this->getIndexId( $item ) . '/_update' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'doc' => $update
) ) );
@@ -879,7 +883,7 @@ class Index extends SearchIndex
{
try
{
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'range' => array(
'index_date_updated' => array(
@@ -941,7 +945,12 @@ class Index extends SearchIndex
)
)
) ) );
if( $error = $this->getResponseError( $r ) )
{
throw new ElasticException( $error['type'] . ' ' . $error['reason'] ?? '' );
}
/* Get the latest comment */
$itemClass = NULL;
foreach ( $classes as $class )
@@ -980,6 +989,11 @@ class Index extends SearchIndex
)
) ) );
if( $error = $this->getResponseError( $r ) )
{
throw new ElasticException( $error['type'] . ' ' . $error['reason'] ?? '' );
}
/* And set the updated time on the main item (done as _update_by_query because it might not exist if the first comment is required) */
$indexDataForLastComment = $this->indexData( $lastComment );
$r = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
@@ -1013,6 +1027,11 @@ class Index extends SearchIndex
)
)
) ) );
if( $error = $this->getResponseError( $r ) )
{
throw new ElasticException( $error['type'] . ' ' . $error['reason'] ?? '' );
}
}
}
catch ( OutOfRangeException ) {}
@@ -1054,7 +1073,7 @@ class Index extends SearchIndex
);
/* Get the count */
$count = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$count = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 0,
'query' => $query
) ) )->decodeJson();
@@ -1065,7 +1084,7 @@ class Index extends SearchIndex
}
/* Now get the unique item ids */
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'aggs' => array(
'itemIds' => array(
'terms' => array(
@@ -1107,7 +1126,7 @@ class Index extends SearchIndex
{
try
{
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'query' => array(
'bool' => array(
'filter' => array(
@@ -1156,7 +1175,7 @@ class Index extends SearchIndex
{
try
{
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = Index::request( $this->url->setPath( $this->url->data[ Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 1,
'sort' => array( array( 'index_date_updated' => 'asc' ) )
) ) )->decodeJson();