sydney/IPB
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 4.7.0

Browse Source
This commit is contained in:
Neo committed 2025-12-19 16:00:35 -08:00
1 parent 517a5e1f70
commit 8ba7d43898
1933 files changed
+65613 -11332
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

system/Content/Search/Elastic/Index.php
+71 -57
View File
@@ -25,8 +25,8 @@ class _Index extends \IPS\Content\Search\Index
/**
* @brief Elasticsearch version requirements
*/
const MINIMUM_VERSION = '6.0.0';
const UNSUPPORTED_VERSION = '7.7.0';
const MINIMUM_VERSION = '7.2.0';
const UNSUPPORTED_VERSION = '8.0.0';
/**
* @brief The server URL
@@ -61,7 +61,7 @@ class _Index extends \IPS\Content\Search\Index
$analyzer = key( $settings['analysis']['analyzer'] );
}
$this->url->request()->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url )->delete();
$definition = array(
'settings' => $settings,
@@ -106,6 +106,8 @@ class _Index extends \IPS\Content\Search\Index
'index_comments' => array( 'type' => 'long' ),
'index_reviews' => array( 'type' => 'long' ),
'index_participants' => array( 'type' => 'long' ),
'index_is_anon' => array( 'type' => 'byte' ),
'index_item_solved' => array( 'type' => 'byte' )
)
)
)
@@ -113,7 +115,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$response = $this->url->setQueryString( 'include_type_name', 'true' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setQueryString( 'include_type_name', 'true' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
if( $response->httpResponseCode != 200 )
{
@@ -122,7 +124,7 @@ class _Index extends \IPS\Content\Search\Index
}
catch( \Exception $e )
{
$response = $this->url->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
$response = \IPS\Content\Search\Elastic\Index::request( $this->url )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
}
}
@@ -137,16 +139,15 @@ class _Index extends \IPS\Content\Search\Index
if ( $indexData = parent::indexData( $object ) )
{
$indexData['index_permissions'] = explode( ',', $indexData['index_permissions'] );
$indexData['index_tags'] = $indexData['index_tags'] ? explode( ',', $indexData['index_tags'] ) : array();
$indexData['index_is_last_comment'] = (bool) $indexData['index_is_last_comment'];
if ( $object instanceof \IPS\Content\Item )
{
$indexData += $this->metaCounts( $object );
$indexData = array_merge( $indexData, $this->metaData( $object ) );
}
else
{
$indexData += $this->metaCounts( $object->item() );
$indexData = array_merge( $indexData, $this->metaData( $object->item() ) );
}
return $indexData;
@@ -181,7 +182,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$r = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
if ( $r['found'] )
{
$existingData = $r['_source'];
@@ -204,9 +205,7 @@ class _Index extends \IPS\Content\Search\Index
}
/* Insert into index */
$r = $this->url
->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) )
->request( \IPS\LONG_REQUEST_TIMEOUT )
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ), \IPS\LONG_REQUEST_TIMEOUT )
->setHeaders( array( 'Content-Type' => 'application/json' ) )
->put( json_encode( $indexData ) );
@@ -215,6 +214,11 @@ class _Index extends \IPS\Content\Search\Index
throw new \IPS\Content\Search\Elastic\Exception( $error['type'] . ': ' . $error['reason'] );
}
if ( $resetLastComment )
{
$this->resetLastComment( array( $indexData['index_class'] ), $indexData['index_item_id'] );
}
/* Views / Comments / Reviews */
if ( $object instanceof \IPS\Content\Item )
{
@@ -224,7 +228,8 @@ class _Index extends \IPS\Content\Search\Index
{
$item = $object->item();
}
$this->rebuildMetaCounts( $item, $indexData['index_is_last_comment'] ? $this->getIndexId( $object ) : NULL );
$this->rebuildMetaData( $item );
}
catch ( \IPS\Http\Request\Exception $e )
{
@@ -234,11 +239,6 @@ class _Index extends \IPS\Content\Search\Index
{
\IPS\Log::log( $e, 'elasticsearch_response_error' );
}
if ( $resetLastComment )
{
$this->resetLastComment( array( $indexData['index_class'] ), $indexData['index_item_id'] );
}
}
}
}
@@ -253,7 +253,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
/* This request *intentionally* goes to _tasks and not (ourpath)/_tasks */
$response = $this->url->setPath( '/_tasks' )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get();
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_tasks' ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get();
if( $error = $this->getResponseError( $response ) )
{
@@ -278,7 +278,7 @@ class _Index extends \IPS\Content\Search\Index
continue;
}
$this->url->setPath( '/.tasks/task' . $taskId )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/.tasks/task' . $taskId ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->delete();
}
}
}
@@ -298,7 +298,7 @@ class _Index extends \IPS\Content\Search\Index
* @param \IPS\Content\Searchable $item The content item
* @return void
*/
protected function metaCounts( $item )
protected function metaData( $item )
{
$databaseColumnId = $item::$databaseColumnId;
@@ -318,7 +318,9 @@ class _Index extends \IPS\Content\Search\Index
return array(
'index_comments' => $item->mapped('num_comments'),
'index_reviews' => $item->mapped('num_reviews'),
'index_participants' => $participants
'index_participants' => $participants,
'index_tags' => $item->tags(),
'index_prefix' => $item->prefix(),
);
}
@@ -326,10 +328,9 @@ class _Index extends \IPS\Content\Search\Index
* Rebuild the comment / review counts for an item
*
* @param \IPS\Content\Searchable $item The content item
* @param string $lastCommentIndexId The index ID of the last comment, if you want to reset others
* @return void
*/
protected function rebuildMetaCounts( $item, $lastCommentIndexId = NULL )
protected function rebuildMetaData( $item )
{
$databaseColumnId = $item::$databaseColumnId;
$class = \get_class( $item );
@@ -347,12 +348,12 @@ class _Index extends \IPS\Content\Search\Index
{
$updates = array();
$params = array();
foreach ( $this->metaCounts( $item ) as $k => $v )
foreach ( $this->metaData( $item ) as $k => $v )
{
if ( \is_array( $v ) )
{
$updates[] = "ctx._source.{$k} = params.param_{$k};";
$params[ 'param_' . $k ] = array_map( 'intval', $v );
$params[ 'param_' . $k ] = ( $k !== 'index_tags' ? array_map( 'intval', $v ) : $v );
}
elseif ( \is_null( $v ) )
{
@@ -365,12 +366,8 @@ class _Index extends \IPS\Content\Search\Index
$params['param_' . $k ] = \intval( $v );
}
}
if ( !\is_null( $lastCommentIndexId ) )
{
$updates[] = "ctx._source.index_is_last_comment = ( ctx._id == params.param_lastcommentid );";
$params['param_lastcommentid'] = $lastCommentIndexId;
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => implode( ' ', $updates ),
'lang' => 'painless',
@@ -429,7 +426,7 @@ class _Index extends \IPS\Content\Search\Index
if ( !( $object instanceof \IPS\Content\Item ) )
{
$this->rebuildMetaCounts( $object->item() );
$this->rebuildMetaData( $object->item() );
$itemClass = \get_class( $object->item() );
$itemIdColumn = $itemClass::$databaseColumnId;
@@ -465,14 +462,14 @@ class _Index extends \IPS\Content\Search\Index
try
{
$indexId = \strtolower( str_replace( '\\', '_', \substr( $class, 4 ) ) ) . '-' . $id;
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $indexId )->request()->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $indexId ) )->delete();
if ( is_subclass_of( $class, 'IPS\Content\Item' ) )
{
if ( isset( $class::$commentClass ) )
{
$commentClass = $class::$commentClass;
$response = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -495,7 +492,7 @@ class _Index extends \IPS\Content\Search\Index
if ( isset( $class::$reviewClass ) )
{
$reviewClass = $class::$reviewClass;
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -565,15 +562,15 @@ class _Index extends \IPS\Content\Search\Index
'index_author' => $authorId
)
);
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
}
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => $query
) ) );
}
else
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'term' => array(
'index_class' => $class
@@ -677,7 +674,7 @@ class _Index extends \IPS\Content\Search\Index
if ( \count( $updates ) )
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => implode( ' ', $updates ),
'lang' => 'painless',
@@ -697,8 +694,8 @@ class _Index extends \IPS\Content\Search\Index
'index_author' => 0
)
);
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_permissions.add( 'm' + ctx._source.index_author );",
'lang' => 'painless'
@@ -727,8 +724,8 @@ class _Index extends \IPS\Content\Search\Index
)
);
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_hidden = params.newHiddenStatus;",
'lang' => 'painless',
@@ -766,7 +763,7 @@ class _Index extends \IPS\Content\Search\Index
{
return array(
'bool' => array(
'must' => array( $conditions )
'must' => $conditions
)
);
}
@@ -795,7 +792,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $item ) . '/_update' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $item ) . '/_update' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'doc' => $update
) ) );
@@ -827,7 +824,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'range' => array(
'index_date_updated' => array(
@@ -861,7 +858,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
/* Remove the flag */
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_is_last_comment = false;",
'lang' => 'painless'
@@ -921,15 +918,15 @@ class _Index extends \IPS\Content\Search\Index
if ( $lastComment = $item->comments( 1, 0, 'date', 'desc', NULL, FALSE, NULL, $where ) AND $lastComment instanceof \IPS\Content\Searchable )
{
/* Set that it is the latest comment */
$r = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $lastComment ) . '/_update' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update/' . $this->getIndexId( $lastComment ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'doc' => array(
'index_is_last_comment' => true
)
) ) );
/* And set the updated time on the main item (done as _update_by_query because it might not exist if the first comment is required) */
$indexDataForLastComment = $this->indexData( $lastComment );
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_date_updated = params.dateUpdated; ctx._source.index_date_commented = params.dateCommented;",
'lang' => 'painless',
@@ -1001,7 +998,7 @@ class _Index extends \IPS\Content\Search\Index
);
/* Get the count */
$count = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$count = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 0,
'query' => $query
) ) )->decodeJson();
@@ -1012,7 +1009,7 @@ class _Index extends \IPS\Content\Search\Index
}
/* Now get the unique item ids */
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'aggs' => array(
'itemIds' => array(
'terms' => array(
@@ -1054,7 +1051,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'query' => array(
'bool' => array(
'filter' => array(
@@ -1103,7 +1100,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 1,
'sort' => array( array( 'index_date_updated' => 'asc' ) )
) ) )->decodeJson();
@@ -1139,7 +1136,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$results = $this->url->setPath( '/_analyze' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_analyze' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'analyzer' => $analyzer,
'text' => implode( ' ', $terms )
) ) )->decodeJson();
@@ -1172,6 +1169,23 @@ class _Index extends \IPS\Content\Search\Index
return FALSE;
}
/**
* Wrapper to account for log in where needed
*
* @param $url
* @param null $timeout
* @return \IPS\Http\Request\Curl|Socket
*/
public static function request( $url, $timeout=NULL )
{
if ( \IPS\ELASTICSEARCH_USER or \IPS\ELASTICSEARCH_PASSWORD )
{
return $url->request( $timeout )->login( \IPS\ELASTICSEARCH_USER, \IPS\ELASTICSEARCH_PASSWORD );
}
return $url->request( $timeout );
}
/**
* Check response to see if an error was produced
*
system/Content/Search/Elastic/MassIndexer.php
+1 -1
View File
@@ -73,7 +73,7 @@ class _MassIndexer extends \IPS\Content\Search\Elastic\Index
}
$json = implode( "\n", $json );
$this->url->setPath( '/_bulk' )->request()->setHeaders( array( 'Content-Type' => 'application/x-ndjson' ) )->post( $json . "\n" );
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_bulk' ) )->setHeaders( array( 'Content-Type' => 'application/x-ndjson' ) )->post( $json . "\n" );
}
}
}
system/Content/Search/Elastic/Query.php
+173 -15
View File
@@ -69,7 +69,12 @@ class _Query extends \IPS\Content\Search\Query
parent::__construct( $member );
$this->url = $url;
}
/**
* @var bool Stores whether or not any content type needs to be limited by the latest comment
*/
protected $lastCommentMustBeTrue = false;
/**
* Filter by multiple content types
*
@@ -198,11 +203,14 @@ class _Query extends \IPS\Content\Search\Query
/* Only last comment? */
if ( $filter->onlyLastComment )
{
/* Set this for when the search JSON is built */
$this->lastCommentMustBeTrue = true;
$conditions[] = array(
'term' => array( 'index_is_last_comment' => true )
);
}
/* Put it together */
if( \count( $conditions ) )
{
@@ -215,7 +223,7 @@ class _Query extends \IPS\Content\Search\Query
{
$this->filters[] = array(
'bool' => array(
( $type ? 'should' : 'must_not' ) => array( $contentFilterConditions )
( $type ? 'should' : 'must_not' ) => $contentFilterConditions
)
);
}
@@ -363,7 +371,79 @@ class _Query extends \IPS\Content\Search\Query
/* Return for daisy chaining */
return $this;
}
/**
* Stores the "more like this" Content object for search()
* @param null
*/
protected $moreLikeThis = NULL;
/**
* Filter by more like this
* @param \IPS\Content\Searchable $object
*/
public function filterByMoreLikeThis( \IPS\Content\Searchable $object )
{
$index = new \IPS\Content\Search\Elastic\Index( $this->url );
$this->moreLikeThis = $index->getIndexId( ( $object instanceof \IPS\Content\Item and $object::$firstCommentRequired ) ? $object->firstComment() : $object );
/* Some container types cannot be cached */
$classes = [];
$conditions = [];
foreach ( \IPS\Application::allExtensions( 'core', 'ContentRouter', FALSE ) as $object )
{
$classes = array_merge( $object->classes, $classes );
}
/* Some nodes have complex permissions, so avoid any permission issues by not selecting these */
foreach( $classes as $itemClass )
{
if ( isset( $itemClass::$containerNodeClass ) )
{
$containerClass = $itemClass::$containerNodeClass;
$blockIds = [];
if ( $customNodes = $containerClass::customPermissionNodes() )
{
foreach ( $customNodes as $key => $ids )
{
if ( $key !== 'count' )
{
$blockIds = array_merge( $blockIds, $ids );
}
}
}
if ( \count( $blockIds ) )
{
$conditions[] = array(
'bool' => array(
'filter' => array(
array(
'terms' => array( 'index_container_class' => [ $containerClass ] )
),
array(
'terms' => array( 'index_container_id' => $blockIds )
),
)
)
);
}
if ( \count( $conditions ) )
{
foreach( $conditions as $condition )
{
$this->mustNot[] = $condition;
}
}
}
}
/* Only show non-hidden and approved items */
$this->setHiddenFilter( static::HIDDEN_VISIBLE );
}
/**
* Filter by container class
*
@@ -558,9 +638,7 @@ class _Query extends \IPS\Content\Search\Query
if ( $includeMembers and $followed = iterator_to_array( \IPS\Db::i()->select( 'follow_rel_id', 'core_follow', array( 'follow_app=? AND follow_area=? AND follow_member_id=?', 'core', 'member', $this->member->member_id ), 'follow_rel_id asc' ) ) )
{
$conditions[] = array(
array(
'terms' => array( 'index_author' => $followed )
)
'terms' => array( 'index_author' => $followed )
);
}
@@ -799,7 +877,35 @@ class _Query extends \IPS\Content\Search\Query
return $this;
}
/**
* Filter by solved
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterBySolved()
{
$this->filters[] = array(
'term' => array( 'index_item_solved' => 1 )
);
return $this;
}
/**
* Filter by Unsolved
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterByUnsolved()
{
$this->filters[] = array(
'term' => array( 'index_item_solved' => 0 )
);
return $this;
}
/**
* Filter by start date
*
@@ -922,7 +1028,35 @@ class _Query extends \IPS\Content\Search\Query
return $this;
}
/**
* Debug by itemId, used for debugging purposes only
* @note No permission checks run, do not use in production
* @param $itemId
* @return void
*/
public function debugByItemId( $itemId )
{
$array = array(
'query' => [
'term' => [
'index_item_id' => [
'value' => \intval( $itemId )
]
]
],
'sort' => $this->sort ?: array(),
'from' => 0,
'size' => 50,
);
$json = json_encode( $array, JSON_PARTIAL_OUTPUT_ON_ERROR );
return \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
}
/**
* Search
*
@@ -1017,6 +1151,12 @@ class _Query extends \IPS\Content\Search\Query
)
)
);
/* If we're not searching with a term, then just show the title record, not comments */
if ( $term === NULL and ! $this->lastCommentMustBeTrue )
{
$must[] = ['exists' => ['field' => 'index_title']];
}
}
/* Put that with the rest of the conditions */
@@ -1050,7 +1190,25 @@ class _Query extends \IPS\Content\Search\Query
$filters[] = array( 'term' => array( 'index_hidden' => $this->hiddenStatuses ) );
}
}
$searchUrl = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' );
if ( $this->moreLikeThis )
{
$must[] = [
'more_like_this' => [
'fields' => [ 'index_title' ],
'like' => [ '_index' => \IPS\Settings::i()->search_elastic_index, '_id' => $this->moreLikeThis ],
'min_term_freq' => 1,
'max_query_terms' => 12,
'min_word_length' => 3,
]
];
/* Let's enforce a relatively short timeout to prevent slow ES server from dragging down topics */
$searchUrl = $searchUrl->setQueryString( 'timeout', '3s' );
}
/* Peform the search */
try
{
@@ -1062,9 +1220,9 @@ class _Query extends \IPS\Content\Search\Query
'filter' => $filters
)
);
/* Add the time decay */
if ( \IPS\Settings::i()->search_decay_factor and !$this->sort )
if ( \IPS\Settings::i()->search_decay_factor and !$this->sort and ! $this->sort )
{
$query = array(
'function_score' => array(
@@ -1081,7 +1239,7 @@ class _Query extends \IPS\Content\Search\Query
}
/* Add the self boost */
if ( \IPS\Settings::i()->search_elastic_self_boost and $this->member->member_id and !$this->sort )
if ( \IPS\Settings::i()->search_elastic_self_boost and $this->member->member_id and !$this->sort and ! $this->moreLikeThis )
{
$query = array(
'function_score' => array(
@@ -1117,10 +1275,10 @@ class _Query extends \IPS\Content\Search\Query
}
/* Make the call! */
$return = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
$return = \IPS\Content\Search\Elastic\Index::request( $searchUrl )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
if ( isset( $return['error'] ) )
{
\IPS\Log::log( print_r( $return['error'], TRUE ), 'elasticsearch' );
\IPS\Log::log( print_r( array_merge( $array, ['error' => $return['error'] ] ), TRUE ), 'elasticsearch' );
return new \IPS\Content\Search\Results( array(), 0 );
}
system/Content/Search/Index.php
+12 -1
View File
@@ -309,6 +309,7 @@ abstract class _Index extends \IPS\Patterns\Singleton
/* Get the item index ID */
$itemIndexId = NULL;
$itemClass = NULL;
if ( $object instanceof \IPS\Content\Comment )
{
$itemClass = $object::$itemClass;
@@ -390,6 +391,15 @@ abstract class _Index extends \IPS\Patterns\Singleton
$containerClass = ( $object->item()->searchIndexContainerClass() ) ? \get_class( $object->item()->searchIndexContainerClass() ) : NULL;
}
/* Do we have an extension to modify this? */
foreach( \IPS\Application::enabledApplications() as $app )
{
foreach ( $app->extensions( 'core', 'SearchIndex' ) as $extension )
{
$content = $extension->content( $object, $content );
}
}
/* Return */
return array(
'index_class' => $class,
@@ -412,7 +422,8 @@ abstract class _Index extends \IPS\Patterns\Singleton
'index_is_last_comment' => $isLastComment,
'index_club_id' => $clubId,
'index_class_type_id_hash' => md5( $class . ':' . $object->$idColumn ),
'index_is_anon' => $object->isAnonymous()
'index_is_anon' => (int) $object->isAnonymous(),
'index_item_solved' => ( $itemClass and \IPS\IPS::classUsesTrait( $itemClass, 'IPS\Content\Solvable' ) ) ? ( $object->item()->mapped('solved_comment_id') == $object->$idColumn ) ? 1 : 0 : NULL
);
}
system/Content/Search/Mysql/Query.php
+73 -65
View File
@@ -609,7 +609,31 @@ class _Query extends \IPS\Content\Search\Query
$this->filterByUnread = TRUE;
}
/**
* Filter only solved content
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterBySolved()
{
$this->where[] = array( 'index_item_solved=?', 1 );
return $this;
}
/**
* Filter only unsolved content
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterByUnsolved()
{
$this->where[] = array( 'index_item_solved=?', 0 );
return $this;
}
/**
* Get the 'unread' where SQL
*
@@ -629,54 +653,61 @@ class _Query extends \IPS\Content\Search\Query
$unreadWheres = array();
$containerIds = array();
$containerClass = isset( $class::$containerNodeClass ) ? $class::$containerNodeClass : NULL;
foreach( $resetTimes as $containerId => $timestamp )
{
/* Pages has different classes per database, but recorded as 'cms' and the container ID in the marking tables */
if ( $containerClass and method_exists( $containerClass, 'isValidContainerId' ) )
{
if ( ! $containerClass::isValidContainerId( $containerId ) )
{
continue;
}
}
$timestamp = $timestamp ?: $this->member->marked_site_read;
$containerIds[] = $containerId;
$unreadWheres[] = '( index_container_id=' . $containerId . ' AND index_date_updated > ' . (int) $timestamp . ')';
$items = $this->member->markersItems( $application, \IPS\Content\Item::makeMarkerKey( $containerId ) );
if ( \count( $items ) )
if ( \is_array( $resetTimes ) )
{
foreach( $resetTimes as $containerId => $timestamp )
{
foreach( $items as $mid => $mtime )
/* Pages has different classes per database, but recorded as 'cms' and the container ID in the marking tables */
if ( $containerClass and method_exists( $containerClass, 'isValidContainerId' ) )
{
if ( $mtime > $timestamp )
if ( ! $containerClass::isValidContainerId( $containerId ) )
{
/* If an item has been moved from one container to another, the user may have a marker
in it's old location, with the previously 'read' time. In this circumstance, we need
to only use more recent read time, otherwise the topic may be incorrectly included
in the results */
if ( \in_array( $mid, $markers ) )
continue;
}
}
$timestamp = $timestamp ?: $this->member->marked_site_read;
$containerIds[] = $containerId;
$unreadWheres[] = '( index_container_id=' . $containerId . ' AND index_date_updated > ' . (int) $timestamp . ')';
$items = $this->member->markersItems( $application, \IPS\Content\Item::makeMarkerKey( $containerId ) );
if ( \count( $items ) )
{
foreach( $items as $mid => $mtime )
{
if ( $mtime > $timestamp )
{
$_key = array_search( $mid, $markers );
$_mtime = \intval( mb_substr( $_key, 0, mb_strpos( $_key, '.' ) ) );
if ( $_mtime < $mtime )
/* If an item has been moved from one container to another, the user may have a marker
in it's old location, with the previously 'read' time. In this circumstance, we need
to only use more recent read time, otherwise the topic may be incorrectly included
in the results */
if ( \in_array( $mid, $markers ) )
{
unset( $markers[ $_key ] );
}
/* If the existing timestamp is higher, retain that since we reset the $markers array below */
else
{
$mtime = $_mtime;
$_key = array_search( $mid, $markers );
$_mtime = \intval( mb_substr( $_key, 0, mb_strpos( $_key, '.' ) ) );
if ( $_mtime < $mtime )
{
unset( $markers[ $_key ] );
}
/* If the existing timestamp is higher, retain that since we reset the $markers array below */
else
{
$mtime = $_mtime;
}
}
$markers[ $mtime . '.' . $mid ] = $mid;
}
$markers[ $mtime . '.' . $mid ] = $mid;
}
}
}
}
else
{
$unreadWheres[] = "( index_date_updated > " . \intval( $resetTimes ) . ")";
}
if( \count( $containerIds ) )
@@ -972,37 +1003,13 @@ class _Query extends \IPS\Content\Search\Query
$results = iterator_to_array( \IPS\Db::i()->select( 'index_item_id, index_class', array( 'core_search_index_item_map', 'sub' ), array( array( 'index_author_id=' . \intval( $this->member->member_id ) . ' AND ' . \IPS\Db::i()->in( 'index_class', $classes ) ) ), 'index_item_id desc', array( 0, 501 ) ) );
if ( \count( $results ) == 501 )
if ( \count( $results ) )
{
/* Assume we have loads of matches, so do a full query */
$subClause = array( array( 'sub.index_item_id=main.index_item_id AND index_author_id=' . \intval( $this->member->member_id ) . ' AND ' . $this->makeEmbeddedIfClauseForSearchMapTable('sub.index_class') ) );
$query = \IPS\Db::i()->select( 'index_item_id', array( 'core_search_index_item_map', 'sub' ), $subClause );
$where[] = array( 'EXISTS(?)', $query );
}
else if ( \count( $results ) )
{
$classIds = array();
foreach( $results as $result )
{
$classIds[ $result['index_class'] ][] = $result['index_item_id'];
}
$subWhere = array();
foreach( $classIds as $class => $ids )
{
$stack = array( $class );
if ( isset( $class::$commentClass ) )
{
$stack[] = $class::$commentClass;
}
$subWhere[] = '(' . \IPS\Db::i()->in( 'index_class', $stack ) . ' and ' . \IPS\Db::i()->in( 'index_item_id', $ids ) . ')';
}
/* IN is not a SIN. It's been a long day */
$where[] = '(' . implode( ' OR ', $subWhere ) . ')';
}
else
{
/* There are no results */
@@ -1260,8 +1267,9 @@ class _Query extends \IPS\Content\Search\Query
{
$order .= ", index_date_updated";
}
if( $this->filterByUnread === TRUE )
/* If we're filtering by unread, and not using MariaDB, then force an index because MySQL tends to use the incorrect one in this scenario. */
if( !mb_stristr( \IPS\Db::i()->server_info, '-MariaDB' ) AND $this->filterByUnread === TRUE )
{
$this->forceIndex = 'index_date_updated';
}
system/Content/Search/Query.php
+14
View File
@@ -174,6 +174,20 @@ abstract class _Query
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
abstract public function filterByUnread();
/**
* Filter by content the user has not read
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
abstract public function filterBySolved();
/**
* Filter by content the user has not read
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
abstract public function filterByUnsolved();
/**
* Filter by start date
system/Content/Search/Result/Content.php
+1 -1
View File
@@ -138,7 +138,7 @@ class _Content extends \IPS\Content\Search\Result
catch( \Exception $ex ) { }
}
$document->addItem( $object instanceof \IPS\Content\Comment ? $object->item()->searchIndexTitle() : $object->searchIndexTitle(), $object->url(), $this->indexData['index_content'], \IPS\DateTime::ts( $this->indexData['index_date_created'] ), NULL, $enclosure );
$document->addItem( $object instanceof \IPS\Content\Comment ? $object->item()->searchIndexTitle() : $object->searchIndexTitle(), $object->url(), \IPS\Content\Search\Result::preDisplay( $this->indexData['index_content'] ), \IPS\DateTime::ts( $this->indexData['index_date_created'] ), NULL, $enclosure );
}
/* If the search result was orphaned, let us continue */
catch( \OutOfRangeException $e ){}
system/Content/Search/Result/Result.php
+3 -5
View File
@@ -38,12 +38,10 @@ abstract class _Result
if ( \IPS\Settings::i()->emoji_style == 'twemoji' )
{
return preg_replace( '/(?:' . \IPS\Text\Parser::EMOJI_REGEX . '|\x{200d})+/u', '', $content );
}
else
{
return $content;
$content = preg_replace( '/(?:' . \IPS\Text\Parser::EMOJI_REGEX . '|\x{200d})+/u', '', $content );
}
return $content;
}
/**
system/Content/Search/Results.php
+56 -3
View File
@@ -56,6 +56,11 @@ class _Results extends \ArrayIterator
* @brief Reaction Data
*/
protected $reactions = array();
/**
* @brief Attached Images
*/
protected $attachedImages = array();
/**
* @brief Review Ratings
@@ -100,7 +105,8 @@ class _Results extends \ArrayIterator
$reviewRatings = array();
$itemIndexIds = array();
$indexIds = array();
$attachedImages = array();
/* Loop */
foreach ( $this->getArrayCopy() as $result )
{
@@ -162,8 +168,10 @@ class _Results extends \ArrayIterator
{
$containersToLoad[ 'IPS\Member\Club' ][ $result['index_club_id'] ] = $result['index_club_id'];
}
$attachedImages[ $result['index_class'] ][ $result['index_item_id'] ][] = $result['index_object_id'];
}
/* Load item data */
foreach ( $itemsToLoad as $itemClass => $itemIds )
{
@@ -245,7 +253,47 @@ class _Results extends \ArrayIterator
$this->tags = iterator_to_array( \IPS\Db::i()->select( 'index_id, GROUP_CONCAT(index_tag) as index_tags', 'core_search_index_tags', array( array( \IPS\Db::i()->in( 'index_id', $itemIndexIds ) ), array( 'index_is_prefix = 0' ) ), NULL, NULL, 'index_id' )->setKeyField('index_id') );
$this->prefixes = iterator_to_array( \IPS\Db::i()->select( 'index_id, index_tag as index_prefix', 'core_search_index_tags', array( array( \IPS\Db::i()->in( 'index_id', $itemIndexIds ) ), array( 'index_is_prefix = 1' ) ) )->setKeyField('index_id') );
}
/* Load attached images */
$clause = [];
$binds = [];
$locationKeyMap = [];
foreach ( $attachedImages as $class => $ids )
{
$itemClass = \in_array( 'IPS\Content\Comment', class_parents( $class ) ) ? $class::$itemClass : $class;
foreach( $ids as $indexId => $commentIds )
{
foreach( $commentIds as $commentId )
{
$locationKey = $itemClass::$application . '_' . mb_ucfirst( $itemClass::$module );
$locationKeyMap[$locationKey] = $class;
$clause[] = "( location_key=? AND id1=? AND id2=?)";
$binds[] = $locationKey;
$binds[] = $indexId;
$binds[] = $commentId;
}
}
}
if ( \count( $attachedImages ) and \count( $clause ) )
{
/* As we need to join back onto the map table, 2 queries are much more efficient */
$ids = iterator_to_array( \IPS\Db::i()->select( 'attachment_id', 'core_attachments_map', array( array_merge( array( implode( ' OR ', $clause ) ), $binds ) ) ) );
foreach ( \IPS\Db::i()->select( '*', 'core_attachments', array( array( \IPS\Db::i()->in( 'attach_id', $ids ) ), array('attach_is_image=1') ), 'attach_id ASC' )
->join( 'core_attachments_map', 'core_attachments_map.attachment_id=core_attachments.attach_id' ) as $row )
{
if ( isset( $locationKeyMap[ $row['location_key'] ] ) and isset( $row['id2'] ) )
{
$this->attachedImages[ $locationKeyMap[ $row['location_key'] ] ][ $row['id2'] ][] = [
'extension' => 'core_Attachment',
'location' => $row['attach_location'],
'thumb_location' => $row['attach_thumb_location'],
'labels' => implode( ', ', \IPS\Text\Parser::getAttachmentLabels( $row ) )
];
}
}
}
/* Set that we're initiated */
$this->initated = TRUE;
}
@@ -303,6 +351,11 @@ class _Results extends \ArrayIterator
}
}
if ( \is_array( $this->attachedImages ) and isset( $this->attachedImages[ $class ][ $data['index_object_id'] ] ) )
{
$itemData['attachedImages'] = $this->attachedImages[ $class ][ $data['index_object_id'] ];
}
if ( isset( $this->tags[ $data['index_item_index_id'] ] ) )
{
$data['index_tags'] = $this->tags[ $data['index_item_index_id'] ]['index_tags'];