sydney/IPB
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 4.7.0

Browse Source
This commit is contained in:
Neo committed 2025-12-19 16:00:35 -08:00
1 parent 517a5e1f70
commit 8ba7d43898
1933 files changed
+65613 -11332
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

system/Content/Search/Elastic/Index.php
+71 -57
View File
@@ -25,8 +25,8 @@ class _Index extends \IPS\Content\Search\Index
/**
* @brief Elasticsearch version requirements
*/
const MINIMUM_VERSION = '6.0.0';
const UNSUPPORTED_VERSION = '7.7.0';
const MINIMUM_VERSION = '7.2.0';
const UNSUPPORTED_VERSION = '8.0.0';
/**
* @brief The server URL
@@ -61,7 +61,7 @@ class _Index extends \IPS\Content\Search\Index
$analyzer = key( $settings['analysis']['analyzer'] );
}
$this->url->request()->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url )->delete();
$definition = array(
'settings' => $settings,
@@ -106,6 +106,8 @@ class _Index extends \IPS\Content\Search\Index
'index_comments' => array( 'type' => 'long' ),
'index_reviews' => array( 'type' => 'long' ),
'index_participants' => array( 'type' => 'long' ),
'index_is_anon' => array( 'type' => 'byte' ),
'index_item_solved' => array( 'type' => 'byte' )
)
)
)
@@ -113,7 +115,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$response = $this->url->setQueryString( 'include_type_name', 'true' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setQueryString( 'include_type_name', 'true' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
if( $response->httpResponseCode != 200 )
{
@@ -122,7 +124,7 @@ class _Index extends \IPS\Content\Search\Index
}
catch( \Exception $e )
{
$response = $this->url->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
$response = \IPS\Content\Search\Elastic\Index::request( $this->url )->setHeaders( array( 'Content-Type' => 'application/json' ) )->put( json_encode( $definition ) );
}
}
@@ -137,16 +139,15 @@ class _Index extends \IPS\Content\Search\Index
if ( $indexData = parent::indexData( $object ) )
{
$indexData['index_permissions'] = explode( ',', $indexData['index_permissions'] );
$indexData['index_tags'] = $indexData['index_tags'] ? explode( ',', $indexData['index_tags'] ) : array();
$indexData['index_is_last_comment'] = (bool) $indexData['index_is_last_comment'];
if ( $object instanceof \IPS\Content\Item )
{
$indexData += $this->metaCounts( $object );
$indexData = array_merge( $indexData, $this->metaData( $object ) );
}
else
{
$indexData += $this->metaCounts( $object->item() );
$indexData = array_merge( $indexData, $this->metaData( $object->item() ) );
}
return $indexData;
@@ -181,7 +182,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$r = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get()->decodeJson();
if ( $r['found'] )
{
$existingData = $r['_source'];
@@ -204,9 +205,7 @@ class _Index extends \IPS\Content\Search\Index
}
/* Insert into index */
$r = $this->url
->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) )
->request( \IPS\LONG_REQUEST_TIMEOUT )
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $object ) ), \IPS\LONG_REQUEST_TIMEOUT )
->setHeaders( array( 'Content-Type' => 'application/json' ) )
->put( json_encode( $indexData ) );
@@ -215,6 +214,11 @@ class _Index extends \IPS\Content\Search\Index
throw new \IPS\Content\Search\Elastic\Exception( $error['type'] . ': ' . $error['reason'] );
}
if ( $resetLastComment )
{
$this->resetLastComment( array( $indexData['index_class'] ), $indexData['index_item_id'] );
}
/* Views / Comments / Reviews */
if ( $object instanceof \IPS\Content\Item )
{
@@ -224,7 +228,8 @@ class _Index extends \IPS\Content\Search\Index
{
$item = $object->item();
}
$this->rebuildMetaCounts( $item, $indexData['index_is_last_comment'] ? $this->getIndexId( $object ) : NULL );
$this->rebuildMetaData( $item );
}
catch ( \IPS\Http\Request\Exception $e )
{
@@ -234,11 +239,6 @@ class _Index extends \IPS\Content\Search\Index
{
\IPS\Log::log( $e, 'elasticsearch_response_error' );
}
if ( $resetLastComment )
{
$this->resetLastComment( array( $indexData['index_class'] ), $indexData['index_item_id'] );
}
}
}
}
@@ -253,7 +253,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
/* This request *intentionally* goes to _tasks and not (ourpath)/_tasks */
$response = $this->url->setPath( '/_tasks' )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get();
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_tasks' ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get();
if( $error = $this->getResponseError( $response ) )
{
@@ -278,7 +278,7 @@ class _Index extends \IPS\Content\Search\Index
continue;
}
$this->url->setPath( '/.tasks/task' . $taskId )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/.tasks/task' . $taskId ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->delete();
}
}
}
@@ -298,7 +298,7 @@ class _Index extends \IPS\Content\Search\Index
* @param \IPS\Content\Searchable $item The content item
* @return void
*/
protected function metaCounts( $item )
protected function metaData( $item )
{
$databaseColumnId = $item::$databaseColumnId;
@@ -318,7 +318,9 @@ class _Index extends \IPS\Content\Search\Index
return array(
'index_comments' => $item->mapped('num_comments'),
'index_reviews' => $item->mapped('num_reviews'),
'index_participants' => $participants
'index_participants' => $participants,
'index_tags' => $item->tags(),
'index_prefix' => $item->prefix(),
);
}
@@ -326,10 +328,9 @@ class _Index extends \IPS\Content\Search\Index
* Rebuild the comment / review counts for an item
*
* @param \IPS\Content\Searchable $item The content item
* @param string $lastCommentIndexId The index ID of the last comment, if you want to reset others
* @return void
*/
protected function rebuildMetaCounts( $item, $lastCommentIndexId = NULL )
protected function rebuildMetaData( $item )
{
$databaseColumnId = $item::$databaseColumnId;
$class = \get_class( $item );
@@ -347,12 +348,12 @@ class _Index extends \IPS\Content\Search\Index
{
$updates = array();
$params = array();
foreach ( $this->metaCounts( $item ) as $k => $v )
foreach ( $this->metaData( $item ) as $k => $v )
{
if ( \is_array( $v ) )
{
$updates[] = "ctx._source.{$k} = params.param_{$k};";
$params[ 'param_' . $k ] = array_map( 'intval', $v );
$params[ 'param_' . $k ] = ( $k !== 'index_tags' ? array_map( 'intval', $v ) : $v );
}
elseif ( \is_null( $v ) )
{
@@ -365,12 +366,8 @@ class _Index extends \IPS\Content\Search\Index
$params['param_' . $k ] = \intval( $v );
}
}
if ( !\is_null( $lastCommentIndexId ) )
{
$updates[] = "ctx._source.index_is_last_comment = ( ctx._id == params.param_lastcommentid );";
$params['param_lastcommentid'] = $lastCommentIndexId;
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request( \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ), \IPS\LONG_REQUEST_TIMEOUT )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => implode( ' ', $updates ),
'lang' => 'painless',
@@ -429,7 +426,7 @@ class _Index extends \IPS\Content\Search\Index
if ( !( $object instanceof \IPS\Content\Item ) )
{
$this->rebuildMetaCounts( $object->item() );
$this->rebuildMetaData( $object->item() );
$itemClass = \get_class( $object->item() );
$itemIdColumn = $itemClass::$databaseColumnId;
@@ -465,14 +462,14 @@ class _Index extends \IPS\Content\Search\Index
try
{
$indexId = \strtolower( str_replace( '\\', '_', \substr( $class, 4 ) ) ) . '-' . $id;
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $indexId )->request()->delete();
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $indexId ) )->delete();
if ( is_subclass_of( $class, 'IPS\Content\Item' ) )
{
if ( isset( $class::$commentClass ) )
{
$commentClass = $class::$commentClass;
$response = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$response = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -495,7 +492,7 @@ class _Index extends \IPS\Content\Search\Index
if ( isset( $class::$reviewClass ) )
{
$reviewClass = $class::$reviewClass;
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'bool' => array(
'must' => array(
@@ -565,15 +562,15 @@ class _Index extends \IPS\Content\Search\Index
'index_author' => $authorId
)
);
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
}
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => $query
) ) );
}
else
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'term' => array(
'index_class' => $class
@@ -677,7 +674,7 @@ class _Index extends \IPS\Content\Search\Index
if ( \count( $updates ) )
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => implode( ' ', $updates ),
'lang' => 'painless',
@@ -697,8 +694,8 @@ class _Index extends \IPS\Content\Search\Index
'index_author' => 0
)
);
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_permissions.add( 'm' + ctx._source.index_author );",
'lang' => 'painless'
@@ -727,8 +724,8 @@ class _Index extends \IPS\Content\Search\Index
)
);
}
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_hidden = params.newHiddenStatus;",
'lang' => 'painless',
@@ -766,7 +763,7 @@ class _Index extends \IPS\Content\Search\Index
{
return array(
'bool' => array(
'must' => array( $conditions )
'must' => $conditions
)
);
}
@@ -795,7 +792,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $item ) . '/_update' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $item ) . '/_update' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'doc' => $update
) ) );
@@ -827,7 +824,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_delete_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'query' => array(
'range' => array(
'index_date_updated' => array(
@@ -861,7 +858,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
/* Remove the flag */
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_is_last_comment = false;",
'lang' => 'painless'
@@ -921,15 +918,15 @@ class _Index extends \IPS\Content\Search\Index
if ( $lastComment = $item->comments( 1, 0, 'date', 'desc', NULL, FALSE, NULL, $where ) AND $lastComment instanceof \IPS\Content\Searchable )
{
/* Set that it is the latest comment */
$r = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/' . $this->getIndexId( $lastComment ) . '/_update' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update/' . $this->getIndexId( $lastComment ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'doc' => array(
'index_is_last_comment' => true
)
) ) );
/* And set the updated time on the main item (done as _update_by_query because it might not exist if the first comment is required) */
$indexDataForLastComment = $this->indexData( $lastComment );
$this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false', 'scroll_size' => \IPS\Settings::i()->search_index_maxresults ) )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
$r = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_update_by_query' )->setQueryString( array( 'conflicts' => 'proceed', 'wait_for_completion' => 'false' ) ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->post( json_encode( array(
'script' => array(
'source' => "ctx._source.index_date_updated = params.dateUpdated; ctx._source.index_date_commented = params.dateCommented;",
'lang' => 'painless',
@@ -1001,7 +998,7 @@ class _Index extends \IPS\Content\Search\Index
);
/* Get the count */
$count = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$count = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 0,
'query' => $query
) ) )->decodeJson();
@@ -1012,7 +1009,7 @@ class _Index extends \IPS\Content\Search\Index
}
/* Now get the unique item ids */
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'aggs' => array(
'itemIds' => array(
'terms' => array(
@@ -1054,7 +1051,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'query' => array(
'bool' => array(
'filter' => array(
@@ -1103,7 +1100,7 @@ class _Index extends \IPS\Content\Search\Index
{
try
{
$results = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_doc/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'size' => 1,
'sort' => array( array( 'index_date_updated' => 'asc' ) )
) ) )->decodeJson();
@@ -1139,7 +1136,7 @@ class _Index extends \IPS\Content\Search\Index
try
{
$results = $this->url->setPath( '/_analyze' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
$results = \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_analyze' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( json_encode( array(
'analyzer' => $analyzer,
'text' => implode( ' ', $terms )
) ) )->decodeJson();
@@ -1172,6 +1169,23 @@ class _Index extends \IPS\Content\Search\Index
return FALSE;
}
/**
* Wrapper to account for log in where needed
*
* @param $url
* @param null $timeout
* @return \IPS\Http\Request\Curl|Socket
*/
public static function request( $url, $timeout=NULL )
{
if ( \IPS\ELASTICSEARCH_USER or \IPS\ELASTICSEARCH_PASSWORD )
{
return $url->request( $timeout )->login( \IPS\ELASTICSEARCH_USER, \IPS\ELASTICSEARCH_PASSWORD );
}
return $url->request( $timeout );
}
/**
* Check response to see if an error was produced
*
system/Content/Search/Elastic/MassIndexer.php
+1 -1
View File
@@ -73,7 +73,7 @@ class _MassIndexer extends \IPS\Content\Search\Elastic\Index
}
$json = implode( "\n", $json );
$this->url->setPath( '/_bulk' )->request()->setHeaders( array( 'Content-Type' => 'application/x-ndjson' ) )->post( $json . "\n" );
\IPS\Content\Search\Elastic\Index::request( $this->url->setPath( '/_bulk' ) )->setHeaders( array( 'Content-Type' => 'application/x-ndjson' ) )->post( $json . "\n" );
}
}
}
system/Content/Search/Elastic/Query.php
+173 -15
View File
@@ -69,7 +69,12 @@ class _Query extends \IPS\Content\Search\Query
parent::__construct( $member );
$this->url = $url;
}
/**
* @var bool Stores whether or not any content type needs to be limited by the latest comment
*/
protected $lastCommentMustBeTrue = false;
/**
* Filter by multiple content types
*
@@ -198,11 +203,14 @@ class _Query extends \IPS\Content\Search\Query
/* Only last comment? */
if ( $filter->onlyLastComment )
{
/* Set this for when the search JSON is built */
$this->lastCommentMustBeTrue = true;
$conditions[] = array(
'term' => array( 'index_is_last_comment' => true )
);
}
/* Put it together */
if( \count( $conditions ) )
{
@@ -215,7 +223,7 @@ class _Query extends \IPS\Content\Search\Query
{
$this->filters[] = array(
'bool' => array(
( $type ? 'should' : 'must_not' ) => array( $contentFilterConditions )
( $type ? 'should' : 'must_not' ) => $contentFilterConditions
)
);
}
@@ -363,7 +371,79 @@ class _Query extends \IPS\Content\Search\Query
/* Return for daisy chaining */
return $this;
}
/**
* Stores the "more like this" Content object for search()
* @param null
*/
protected $moreLikeThis = NULL;
/**
* Filter by more like this
* @param \IPS\Content\Searchable $object
*/
public function filterByMoreLikeThis( \IPS\Content\Searchable $object )
{
$index = new \IPS\Content\Search\Elastic\Index( $this->url );
$this->moreLikeThis = $index->getIndexId( ( $object instanceof \IPS\Content\Item and $object::$firstCommentRequired ) ? $object->firstComment() : $object );
/* Some container types cannot be cached */
$classes = [];
$conditions = [];
foreach ( \IPS\Application::allExtensions( 'core', 'ContentRouter', FALSE ) as $object )
{
$classes = array_merge( $object->classes, $classes );
}
/* Some nodes have complex permissions, so avoid any permission issues by not selecting these */
foreach( $classes as $itemClass )
{
if ( isset( $itemClass::$containerNodeClass ) )
{
$containerClass = $itemClass::$containerNodeClass;
$blockIds = [];
if ( $customNodes = $containerClass::customPermissionNodes() )
{
foreach ( $customNodes as $key => $ids )
{
if ( $key !== 'count' )
{
$blockIds = array_merge( $blockIds, $ids );
}
}
}
if ( \count( $blockIds ) )
{
$conditions[] = array(
'bool' => array(
'filter' => array(
array(
'terms' => array( 'index_container_class' => [ $containerClass ] )
),
array(
'terms' => array( 'index_container_id' => $blockIds )
),
)
)
);
}
if ( \count( $conditions ) )
{
foreach( $conditions as $condition )
{
$this->mustNot[] = $condition;
}
}
}
}
/* Only show non-hidden and approved items */
$this->setHiddenFilter( static::HIDDEN_VISIBLE );
}
/**
* Filter by container class
*
@@ -558,9 +638,7 @@ class _Query extends \IPS\Content\Search\Query
if ( $includeMembers and $followed = iterator_to_array( \IPS\Db::i()->select( 'follow_rel_id', 'core_follow', array( 'follow_app=? AND follow_area=? AND follow_member_id=?', 'core', 'member', $this->member->member_id ), 'follow_rel_id asc' ) ) )
{
$conditions[] = array(
array(
'terms' => array( 'index_author' => $followed )
)
'terms' => array( 'index_author' => $followed )
);
}
@@ -799,7 +877,35 @@ class _Query extends \IPS\Content\Search\Query
return $this;
}
/**
* Filter by solved
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterBySolved()
{
$this->filters[] = array(
'term' => array( 'index_item_solved' => 1 )
);
return $this;
}
/**
* Filter by Unsolved
*
* @return \IPS\Content\Search\Query (for daisy chaining)
*/
public function filterByUnsolved()
{
$this->filters[] = array(
'term' => array( 'index_item_solved' => 0 )
);
return $this;
}
/**
* Filter by start date
*
@@ -922,7 +1028,35 @@ class _Query extends \IPS\Content\Search\Query
return $this;
}
/**
* Debug by itemId, used for debugging purposes only
* @note No permission checks run, do not use in production
* @param $itemId
* @return void
*/
public function debugByItemId( $itemId )
{
$array = array(
'query' => [
'term' => [
'index_item_id' => [
'value' => \intval( $itemId )
]
]
],
'sort' => $this->sort ?: array(),
'from' => 0,
'size' => 50,
);
$json = json_encode( $array, JSON_PARTIAL_OUTPUT_ON_ERROR );
return \IPS\Content\Search\Elastic\Index::request( $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' ) )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
}
/**
* Search
*
@@ -1017,6 +1151,12 @@ class _Query extends \IPS\Content\Search\Query
)
)
);
/* If we're not searching with a term, then just show the title record, not comments */
if ( $term === NULL and ! $this->lastCommentMustBeTrue )
{
$must[] = ['exists' => ['field' => 'index_title']];
}
}
/* Put that with the rest of the conditions */
@@ -1050,7 +1190,25 @@ class _Query extends \IPS\Content\Search\Query
$filters[] = array( 'term' => array( 'index_hidden' => $this->hiddenStatuses ) );
}
}
$searchUrl = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' );
if ( $this->moreLikeThis )
{
$must[] = [
'more_like_this' => [
'fields' => [ 'index_title' ],
'like' => [ '_index' => \IPS\Settings::i()->search_elastic_index, '_id' => $this->moreLikeThis ],
'min_term_freq' => 1,
'max_query_terms' => 12,
'min_word_length' => 3,
]
];
/* Let's enforce a relatively short timeout to prevent slow ES server from dragging down topics */
$searchUrl = $searchUrl->setQueryString( 'timeout', '3s' );
}
/* Peform the search */
try
{
@@ -1062,9 +1220,9 @@ class _Query extends \IPS\Content\Search\Query
'filter' => $filters
)
);
/* Add the time decay */
if ( \IPS\Settings::i()->search_decay_factor and !$this->sort )
if ( \IPS\Settings::i()->search_decay_factor and !$this->sort and ! $this->sort )
{
$query = array(
'function_score' => array(
@@ -1081,7 +1239,7 @@ class _Query extends \IPS\Content\Search\Query
}
/* Add the self boost */
if ( \IPS\Settings::i()->search_elastic_self_boost and $this->member->member_id and !$this->sort )
if ( \IPS\Settings::i()->search_elastic_self_boost and $this->member->member_id and !$this->sort and ! $this->moreLikeThis )
{
$query = array(
'function_score' => array(
@@ -1117,10 +1275,10 @@ class _Query extends \IPS\Content\Search\Query
}
/* Make the call! */
$return = $this->url->setPath( $this->url->data[ \IPS\Http\Url::COMPONENT_PATH ] . '/_search' )->request()->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
$return = \IPS\Content\Search\Elastic\Index::request( $searchUrl )->setHeaders( array( 'Content-Type' => 'application/json' ) )->get( $json )->decodeJson();
if ( isset( $return['error'] ) )
{
\IPS\Log::log( print_r( $return['error'], TRUE ), 'elasticsearch' );
\IPS\Log::log( print_r( array_merge( $array, ['error' => $return['error'] ] ), TRUE ), 'elasticsearch' );
return new \IPS\Content\Search\Results( array(), 0 );
}