See Release Notes
Long Term Support Release
<?php // This file is part of Moodle - http://moodle.org/ // // Moodle is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // Moodle is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with Moodle. If not, see <http://www.gnu.org/licenses/>. /** * Solr engine. * * @package search_solr * @copyright 2015 Daniel Neis Araujo * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ namespace search_solr; defined('MOODLE_INTERNAL') || die(); /** * Solr engine. * * @package search_solr * @copyright 2015 Daniel Neis Araujo * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class engine extends \core_search\engine { /** * @var string The date format used by solr. */ const DATE_FORMAT = 'Y-m-d\TH:i:s\Z'; /** * @var int Commit documents interval (number of miliseconds). */ const AUTOCOMMIT_WITHIN = 15000; /** * The maximum number of results to fetch at a time. */ const QUERY_SIZE = 120; /** * Highlighting fragsize. Slightly larger than output size (500) to allow for ... appending. */ const FRAG_SIZE = 510; /** * Marker for the start of a highlight. */ const HIGHLIGHT_START = '@@HI_S@@'; /** * Marker for the end of a highlight. */ const HIGHLIGHT_END = '@@HI_E@@'; /** @var float Boost value for matching course in location-ordered searches */ const COURSE_BOOST = 1; /** @var float Boost value for matching context (in addition to course boost) */ const CONTEXT_BOOST = 0.5; /** * @var \SolrClient */ protected $client = null; /** * @var bool True if we should reuse SolrClients, false if not. */ protected $cacheclient = true; /** * @var \curl Direct curl object. */ protected $curl = null; /** * @var array Fields that can be highlighted. */ protected $highlightfields = array('title', 'content', 'description1', 'description2'); /** * @var int Number of total docs reported by Sorl for the last query. */ protected $totalenginedocs = 0; /** * @var int Number of docs we have processed for the last query. */ protected $processeddocs = 0; /** * @var int Number of docs that have been skipped while processing the last query. */ protected $skippeddocs = 0; /** * Solr server major version. * * @var int */ protected $solrmajorversion = null; /** * Initialises the search engine configuration. *> * @param bool $alternateconfiguration If true, use alternate configuration settings* @return void */< public function __construct() { < parent::__construct();> public function __construct(bool $alternateconfiguration = false) { > parent::__construct($alternateconfiguration);$curlversion = curl_version(); if (isset($curlversion['version']) && stripos($curlversion['version'], '7.35.') === 0) { // There is a flaw with curl 7.35.0 that causes problems with client reuse. $this->cacheclient = false; } } /** * Prepares a Solr query, applies filters and executes it returning its results. * * @throws \core_search\engine_exception * @param \stdClass $filters Containing query and filters. * @param \stdClass $accessinfo Information about areas user can access. * @param int $limit The maximum number of results to return. * @return \core_search\document[] Results or false if no results */ public function execute_query($filters, $accessinfo, $limit = 0) { global $USER; if (empty($limit)) { $limit = \core_search\manager::MAX_RESULTS; } // If there is any problem we trigger the exception as soon as possible. $client = $this->get_search_client(); // Create the query object. $query = $this->create_user_query($filters, $accessinfo); // If the query cannot have results, return none. if (!$query) { return []; } // We expect good match rates, so for our first get, we will get a small number of records. // This significantly speeds solr response time for first few pages. $query->setRows(min($limit * 3, static::QUERY_SIZE)); $response = $this->get_query_response($query); // Get count data out of the response, and reset our counters. list($included, $found) = $this->get_response_counts($response); $this->totalenginedocs = $found; $this->processeddocs = 0; $this->skippeddocs = 0; if ($included == 0 || $this->totalenginedocs == 0) { // No results. return array(); } // Get valid documents out of the response. $results = $this->process_response($response, $limit); // We have processed all the docs in the response at this point. $this->processeddocs += $included; // If we haven't reached the limit, and there are more docs left in Solr, lets keep trying. while (count($results) < $limit && ($this->totalenginedocs - $this->processeddocs) > 0) { // Offset the start of the query, and since we are making another call, get more per call. $query->setStart($this->processeddocs); $query->setRows(static::QUERY_SIZE); $response = $this->get_query_response($query); list($included, $found) = $this->get_response_counts($response); if ($included == 0 || $found == 0) { // No new results were found. Found being empty would be weird, so we will just return. return $results; } $this->totalenginedocs = $found; // Get the new response docs, limiting to remaining we need, then add it to the end of the results array. $newdocs = $this->process_response($response, $limit - count($results)); $results = array_merge($results, $newdocs); // Add to our processed docs count. $this->processeddocs += $included; } return $results; } /** * Takes a query and returns the response in SolrObject format. * * @param SolrQuery $query Solr query object. * @return SolrObject|false Response document or false on error. */ protected function get_query_response($query) { try { return $this->get_search_client()->query($query)->getResponse(); } catch (\SolrClientException $ex) { debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER); $this->queryerror = $ex->getMessage(); return false; } catch (\SolrServerException $ex) { debugging('Error executing the provided query: ' . $ex->getMessage(), DEBUG_DEVELOPER); $this->queryerror = $ex->getMessage(); return false; } } /** * Returns the total number of documents available for the most recently call to execute_query. * * @return int */ public function get_query_total_count() { // Return the total engine count minus the docs we have determined are bad. return $this->totalenginedocs - $this->skippeddocs; } /** * Returns count information for a provided response. Will return 0, 0 for invalid or empty responses. * * @param SolrDocument $response The response document from Solr. * @return array A two part array. First how many response docs are in the response. * Second, how many results are vailable in the engine. */ protected function get_response_counts($response) { $found = 0; $included = 0; if (isset($response->grouped->solr_filegroupingid->ngroups)) { // Get the number of results for file grouped queries. $found = $response->grouped->solr_filegroupingid->ngroups; $included = count($response->grouped->solr_filegroupingid->groups); } else if (isset($response->response->numFound)) { // Get the number of results for standard queries. $found = $response->response->numFound; if ($found > 0 && is_array($response->response->docs)) { $included = count($response->response->docs); } } return array($included, $found); } /** * Prepares a new query object with needed limits, filters, etc. * * @param \stdClass $filters Containing query and filters. * @param \stdClass $accessinfo Information about contexts the user can access * @return \SolrDisMaxQuery|null Query object or null if they can't get any results */ protected function create_user_query($filters, $accessinfo) { global $USER; // Let's keep these changes internal. $data = clone $filters; $query = new \SolrDisMaxQuery(); $this->set_query($query, self::replace_underlines($data->q)); $this->add_fields($query); // Search filters applied, we don't cache these filters as we don't want to pollute the cache with tmp filters // we are really interested in caching contexts filters instead. if (!empty($data->title)) { $query->addFilterQuery('{!field cache=false f=title}' . $data->title); } if (!empty($data->areaids)) { // If areaids are specified, we want to get any that match. $query->addFilterQuery('{!cache=false}areaid:(' . implode(' OR ', $data->areaids) . ')'); } if (!empty($data->courseids)) { $query->addFilterQuery('{!cache=false}courseid:(' . implode(' OR ', $data->courseids) . ')'); } if (!empty($data->groupids)) { $query->addFilterQuery('{!cache=false}groupid:(' . implode(' OR ', $data->groupids) . ')'); } if (!empty($data->userids)) { $query->addFilterQuery('{!cache=false}userid:(' . implode(' OR ', $data->userids) . ')'); } if (!empty($data->timestart) or !empty($data->timeend)) { if (empty($data->timestart)) { $data->timestart = '*'; } else { $data->timestart = \search_solr\document::format_time_for_engine($data->timestart); } if (empty($data->timeend)) { $data->timeend = '*'; } else { $data->timeend = \search_solr\document::format_time_for_engine($data->timeend); } // No cache. $query->addFilterQuery('{!cache=false}modified:[' . $data->timestart . ' TO ' . $data->timeend . ']'); } // Restrict to users who are supposed to be able to see a particular result. $query->addFilterQuery('owneruserid:(' . \core_search\manager::NO_OWNER_ID . ' OR ' . $USER->id . ')'); // And finally restrict it to the context where the user can access, we want this one cached. // If the user can access all contexts $usercontexts value is just true, we don't need to filter // in that case. if (!$accessinfo->everything && is_array($accessinfo->usercontexts)) { // Join all area contexts into a single array and implode. $allcontexts = array(); foreach ($accessinfo->usercontexts as $areaid => $areacontexts) { if (!empty($data->areaids) && !in_array($areaid, $data->areaids)) { // Skip unused areas. continue; } foreach ($areacontexts as $contextid) { // Ensure they are unique. $allcontexts[$contextid] = $contextid; } } if (empty($allcontexts)) { // This means there are no valid contexts for them, so they get no results. return null; } $query->addFilterQuery('contextid:(' . implode(' OR ', $allcontexts) . ')'); } if (!$accessinfo->everything && $accessinfo->separategroupscontexts) { // Add another restriction to handle group ids. If there are any contexts using separate // groups, then results in that context will not show unless you belong to the group. // (Note: Access all groups is taken care of earlier, when computing these arrays.) // This special exceptions list allows for particularly pig-headed developers to create // multiple search areas within the same module, where one of them uses separate // groups and the other uses visible groups. It is a little inefficient, but this should // be rare. $exceptions = ''; if ($accessinfo->visiblegroupscontextsareas) { foreach ($accessinfo->visiblegroupscontextsareas as $contextid => $areaids) { $exceptions .= ' OR (contextid:' . $contextid . ' AND areaid:(' . implode(' OR ', $areaids) . '))'; } } if ($accessinfo->usergroups) { // Either the document has no groupid, or the groupid is one that the user // belongs to, or the context is not one of the separate groups contexts. $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' . 'groupid:(' . implode(' OR ', $accessinfo->usergroups) . ') OR ' . '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' . $exceptions); } else { // Either the document has no groupid, or the context is not a restricted one. $query->addFilterQuery('(*:* -groupid:[* TO *]) OR ' . '(*:* -contextid:(' . implode(' OR ', $accessinfo->separategroupscontexts) . '))' . $exceptions); } } if ($this->file_indexing_enabled()) { // Now group records by solr_filegroupingid. Limit to 3 results per group. $query->setGroup(true); $query->setGroupLimit(3); $query->setGroupNGroups(true); $query->addGroupField('solr_filegroupingid'); } else { // Make sure we only get text files, in case the index has pre-existing files. $query->addFilterQuery('type:'.\core_search\manager::TYPE_TEXT); } // If ordering by location, add in boost for the relevant course or context ids. if (!empty($filters->order) && $filters->order === 'location') { $coursecontext = $filters->context->get_course_context(); $query->addBoostQuery('courseid', $coursecontext->instanceid, self::COURSE_BOOST); if ($filters->context->contextlevel !== CONTEXT_COURSE) { // If it's a block or activity, also add a boost for the specific context id. $query->addBoostQuery('contextid', $filters->context->id, self::CONTEXT_BOOST); } } return $query; } /** * Prepares a new query by setting the query, start offset and rows to return. * * @param SolrQuery $query * @param object $q Containing query and filters. */ protected function set_query($query, $q) { // Set hightlighting. $query->setHighlight(true); foreach ($this->highlightfields as $field) { $query->addHighlightField($field); } $query->setHighlightFragsize(static::FRAG_SIZE); $query->setHighlightSimplePre(self::HIGHLIGHT_START); $query->setHighlightSimplePost(self::HIGHLIGHT_END); $query->setHighlightMergeContiguous(true); $query->setQuery($q); // A reasonable max. $query->setRows(static::QUERY_SIZE); } /** * Sets fields to be returned in the result. * * @param SolrDisMaxQuery|SolrQuery $query object. */ public function add_fields($query) { $documentclass = $this->get_document_classname(); $fields = $documentclass::get_default_fields_definition(); $dismax = false; if ($query instanceof \SolrDisMaxQuery) { $dismax = true; } foreach ($fields as $key => $field) { $query->addField($key); if ($dismax && !empty($field['mainquery'])) { // Add fields the main query should be run against. // Due to a regression in the PECL solr extension, https://bugs.php.net/bug.php?id=72740, // a boost value is required, even if it is optional; to avoid boosting one among other fields, // the explicit boost value will be the default one, for every field. $query->addQueryField($key, 1); } } } /** * Finds the key common to both highlighing and docs array returned from response. * @param object $response containing results. */ public function add_highlight_content($response) { if (!isset($response->highlighting)) { // There is no highlighting to add. return; } $highlightedobject = $response->highlighting; foreach ($response->response->docs as $doc) { $x = $doc->id; $highlighteddoc = $highlightedobject->$x; $this->merge_highlight_field_values($doc, $highlighteddoc); } } /** * Adds the highlighting array values to docs array values. * * @throws \core_search\engine_exception * @param object $doc containing the results. * @param object $highlighteddoc containing the highlighted results values. */ public function merge_highlight_field_values($doc, $highlighteddoc) { foreach ($this->highlightfields as $field) { if (!empty($doc->$field)) { // Check that the returned value is not an array. No way we can make this work with multivalued solr fields. if (is_array($doc->{$field})) { throw new \core_search\engine_exception('multivaluedfield', 'search_solr', '', $field); } if (!empty($highlighteddoc->$field)) { // Replace by the highlighted result. $doc->$field = reset($highlighteddoc->$field); } } } } /** * Filters the response on Moodle side. * * @param SolrObject $response Solr object containing the response return from solr server. * @param int $limit The maximum number of results to return. 0 for all. * @param bool $skipaccesscheck Don't use check_access() on results. Only to be used when results have known access. * @return array $results containing final results to be displayed. */ protected function process_response($response, $limit = 0, $skipaccesscheck = false) { global $USER; if (empty($response)) { return array(); } if (isset($response->grouped)) { return $this->grouped_files_process_response($response, $limit); } $userid = $USER->id; $noownerid = \core_search\manager::NO_OWNER_ID; $numgranted = 0; if (!$docs = $response->response->docs) { return array(); } $out = array(); if (!empty($response->response->numFound)) { $this->add_highlight_content($response); // Iterate through the results checking its availability and whether they are available for the user or not. foreach ($docs as $key => $docdata) { if ($docdata['owneruserid'] != $noownerid && $docdata['owneruserid'] != $userid) { // If owneruserid is set, no other user should be able to access this record. continue; } if (!$searcharea = $this->get_search_area($docdata->areaid)) { continue; } $docdata = $this->standarize_solr_obj($docdata); if ($skipaccesscheck) { $access = \core_search\manager::ACCESS_GRANTED; } else { $access = $searcharea->check_access($docdata['itemid']); } switch ($access) { case \core_search\manager::ACCESS_DELETED: $this->delete_by_id($docdata['id']); // Remove one from our processed and total counters, since we promptly deleted. $this->processeddocs--; $this->totalenginedocs--; break; case \core_search\manager::ACCESS_DENIED: $this->skippeddocs++; break; case \core_search\manager::ACCESS_GRANTED: $numgranted++; // Add the doc. $out[] = $this->to_document($searcharea, $docdata); break; } // Stop when we hit our limit. if (!empty($limit) && count($out) >= $limit) { break; } } } return $out; } /** * Processes grouped file results into documents, with attached matching files. * * @param SolrObject $response The response returned from solr server * @param int $limit The maximum number of results to return. 0 for all. * @return array Final results to be displayed. */ protected function grouped_files_process_response($response, $limit = 0) { // If we can't find the grouping, or there are no matches in the grouping, return empty. if (!isset($response->grouped->solr_filegroupingid) || empty($response->grouped->solr_filegroupingid->matches)) { return array(); } $numgranted = 0; $orderedids = array(); $completedocs = array(); $incompletedocs = array(); $highlightingobj = $response->highlighting; // Each group represents a "master document". $groups = $response->grouped->solr_filegroupingid->groups; foreach ($groups as $group) { $groupid = $group->groupValue; $groupdocs = $group->doclist->docs; $firstdoc = reset($groupdocs); if (!$searcharea = $this->get_search_area($firstdoc->areaid)) { // Well, this is a problem. continue; } // Check for access. $access = $searcharea->check_access($firstdoc->itemid); switch ($access) { case \core_search\manager::ACCESS_DELETED: // If deleted from Moodle, delete from index and then continue. $this->delete_by_id($firstdoc->id); // Remove one from our processed and total counters, since we promptly deleted. $this->processeddocs--; $this->totalenginedocs--; continue 2; break; case \core_search\manager::ACCESS_DENIED: // This means we should just skip for the current user. $this->skippeddocs++; continue 2; break; } $numgranted++; $maindoc = false; $fileids = array(); // Seperate the main document and any files returned. foreach ($groupdocs as $groupdoc) { if ($groupdoc->id == $groupid) { $maindoc = $groupdoc; } else if (isset($groupdoc->solr_fileid)) { $fileids[] = $groupdoc->solr_fileid; } } // Store the id of this group, in order, for later merging. $orderedids[] = $groupid; if (!$maindoc) { // We don't have the main doc, store what we know for later building. $incompletedocs[$groupid] = $fileids; } else { if (isset($highlightingobj->$groupid)) { // Merge the highlighting for this doc. $this->merge_highlight_field_values($maindoc, $highlightingobj->$groupid); } $docdata = $this->standarize_solr_obj($maindoc); $doc = $this->to_document($searcharea, $docdata); // Now we need to attach the result files to the doc. foreach ($fileids as $fileid) { $doc->add_stored_file($fileid); } $completedocs[$groupid] = $doc; } if (!empty($limit) && $numgranted >= $limit) { // We have hit the max results, we will just ignore the rest. break; } } $incompletedocs = $this->get_missing_docs($incompletedocs); $out = array(); // Now merge the complete and incomplete documents, in results order. foreach ($orderedids as $docid) { if (isset($completedocs[$docid])) { $out[] = $completedocs[$docid]; } else if (isset($incompletedocs[$docid])) { $out[] = $incompletedocs[$docid]; } } return $out; } /** * Retreive any missing main documents and attach provided files. * * The missingdocs array should be an array, indexed by document id, of main documents we need to retrieve. The value * associated to the key should be an array of stored_files or stored file ids to attach to the result document. * * Return array also indexed by document id. * * @param array() $missingdocs An array, indexed by document id, with arrays of files/ids to attach. * @return document[] */ protected function get_missing_docs($missingdocs) { if (empty($missingdocs)) { return array(); } $docids = array_keys($missingdocs); // Build a custom query that will get all the missing documents. $query = new \SolrQuery(); $this->set_query($query, '*'); $this->add_fields($query); $query->setRows(count($docids)); $query->addFilterQuery('{!cache=false}id:(' . implode(' OR ', $docids) . ')'); $response = $this->get_query_response($query); // We know the missing docs have already been checked for access, so don't recheck. $results = $this->process_response($response, 0, true); $out = array(); foreach ($results as $result) { $resultid = $result->get('id'); if (!isset($missingdocs[$resultid])) { // We got a result we didn't expect. Skip it. continue; } // Attach the files. foreach ($missingdocs[$resultid] as $filedoc) { $result->add_stored_file($filedoc); } $out[$resultid] = $result; } return $out; } /** * Returns a standard php array from a \SolrObject instance. * * @param \SolrObject $obj * @return array The returned document as an array. */ public function standarize_solr_obj(\SolrObject $obj) { $properties = $obj->getPropertyNames(); $docdata = array(); foreach($properties as $name) { // http://php.net/manual/en/solrobject.getpropertynames.php#98018. $name = trim($name); $docdata[$name] = $obj->offsetGet($name); } return $docdata; } /** * Adds a document to the search engine. * * This does not commit to the search engine. * * @param document $document * @param bool $fileindexing True if file indexing is to be used * @return bool */ public function add_document($document, $fileindexing = false) { $docdata = $document->export_for_engine(); if (!$this->add_solr_document($docdata)) { return false; } if ($fileindexing) { // This will take care of updating all attached files in the index. $this->process_document_files($document); } return true; } /**> * Adds a batch of documents to the engine at once. * Replaces underlines at edges of words in the content with spaces. > * * > * @param \core_search\document[] $documents Documents to add * For example '_frogs_' will become 'frogs', '_frogs and toads_' will become 'frogs and toads', > * @param bool $fileindexing If true, indexes files (these are done one at a time) * and 'frogs_and_toads' will be left as 'frogs_and_toads'. > * @return int[] Array of three elements: successfully processed, failed processed, batch count * > */ * The reason for this is that for italic content_to_text puts _italic_ underlines at the start > public function add_document_batch(array $documents, bool $fileindexing = false): array { * and end of the italicised phrase (not between words). Solr treats underlines as part of the > $docdatabatch = []; * word, which means that if you search for a word in italic then you can't find it. > foreach ($documents as $document) { * > $docdatabatch[] = $document->export_for_engine(); * @param string $str String to replace > } * @return string Replaced string > */ > $resultcounts = $this->add_solr_documents($docdatabatch); protected static function replace_underlines(string $str): string { > return preg_replace('~\b_|_\b~', '', $str); > // Files are processed one document at a time (if there are files it's slow anyway). } > if ($fileindexing) { > foreach ($documents as $document) { /** > // This will take care of updating all attached files in the index. * Adds a text document to the search engine. > $this->process_document_files($document); * > } * @param array $doc > } * @return bool > */ > return $resultcounts; protected function add_solr_document($doc) { > } $solrdoc = new \SolrInputDocument(); > > /**< * Adds a text document to the search engine.> * Creates a Solr document object.< * @param array $doc < * @return bool> * @param array $doc Array of document fields > * @return \SolrInputDocument Created document< protected function add_solr_document($doc) {> protected function create_solr_document(array $doc): \SolrInputDocument {$doc['content'] = self::replace_underlines($doc['content']); }> // Set all the fields.foreach ($doc as $field => $value) { $solrdoc->addField($field, $value); }> return $solrdoc; try { > } $result = $this->get_search_client()->addDocument($solrdoc, true, static::AUTOCOMMIT_WITHIN); > return true; > /** } catch (\SolrClientException $e) { > * Adds a text document to the search engine. debugging('Solr client error adding document with id ' . $doc['id'] . ': ' . $e->getMessage(), DEBUG_DEVELOPER); > * } catch (\SolrServerException $e) { > * @param array $doc // We only use the first line of the message, as it's a fully java stacktrace behind it. > * @return bool $msg = strtok($e->getMessage(), "\n"); > */ debugging('Solr server error adding document with id ' . $doc['id'] . ': ' . $msg, DEBUG_DEVELOPER); > protected function add_solr_document($doc) { } > $solrdoc = $this->create_solr_document($doc); >return false; } /**> * Adds multiple text documents to the search engine. * Index files attached to the docuemnt, ensuring the index matches the current document files. > * * > * @param array $docs Array of documents (each an array of fields) to add * For documents that aren't known to be new, we check the index for existing files. > * @return int[] Array of success, failure, batch count * - New files we will add. > * @throws \core_search\engine_exception * - Existing and unchanged files we will skip. > */ * - File that are in the index but not on the document will be deleted from the index. > protected function add_solr_documents(array $docs): array { * - Files that have changed will be re-indexed. > $solrdocs = []; * > foreach ($docs as $doc) { * @param document $document > $solrdocs[] = $this->create_solr_document($doc); */ > } protected function process_document_files($document) { > if (!$this->file_indexing_enabled()) { > try { return; > // Add documents in a batch and report that they all succeeded. } > $this->get_search_client()->addDocuments($solrdocs, true, static::AUTOCOMMIT_WITHIN); > return [count($solrdocs), 0, 1]; // Maximum rows to process at a time. > } catch (\SolrClientException $e) { $rows = 500; > // If there is an exception, fall through... > $donothing = true; // Get the attached files. > } catch (\SolrServerException $e) { $files = $document->get_files(); > // If there is an exception, fall through... > $donothing = true; // If this isn't a new document, we need to check the exiting indexed files. > } if (!$document->get_is_new()) { > // We do this progressively, so we can handle lots of files cleanly. > // When there is an error, we fall back to adding them individually so that we can report list($numfound, $indexedfiles) = $this->get_indexed_files($document, 0, $rows); > // which document(s) failed. Since it overwrites, adding the successful ones multiple $count = 0; > // times won't hurt. $idstodelete = array(); > $success = 0; > $failure = 0; do { > $batches = 0; // Go through each indexed file. We want to not index any stored and unchanged ones, delete any missing ones. > foreach ($docs as $doc) { foreach ($indexedfiles as $indexedfile) { > $result = $this->add_solr_document($doc); $fileid = $indexedfile->solr_fileid; > $batches++; > if ($result) { if (isset($files[$fileid])) { > $success++; // Check for changes that would mean we need to re-index the file. If so, just leave in $files. > } else { // Filelib does not guarantee time modified is updated, so we will check important values. > $failure++; if ($indexedfile->modified != $files[$fileid]->get_timemodified()) { > } continue; > } } > if (strcmp($indexedfile->title, $files[$fileid]->get_filename()) !== 0) { > return [$success, $failure, $batches]; continue; > } } > if ($indexedfile->solr_filecontenthash != $files[$fileid]->get_contenthash()) { > /**continue; } if ($indexedfile->solr_fileindexstatus == document::INDEXED_FILE_FALSE && $this->file_is_indexable($files[$fileid])) { // This means that the last time we indexed this file, filtering blocked it. // Current settings say it is indexable, so we will allow it to be indexed. continue; } // If the file is already indexed, we can just remove it from the files array and skip it. unset($files[$fileid]); } else { // This means we have found a file that is no longer attached, so we need to delete from the index. // We do it later, since this is progressive, and it could reorder results. $idstodelete[] = $indexedfile->id; } } $count += $rows; if ($count < $numfound) { // If we haven't hit the total count yet, fetch the next batch. list($numfound, $indexedfiles) = $this->get_indexed_files($document, $count, $rows); } } while ($count < $numfound); // Delete files that are no longer attached. foreach ($idstodelete as $id) { // We directly delete the item using the client, as the engine delete_by_id won't work on file docs. $this->get_search_client()->deleteById($id); } } // Now we can actually index all the remaining files. foreach ($files as $file) { $this->add_stored_file($document, $file); } } /** * Get the currently indexed files for a particular document, returns the total count, and a subset of files. * * @param document $document * @param int $start The row to start the results on. Zero indexed. * @param int $rows The number of rows to fetch * @return array A two element array, the first is the total number of availble results, the second is an array * of documents for the current request. */ protected function get_indexed_files($document, $start = 0, $rows = 500) { // Build a custom query that will get any document files that are in our solr_filegroupingid. $query = new \SolrQuery(); // We want to get all file records tied to a document. // For efficiency, we are building our own, stripped down, query. $query->setQuery('*'); $query->setRows($rows); $query->setStart($start); // We want a consistent sorting. $query->addSortField('id'); // We only want the bare minimum of fields. $query->addField('id'); $query->addField('modified'); $query->addField('title'); $query->addField('solr_fileid'); $query->addField('solr_filecontenthash'); $query->addField('solr_fileindexstatus'); $query->addFilterQuery('{!cache=false}solr_filegroupingid:(' . $document->get('id') . ')'); $query->addFilterQuery('type:' . \core_search\manager::TYPE_FILE); $response = $this->get_query_response($query); if (empty($response->response->numFound)) { return array(0, array()); } return array($response->response->numFound, $this->convert_file_results($response)); } /** * A very lightweight handler for getting information about already indexed files from a Solr response. * * @param SolrObject $responsedoc A Solr response document * @return stdClass[] An array of objects that contain the basic information for file processing. */ protected function convert_file_results($responsedoc) { if (!$docs = $responsedoc->response->docs) { return array(); } $out = array(); foreach ($docs as $doc) { // Copy the bare minimim needed info. $result = new \stdClass(); $result->id = $doc->id; $result->modified = document::import_time_from_engine($doc->modified); $result->title = $doc->title; $result->solr_fileid = $doc->solr_fileid; $result->solr_filecontenthash = $doc->solr_filecontenthash; $result->solr_fileindexstatus = $doc->solr_fileindexstatus; $out[] = $result; } return $out; } /** * Adds a file to the search engine. * * Notes about Solr and Tika indexing. We do not send the mime type, only the filename. * Tika has much better content type detection than Moodle, and we will have many more doc failures * if we try to send mime types. * * @param document $document * @param \stored_file $storedfile * @return void */ protected function add_stored_file($document, $storedfile) { $filedoc = $document->export_file_for_engine($storedfile); if (!$this->file_is_indexable($storedfile)) { // For files that we don't consider indexable, we will still place a reference in the search engine. $filedoc['solr_fileindexstatus'] = document::INDEXED_FILE_FALSE; $this->add_solr_document($filedoc); return; } $curl = $this->get_curl_object(); $url = $this->get_connection_url('/update/extract'); // Return results as XML. $url->param('wt', 'xml'); // This will prevent solr from automatically making fields for every tika output. $url->param('uprefix', 'ignored_'); // Control how content is captured. This will keep our file content clean of non-important metadata. $url->param('captureAttr', 'true'); // Move the content to a field for indexing. $url->param('fmap.content', 'solr_filecontent'); // These are common fields that matches the standard *_point dynamic field and causes an error. $url->param('fmap.media_white_point', 'ignored_mwp'); $url->param('fmap.media_black_point', 'ignored_mbp'); // Copy each key to the url with literal. // We place in a temp name then copy back to the true field, which prevents errors or Tika overwriting common field names. foreach ($filedoc as $key => $value) { // This will take any fields from tika that match our schema and discard them, so they don't overwrite ours. $url->param('fmap.'.$key, 'ignored_'.$key); // Place data in a tmp field. $url->param('literal.mdltmp_'.$key, $value); // Then move to the final field. $url->param('fmap.mdltmp_'.$key, $key); } // This sets the true filename for Tika. $url->param('resource.name', $storedfile->get_filename()); // A giant block of code that is really just error checking around the curl request. try { // We have to post the file directly in binary data (not using multipart) to avoid // Solr bug SOLR-15039 which can cause incorrect data when you use multipart upload. // Note this loads the whole file into memory; see limit in file_is_indexable().< $curl->setHeader('Content-Type: text/plain; charset=UTF-8');$result = $curl->post($url->out(false), $storedfile->get_content());< $curl->resetHeader();$code = $curl->get_errno(); $info = $curl->get_info(); // Now error handling. It is just informational, since we aren't tracking per file/doc results. if ($code != 0) { // This means an internal cURL error occurred error is in result. $message = 'Curl error '.$code.' while indexing file with document id '.$filedoc['id'].': '.$result.'.'; debugging($message, DEBUG_DEVELOPER); } else if (isset($info['http_code']) && ($info['http_code'] !== 200)) { // Unexpected HTTP response code. $message = 'Error while indexing file with document id '.$filedoc['id']; // Try to get error message out of msg or title if it exists. if (preg_match('|<str [^>]*name="msg"[^>]*>(.*?)</str>|i', $result, $matches)) { $message .= ': '.$matches[1]; } else if (preg_match('|<title[^>]*>([^>]*)</title>|i', $result, $matches)) { $message .= ': '.$matches[1]; } // This is a common error, happening whenever a file fails to index for any reason, so we will make it quieter. if (CLI_SCRIPT && !PHPUNIT_TEST) { mtrace($message); } } else { // Check for the expected status field. if (preg_match('|<int [^>]*name="status"[^>]*>(\d*)</int>|i', $result, $matches)) { // Now check for the expected status of 0, if not, error. if ((int)$matches[1] !== 0) { $message = 'Unexpected Solr status code '.(int)$matches[1]; $message .= ' while indexing file with document id '.$filedoc['id'].'.'; debugging($message, DEBUG_DEVELOPER); } else { // The document was successfully indexed. return; } } else { // We received an unprocessable response. $message = 'Unexpected Solr response while indexing file with document id '.$filedoc['id'].': '; $message .= strtok($result, "\n"); debugging($message, DEBUG_DEVELOPER); } } } catch (\Exception $e) { // There was an error, but we are not tracking per-file success, so we just continue on. debugging('Unknown exception while indexing file "'.$storedfile->get_filename().'".', DEBUG_DEVELOPER); } // If we get here, the document was not indexed due to an error. So we will index just the base info without the file. $filedoc['solr_fileindexstatus'] = document::INDEXED_FILE_ERROR; $this->add_solr_document($filedoc); } /** * Checks to see if a passed file is indexable. * * @param \stored_file $file The file to check * @return bool True if the file can be indexed */ protected function file_is_indexable($file) { if (!empty($this->config->maxindexfilekb) && ($file->get_filesize() > ($this->config->maxindexfilekb * 1024))) { // The file is too big to index. return false; } // Because we now load files into memory to index them in Solr, we also have to ensure that // we don't try to index anything bigger than the memory limit (less 100MB for safety). // Memory limit in cron is MEMORY_EXTRA which is usually 256 or 384MB but can be increased // in config, so this will allow files over 100MB to be indexed. $limit = ini_get('memory_limit'); if ($limit && $limit != -1) { $limitbytes = get_real_size($limit); if ($file->get_filesize() > $limitbytes) { return false; } } $mime = $file->get_mimetype(); if ($mime == 'application/vnd.moodle.backup') { // We don't index Moodle backup files. There is nothing usefully indexable in them. return false; } return true; } /** * Commits all pending changes. * * @return void */ protected function commit() { $this->get_search_client()->commit(); } /** * Do any area cleanup needed, and do anything to confirm contents. * * Return false to prevent the search area completed time and stats from being updated. * * @param \core_search\base $searcharea The search area that was complete * @param int $numdocs The number of documents that were added to the index * @param bool $fullindex True if a full index is being performed * @return bool True means that data is considered indexed */ public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) { $this->commit(); return true; } /** * Return true if file indexing is supported and enabled. False otherwise. * * @return bool */ public function file_indexing_enabled() { return (bool)$this->config->fileindexing; } /**< * Defragments the index. < * < * @return void < */ < public function optimize() { < $this->get_search_client()->optimize(1, true, false); < } < < /*** Deletes the specified document. * * @param string $id The document id to delete * @return void */ public function delete_by_id($id) { // We need to make sure we delete the item and all related files, which can be done with solr_filegroupingid. $this->get_search_client()->deleteByQuery('solr_filegroupingid:' . $id); $this->commit(); } /** * Delete all area's documents. * * @param string $areaid * @return void */ public function delete($areaid = null) { if ($areaid) { $this->get_search_client()->deleteByQuery('areaid:' . $areaid); } else { $this->get_search_client()->deleteByQuery('*:*'); } $this->commit(); } /** * Pings the Solr server using search_solr config * * @return true|string Returns true if all good or an error string. */ public function is_server_ready() { $configured = $this->is_server_configured(); if ($configured !== true) { return $configured; } // As part of the above we have already checked that we can contact the server. For pages // where performance is important, we skip doing a full schema check as well. if ($this->should_skip_schema_check()) { return true; } // Update schema if required/possible. $schemalatest = $this->check_latest_schema(); if ($schemalatest !== true) { return $schemalatest; } // Check that the schema is already set up. try {< $schema = new \search_solr\schema();> $schema = new schema($this);$schema->validate_setup(); } catch (\moodle_exception $e) { return $e->getMessage(); } return true; } /** * Is the solr server properly configured?. * * @return true|string Returns true if all good or an error string. */ public function is_server_configured() { if (empty($this->config->server_hostname) || empty($this->config->indexname)) { return 'No solr configuration found'; } if (!$client = $this->get_search_client(false)) { return get_string('engineserverstatus', 'search'); } try { if ($this->get_solr_major_version() < 4) { // Minimum solr 4.0. return get_string('minimumsolr4', 'search_solr'); } } catch (\SolrClientException $ex) { debugging('Solr client error: ' . html_to_text($ex->getMessage()), DEBUG_DEVELOPER); return get_string('engineserverstatus', 'search'); } catch (\SolrServerException $ex) { debugging('Solr server error: ' . html_to_text($ex->getMessage()), DEBUG_DEVELOPER); return get_string('engineserverstatus', 'search'); } return true; } /** * Returns the solr server major version. * * @return int */ public function get_solr_major_version() { if ($this->solrmajorversion !== null) { return $this->solrmajorversion; } // We should really ping first the server to see if the specified indexname is valid but // we want to minimise solr server requests as they are expensive. system() emits a warning // if it can not connect to the configured index in the configured server. $systemdata = @$this->get_search_client()->system(); $solrversion = $systemdata->getResponse()->offsetGet('lucene')->offsetGet('solr-spec-version'); $this->solrmajorversion = intval(substr($solrversion, 0, strpos($solrversion, '.'))); return $this->solrmajorversion; } /** * Checks if the PHP Solr extension is available. * * @return bool */ public function is_installed() { return function_exists('solr_get_version'); } /** * Returns the solr client instance. * * We don't reuse SolrClient if we are on libcurl 7.35.0, due to a bug in that version of curl. * * @throws \core_search\engine_exception * @param bool $triggerexception * @return \SolrClient */ protected function get_search_client($triggerexception = true) { global $CFG; // Type comparison as it is set to false if not available. if ($this->client !== null) { return $this->client; } $options = array( 'hostname' => $this->config->server_hostname, 'path' => '/solr/' . $this->config->indexname, 'login' => !empty($this->config->server_username) ? $this->config->server_username : '', 'password' => !empty($this->config->server_password) ? $this->config->server_password : '', 'port' => !empty($this->config->server_port) ? $this->config->server_port : '', 'secure' => !empty($this->config->secure) ? true : false, 'ssl_cert' => !empty($this->config->ssl_cert) ? $this->config->ssl_cert : '', 'ssl_key' => !empty($this->config->ssl_key) ? $this->config->ssl_key : '', 'ssl_keypassword' => !empty($this->config->ssl_keypassword) ? $this->config->ssl_keypassword : '', 'ssl_cainfo' => !empty($this->config->ssl_cainfo) ? $this->config->ssl_cainfo : '', 'ssl_capath' => !empty($this->config->ssl_capath) ? $this->config->ssl_capath : '', 'timeout' => !empty($this->config->server_timeout) ? $this->config->server_timeout : '30' ); if ($CFG->proxyhost && !is_proxybypass('http://' . $this->config->server_hostname . '/')) { $options['proxy_host'] = $CFG->proxyhost; if (!empty($CFG->proxyport)) { $options['proxy_port'] = $CFG->proxyport; } if (!empty($CFG->proxyuser) && !empty($CFG->proxypassword)) { $options['proxy_login'] = $CFG->proxyuser; $options['proxy_password'] = $CFG->proxypassword; } } if (!class_exists('\SolrClient')) { throw new \core_search\engine_exception('enginenotinstalled', 'search', '', 'solr'); } $client = new \SolrClient($options); if ($client === false && $triggerexception) { throw new \core_search\engine_exception('engineserverstatus', 'search'); } if ($this->cacheclient) { $this->client = $client; } return $client; } /** * Returns a curl object for conntecting to solr. * * @return \curl */ public function get_curl_object() { if (!is_null($this->curl)) { return $this->curl; } // Connection to Solr is allowed to use 'localhost' and other potentially blocked hosts/ports. $this->curl = new \curl(['ignoresecurity' => true]); $options = array(); // Build the SSL options. Based on pecl-solr and general testing. if (!empty($this->config->secure)) { if (!empty($this->config->ssl_cert)) { $options['CURLOPT_SSLCERT'] = $this->config->ssl_cert; $options['CURLOPT_SSLCERTTYPE'] = 'PEM'; } if (!empty($this->config->ssl_key)) { $options['CURLOPT_SSLKEY'] = $this->config->ssl_key; $options['CURLOPT_SSLKEYTYPE'] = 'PEM'; } if (!empty($this->config->ssl_keypassword)) { $options['CURLOPT_KEYPASSWD'] = $this->config->ssl_keypassword; } if (!empty($this->config->ssl_cainfo)) { $options['CURLOPT_CAINFO'] = $this->config->ssl_cainfo; } if (!empty($this->config->ssl_capath)) { $options['CURLOPT_CAPATH'] = $this->config->ssl_capath; } } // Set timeout as for Solr client. $options['CURLOPT_TIMEOUT'] = !empty($this->config->server_timeout) ? $this->config->server_timeout : '30'; $this->curl->setopt($options); if (!empty($this->config->server_username) && !empty($this->config->server_password)) { $authorization = $this->config->server_username . ':' . $this->config->server_password; $this->curl->setHeader('Authorization: Basic ' . base64_encode($authorization)); } return $this->curl; } /** * Return a Moodle url object for the server connection. * * @param string $path The solr path to append. * @return \moodle_url */ public function get_connection_url($path) { // Must use the proper protocol, or SSL will fail. $protocol = !empty($this->config->secure) ? 'https' : 'http'; $url = $protocol . '://' . rtrim($this->config->server_hostname, '/'); if (!empty($this->config->server_port)) { $url .= ':' . $this->config->server_port; } $url .= '/solr/' . $this->config->indexname . '/' . ltrim($path, '/'); return new \moodle_url($url); } /** * Solr includes group support in the execute_query function. * * @return bool True */ public function supports_group_filtering() { return true; } protected function update_schema($oldversion, $newversion) { // Construct schema.< $schema = new schema();> $schema = new schema($this);$cansetup = $schema->can_setup_server(); if ($cansetup !== true) { return $cansetup; } switch ($newversion) { // This version just requires a setup call to add new fields. case 2017091700: $setup = true; break; // If we don't know about the schema version we might not have implemented the // change correctly, so return. default: return get_string('schemaversionunknown', 'search'); } if ($setup) { $schema->setup(); } return true; } /** * Solr supports sort by location within course contexts or below. * * @param \context $context Context that the user requested search from * @return array Array from order name => display text */ public function get_supported_orders(\context $context) { $orders = parent::get_supported_orders($context); // If not within a course, no other kind of sorting supported. $coursecontext = $context->get_course_context(false); if ($coursecontext) { // Within a course or activity/block, support sort by location. $orders['location'] = get_string('order_location', 'search', $context->get_context_name()); } return $orders; } /** * Solr supports search by user id. * * @return bool True */ public function supports_users() { return true; } /**> * Solr supports adding documents in a batch. * Solr supports deleting the index for a context. > * * > * @return bool True * @param int $oldcontextid Context that has been deleted > */ * @return bool True to indicate that any data was actually deleted > public function supports_add_document_batch(): bool { * @throws \core_search\engine_exception > return true; */ > } public function delete_index_for_context(int $oldcontextid) { > $client = $this->get_search_client(); > /**try { $client->deleteByQuery('contextid:' . $oldcontextid); $client->commit(true); return true; } catch (\Exception $e) { throw new \core_search\engine_exception('error_solr', 'search_solr', '', $e->getMessage()); } } /** * Solr supports deleting the index for a course. * * @param int $oldcourseid * @return bool True to indicate that any data was actually deleted * @throws \core_search\engine_exception */ public function delete_index_for_course(int $oldcourseid) { $client = $this->get_search_client(); try { $client->deleteByQuery('courseid:' . $oldcourseid); $client->commit(true); return true; } catch (\Exception $e) { throw new \core_search\engine_exception('error_solr', 'search_solr', '', $e->getMessage()); }> } } > } > /** > * Checks if an alternate configuration has been defined. > * > * @return bool True if alternate configuration is available > */ > public function has_alternate_configuration(): bool { > return !empty($this->config->alternateserver_hostname) && > !empty($this->config->alternateindexname) && > !empty($this->config->alternateserver_port);