[Summary view]

Differences Between: [Versions 310 and 311] [Versions 310 and 400] [Versions 310 and 401] [Versions 310 and 402] [Versions 310 and 403] [Versions 39 and 310]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * Search subsystem manager.
  19   *
  20   * @package   core_search
  21   * @copyright Prateek Sachan {@link http://prateeksachan.com}
  22   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  namespace core_search;
  26  
  27  defined('MOODLE_INTERNAL') || die;
  28  
  29  require_once($CFG->dirroot . '/lib/accesslib.php');
  30  
  31  /**
  32   * Search subsystem manager.
  33   *
  34   * @package   core_search
  35   * @copyright Prateek Sachan {@link http://prateeksachan.com}
  36   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  37   */
  38  class manager {
  39  
  40      /**
  41       * @var int Text contents.
  42       */
  43      const TYPE_TEXT = 1;
  44  
  45      /**
  46       * @var int File contents.
  47       */
  48      const TYPE_FILE = 2;
  49  
  50      /**
  51       * @var int User can not access the document.
  52       */
  53      const ACCESS_DENIED = 0;
  54  
  55      /**
  56       * @var int User can access the document.
  57       */
  58      const ACCESS_GRANTED = 1;
  59  
  60      /**
  61       * @var int The document was deleted.
  62       */
  63      const ACCESS_DELETED = 2;
  64  
  65      /**
  66       * @var int Maximum number of results that will be retrieved from the search engine.
  67       */
  68      const MAX_RESULTS = 100;
  69  
  70      /**
  71       * @var int Number of results per page.
  72       */
  73      const DISPLAY_RESULTS_PER_PAGE = 10;
  74  
  75      /**
  76       * @var int The id to be placed in owneruserid when there is no owner.
  77       */
  78      const NO_OWNER_ID = 0;
  79  
  80      /**
  81       * @var float If initial query takes longer than N seconds, this will be shown in cron log.
  82       */
  83      const DISPLAY_LONG_QUERY_TIME = 5.0;
  84  
  85      /**
  86       * @var float Adds indexing progress within one search area to cron log every N seconds.
  87       */
  88      const DISPLAY_INDEXING_PROGRESS_EVERY = 30.0;
  89  
  90      /**
  91       * @var int Context indexing: normal priority.
  92       */
  93      const INDEX_PRIORITY_NORMAL = 100;
  94  
  95      /**
  96       * @var int Context indexing: low priority for reindexing.
  97       */
  98      const INDEX_PRIORITY_REINDEXING = 50;
  99  
 100      /**
 101       * @var string Core search area category for all results.
 102       */
 103      const SEARCH_AREA_CATEGORY_ALL = 'core-all';
 104  
 105      /**
 106       * @var string Core search area category for course content.
 107       */
 108      const SEARCH_AREA_CATEGORY_COURSE_CONTENT = 'core-course-content';
 109  
 110      /**
 111       * @var string Core search area category for courses.
 112       */
 113      const SEARCH_AREA_CATEGORY_COURSES = 'core-courses';
 114  
 115      /**
 116       * @var string Core search area category for users.
 117       */
 118      const SEARCH_AREA_CATEGORY_USERS = 'core-users';
 119  
 120      /**
 121       * @var string Core search area category for results that do not fit into any of existing categories.
 122       */
 123      const SEARCH_AREA_CATEGORY_OTHER = 'core-other';
 124  
 125      /**
 126       * @var \core_search\base[] Enabled search areas.
 127       */
 128      protected static $enabledsearchareas = null;
 129  
 130      /**
 131       * @var \core_search\base[] All system search areas.
 132       */
 133      protected static $allsearchareas = null;
 134  
 135      /**
 136       * @var \core_search\area_category[] A list of search area categories.
 137       */
 138      protected static $searchareacategories = null;
 139  
 140      /**
 141       * @var \core_search\manager
 142       */
 143      protected static $instance = null;
 144  
 145      /**
 146       * @var array IDs (as keys) of course deletions in progress in this requuest, if any.
 147       */
 148      protected static $coursedeleting = [];
 149  
 150      /**
 151       * @var \core_search\engine
 152       */
 153      protected $engine = null;
 154  
 155      /**
 156       * Note: This should be removed once possible (see MDL-60644).
 157       *
 158       * @var float Fake current time for use in PHPunit tests
 159       */
 160      protected static $phpunitfaketime = 0;
 161  
 162      /**
 163       * @var int Result count when used with mock results for Behat tests.
 164       */
 165      protected $behatresultcount = 0;
 166  
 167      /**
 168       * Constructor, use \core_search\manager::instance instead to get a class instance.
 169       *
 170       * @param \core_search\base The search engine to use
 171       */
 172      public function __construct($engine) {
 173          $this->engine = $engine;
 174      }
 175  
 176      /**
 177       * @var int Record time of each successful schema check, but not more than once per 10 minutes.
 178       */
 179      const SCHEMA_CHECK_TRACKING_DELAY = 10 * 60;
 180  
 181      /**
 182       * @var int Require a new schema check at least every 4 hours.
 183       */
 184      const SCHEMA_CHECK_REQUIRED_EVERY = 4 * 3600;
 185  
 186      /**
 187       * Returns an initialised \core_search instance.
 188       *
 189       * While constructing the instance, checks on the search schema may be carried out. The $fast
 190       * parameter provides a way to skip those checks on pages which are used frequently. It has
 191       * no effect if an instance has already been constructed in this request.
 192       *
 193       * The $query parameter indicates that the page is used for queries rather than indexing. If
 194       * configured, this will cause the query-only search engine to be used instead of the 'normal'
 195       * one.
 196       *
 197       * @see \core_search\engine::is_installed
 198       * @see \core_search\engine::is_server_ready
 199       * @param bool $fast Set to true when calling on a page that requires high performance
 200       * @param bool $query Set true on a page that is used for querying
 201       * @throws \core_search\engine_exception
 202       * @return \core_search\manager
 203       */
 204      public static function instance(bool $fast = false, bool $query = false) {
 205          global $CFG;
 206  
 207          // One per request, this should be purged during testing.
 208          if (static::$instance !== null) {
 209              return static::$instance;
 210          }
 211  
 212          if (empty($CFG->searchengine)) {
 213              throw new \core_search\engine_exception('enginenotselected', 'search');
 214          }
 215  
 216          if (!$engine = static::search_engine_instance($query)) {
 217              throw new \core_search\engine_exception('enginenotfound', 'search', '', $CFG->searchengine);
 218          }
 219  
 220          // Get time now and at last schema check.
 221          $now = (int)self::get_current_time();
 222          $lastschemacheck = get_config($engine->get_plugin_name(), 'lastschemacheck');
 223  
 224          // On pages where performance matters, tell the engine to skip schema checks.
 225          $skipcheck = false;
 226          if ($fast && $now < $lastschemacheck + self::SCHEMA_CHECK_REQUIRED_EVERY) {
 227              $skipcheck = true;
 228              $engine->skip_schema_check();
 229          }
 230  
 231          if (!$engine->is_installed()) {
 232              throw new \core_search\engine_exception('enginenotinstalled', 'search', '', $CFG->searchengine);
 233          }
 234  
 235          $serverstatus = $engine->is_server_ready();
 236          if ($serverstatus !== true) {
 237              // Skip this error in Behat when faking seach results.
 238              if (!defined('BEHAT_SITE_RUNNING') || !get_config('core_search', 'behat_fakeresult')) {
 239                  // Clear the record of successful schema checks since it might have failed.
 240                  unset_config('lastschemacheck', $engine->get_plugin_name());
 241                  // Error message with no details as this is an exception that any user may find if the server crashes.
 242                  throw new \core_search\engine_exception('engineserverstatus', 'search');
 243              }
 244          }
 245  
 246          // If we did a successful schema check, record this, but not more than once per 10 minutes
 247          // (to avoid updating the config db table/cache too often in case it gets called frequently).
 248          if (!$skipcheck && $now >= $lastschemacheck + self::SCHEMA_CHECK_TRACKING_DELAY) {
 249              set_config('lastschemacheck', $now, $engine->get_plugin_name());
 250          }
 251  
 252          static::$instance = new \core_search\manager($engine);
 253          return static::$instance;
 254      }
 255  
 256      /**
 257       * Returns whether global search is enabled or not.
 258       *
 259       * @return bool
 260       */
 261      public static function is_global_search_enabled() {
 262          global $CFG;
 263          return !empty($CFG->enableglobalsearch);
 264      }
 265  
 266      /**
 267       * Returns the search URL for course search
 268       *
 269       * @return moodle_url
 270       */
 271      public static function get_course_search_url() {
 272          if (self::is_global_search_enabled()) {
 273              $searchurl = '/search/index.php';
 274          } else {
 275              $searchurl = '/course/search.php';
 276          }
 277  
 278          return new \moodle_url($searchurl);
 279      }
 280  
 281      /**
 282       * Returns whether indexing is enabled or not (you can enable indexing even when search is not
 283       * enabled at the moment, so as to have it ready for students).
 284       *
 285       * @return bool True if indexing is enabled.
 286       */
 287      public static function is_indexing_enabled() {
 288          global $CFG;
 289          return !empty($CFG->enableglobalsearch) || !empty($CFG->searchindexwhendisabled);
 290      }
 291  
 292      /**
 293       * Returns an instance of the search engine.
 294       *
 295       * @param bool $query If true, gets the query-only search engine (where configured)
 296       * @return \core_search\engine
 297       */
 298      public static function search_engine_instance(bool $query = false) {
 299          global $CFG;
 300  
 301          if ($query && $CFG->searchenginequeryonly) {
 302              return self::search_engine_instance_from_setting($CFG->searchenginequeryonly);
 303          } else {
 304              return self::search_engine_instance_from_setting($CFG->searchengine);
 305          }
 306      }
 307  
 308      /**
 309       * Loads a search engine based on the name given in settings, which can optionally
 310       * include '-alternate' to indicate that an alternate version should be used.
 311       *
 312       * @param string $setting
 313       * @return engine|null
 314       */
 315      protected static function search_engine_instance_from_setting(string $setting): ?engine {
 316          if (preg_match('~^(.*)-alternate$~', $setting, $matches)) {
 317              $enginename = $matches[1];
 318              $alternate = true;
 319          } else {
 320              $enginename = $setting;
 321              $alternate = false;
 322          }
 323  
 324          $classname = '\\search_' . $enginename . '\\engine';
 325          if (!class_exists($classname)) {
 326              return null;
 327          }
 328  
 329          if ($alternate) {
 330              return new $classname(true);
 331          } else {
 332              // Use the constructor with no parameters for compatibility.
 333              return new $classname();
 334          }
 335      }
 336  
 337      /**
 338       * Returns the search engine.
 339       *
 340       * @return \core_search\engine
 341       */
 342      public function get_engine() {
 343          return $this->engine;
 344      }
 345  
 346      /**
 347       * Returns a search area class name.
 348       *
 349       * @param string $areaid
 350       * @return string
 351       */
 352      protected static function get_area_classname($areaid) {
 353          list($componentname, $areaname) = static::extract_areaid_parts($areaid);
 354          return '\\' . $componentname . '\\search\\' . $areaname;
 355      }
 356  
 357      /**
 358       * Returns a new area search indexer instance.
 359       *
 360       * @param string $areaid
 361       * @return \core_search\base|bool False if the area is not available.
 362       */
 363      public static function get_search_area($areaid) {
 364  
 365          // We have them all here.
 366          if (!empty(static::$allsearchareas[$areaid])) {
 367              return static::$allsearchareas[$areaid];
 368          }
 369  
 370          $classname = static::get_area_classname($areaid);
 371  
 372          if (class_exists($classname) && static::is_search_area($classname)) {
 373              return new $classname();
 374          }
 375  
 376          return false;
 377      }
 378  
 379      /**
 380       * Return the list of available search areas.
 381       *
 382       * @param bool $enabled Return only the enabled ones.
 383       * @return \core_search\base[]
 384       */
 385      public static function get_search_areas_list($enabled = false) {
 386  
 387          // Two different arrays, we don't expect these arrays to be big.
 388          if (static::$allsearchareas !== null) {
 389              if (!$enabled) {
 390                  return static::$allsearchareas;
 391              } else {
 392                  return static::$enabledsearchareas;
 393              }
 394          }
 395  
 396          static::$allsearchareas = array();
 397          static::$enabledsearchareas = array();
 398          $searchclasses = \core_component::get_component_classes_in_namespace(null, 'search');
 399  
 400          foreach ($searchclasses as $classname => $classpath) {
 401              $areaname = substr(strrchr($classname, '\\'), 1);
 402              $componentname = strstr($classname, '\\', 1);
 403              if (!static::is_search_area($classname)) {
 404                  continue;
 405              }
 406  
 407              $areaid = static::generate_areaid($componentname, $areaname);
 408              $searchclass = new $classname();
 409              static::$allsearchareas[$areaid] = $searchclass;
 410              if ($searchclass->is_enabled()) {
 411                  static::$enabledsearchareas[$areaid] = $searchclass;
 412              }
 413          }
 414  
 415          if ($enabled) {
 416              return static::$enabledsearchareas;
 417          }
 418          return static::$allsearchareas;
 419      }
 420  
 421      /**
 422       * Return search area category instance by category name.
 423       *
 424       * @param string $name Category name. If name is not valid will return default category.
 425       *
 426       * @return \core_search\area_category
 427       */
 428      public static function get_search_area_category_by_name($name) {
 429          if (key_exists($name, self::get_search_area_categories())) {
 430              return self::get_search_area_categories()[$name];
 431          } else {
 432              return self::get_search_area_categories()[self::get_default_area_category_name()];
 433          }
 434      }
 435  
 436      /**
 437       * Return a list of existing search area categories.
 438       *
 439       * @return \core_search\area_category[]
 440       */
 441      public static function get_search_area_categories() {
 442          if (!isset(static::$searchareacategories)) {
 443              $categories = self::get_core_search_area_categories();
 444  
 445              // Go through all existing search areas and get categories they are assigned to.
 446              $areacategories = [];
 447              foreach (self::get_search_areas_list() as $searcharea) {
 448                  foreach ($searcharea->get_category_names() as $categoryname) {
 449                      if (!key_exists($categoryname, $areacategories)) {
 450                          $areacategories[$categoryname] = [];
 451                      }
 452  
 453                      $areacategories[$categoryname][] = $searcharea;
 454                  }
 455              }
 456  
 457              // Populate core categories by areas.
 458              foreach ($areacategories as $name => $searchareas) {
 459                  if (key_exists($name, $categories)) {
 460                      $categories[$name]->set_areas($searchareas);
 461                  } else {
 462                      throw new \coding_exception('Unknown core search area category ' . $name);
 463                  }
 464              }
 465  
 466              // Get additional categories.
 467              $additionalcategories = self::get_additional_search_area_categories();
 468              foreach ($additionalcategories as $additionalcategory) {
 469                  if (!key_exists($additionalcategory->get_name(), $categories)) {
 470                      $categories[$additionalcategory->get_name()] = $additionalcategory;
 471                  }
 472              }
 473  
 474              // Remove categories without areas.
 475              foreach ($categories as $key => $category) {
 476                  if (empty($category->get_areas())) {
 477                      unset($categories[$key]);
 478                  }
 479              }
 480  
 481              // Sort categories by order.
 482              uasort($categories, function($category1, $category2) {
 483                  return $category1->get_order() > $category2->get_order();
 484              });
 485  
 486              static::$searchareacategories = $categories;
 487          }
 488  
 489          return static::$searchareacategories;
 490      }
 491  
 492      /**
 493       * Get list of core search area categories.
 494       *
 495       * @return \core_search\area_category[]
 496       */
 497      protected static function get_core_search_area_categories() {
 498          $categories = [];
 499  
 500          $categories[self::SEARCH_AREA_CATEGORY_ALL] = new area_category(
 501              self::SEARCH_AREA_CATEGORY_ALL,
 502              get_string('core-all', 'search'),
 503              0,
 504              self::get_search_areas_list(true)
 505          );
 506  
 507          $categories[self::SEARCH_AREA_CATEGORY_COURSE_CONTENT] = new area_category(
 508              self::SEARCH_AREA_CATEGORY_COURSE_CONTENT,
 509              get_string('core-course-content', 'search'),
 510              1
 511          );
 512  
 513          $categories[self::SEARCH_AREA_CATEGORY_COURSES] = new area_category(
 514              self::SEARCH_AREA_CATEGORY_COURSES,
 515              get_string('core-courses', 'search'),
 516              2
 517          );
 518  
 519          $categories[self::SEARCH_AREA_CATEGORY_USERS] = new area_category(
 520              self::SEARCH_AREA_CATEGORY_USERS,
 521              get_string('core-users', 'search'),
 522              3
 523          );
 524  
 525          $categories[self::SEARCH_AREA_CATEGORY_OTHER] = new area_category(
 526              self::SEARCH_AREA_CATEGORY_OTHER,
 527              get_string('core-other', 'search'),
 528              4
 529          );
 530  
 531          return $categories;
 532      }
 533  
 534      /**
 535       * Gets a list of additional search area categories.
 536       *
 537       * @return \core_search\area_category[]
 538       */
 539      protected static function get_additional_search_area_categories() {
 540          $additionalcategories = [];
 541  
 542          // Allow plugins to add custom search area categories.
 543          if ($pluginsfunction = get_plugins_with_function('search_area_categories')) {
 544              foreach ($pluginsfunction as $plugintype => $plugins) {
 545                  foreach ($plugins as $pluginfunction) {
 546                      $plugincategories = $pluginfunction();
 547                      // We're expecting a list of valid area categories.
 548                      if (is_array($plugincategories)) {
 549                          foreach ($plugincategories as $plugincategory) {
 550                              if (self::is_valid_area_category($plugincategory)) {
 551                                  $additionalcategories[] = $plugincategory;
 552                              } else {
 553                                  throw  new \coding_exception('Invalid search area category!');
 554                              }
 555                          }
 556                      } else {
 557                          throw  new \coding_exception($pluginfunction . ' should return a list of search area categories!');
 558                      }
 559                  }
 560              }
 561          }
 562  
 563          return $additionalcategories;
 564      }
 565  
 566      /**
 567       * Check if provided instance of area category is valid.
 568       *
 569       * @param mixed $areacategory Area category instance. Potentially could be anything.
 570       *
 571       * @return bool
 572       */
 573      protected static function is_valid_area_category($areacategory) {
 574          return $areacategory instanceof area_category;
 575      }
 576  
 577      /**
 578       * Clears all static caches.
 579       *
 580       * @return void
 581       */
 582      public static function clear_static() {
 583  
 584          static::$enabledsearchareas = null;
 585          static::$allsearchareas = null;
 586          static::$instance = null;
 587          static::$searchareacategories = null;
 588  
 589          base_block::clear_static();
 590          engine::clear_users_cache();
 591      }
 592  
 593      /**
 594       * Generates an area id from the componentname and the area name.
 595       *
 596       * There should not be any naming conflict as the area name is the
 597       * class name in component/classes/search/.
 598       *
 599       * @param string $componentname
 600       * @param string $areaname
 601       * @return void
 602       */
 603      public static function generate_areaid($componentname, $areaname) {
 604          return $componentname . '-' . $areaname;
 605      }
 606  
 607      /**
 608       * Returns all areaid string components (component name and area name).
 609       *
 610       * @param string $areaid
 611       * @return array Component name (Frankenstyle) and area name (search area class name)
 612       */
 613      public static function extract_areaid_parts($areaid) {
 614          return explode('-', $areaid);
 615      }
 616  
 617      /**
 618       * Parse a search area id and get plugin name and config name prefix from it.
 619       *
 620       * @param string $areaid Search area id.
 621       * @return array Where the first element is a plugin name and the second is config names prefix.
 622       */
 623      public static function parse_areaid($areaid) {
 624          $parts = self::extract_areaid_parts($areaid);
 625  
 626          if (empty($parts[1])) {
 627              throw new \coding_exception('Trying to parse invalid search area id ' . $areaid);
 628          }
 629  
 630          $component = $parts[0];
 631          $area = $parts[1];
 632  
 633          if (strpos($component, 'core') === 0) {
 634              $plugin = 'core_search';
 635              $configprefix = str_replace('-', '_', $areaid);
 636          } else {
 637              $plugin = $component;
 638              $configprefix = 'search_' . $area;
 639          }
 640  
 641          return [$plugin, $configprefix];
 642      }
 643  
 644      /**
 645       * Returns information about the areas which the user can access.
 646       *
 647       * The returned value is a stdClass object with the following fields:
 648       * - everything (bool, true for admin only)
 649       * - usercontexts (indexed by area identifier then context
 650       * - separategroupscontexts (contexts within which group restrictions apply)
 651       * - visiblegroupscontextsareas (overrides to the above when the same contexts also have
 652       *   'visible groups' for certain search area ids - hopefully rare)
 653       * - usergroups (groups which the current user belongs to)
 654       *
 655       * The areas can be limited by course id and context id. If specifying context ids, results
 656       * are limited to the exact context ids specified and not their children (for example, giving
 657       * the course context id would result in including search items with the course context id, and
 658       * not anything from a context inside the course). For performance, you should also specify
 659       * course id(s) when using context ids.
 660       *
 661       * @param array|false $limitcourseids An array of course ids to limit the search to. False for no limiting.
 662       * @param array|false $limitcontextids An array of context ids to limit the search to. False for no limiting.
 663       * @return \stdClass Object as described above
 664       */
 665      protected function get_areas_user_accesses($limitcourseids = false, $limitcontextids = false) {
 666          global $DB, $USER;
 667  
 668          // All results for admins (unless they have chosen to limit results). Eventually we could
 669          // add a new capability for managers.
 670          if (is_siteadmin() && !$limitcourseids && !$limitcontextids) {
 671              return (object)array('everything' => true);
 672          }
 673  
 674          $areasbylevel = array();
 675  
 676          // Split areas by context level so we only iterate only once through courses and cms.
 677          $searchareas = static::get_search_areas_list(true);
 678          foreach ($searchareas as $areaid => $unused) {
 679              $classname = static::get_area_classname($areaid);
 680              $searcharea = new $classname();
 681              foreach ($classname::get_levels() as $level) {
 682                  $areasbylevel[$level][$areaid] = $searcharea;
 683              }
 684          }
 685  
 686          // This will store area - allowed contexts relations.
 687          $areascontexts = array();
 688  
 689          // Initialise two special-case arrays for storing other information related to the contexts.
 690          $separategroupscontexts = array();
 691          $visiblegroupscontextsareas = array();
 692          $usergroups = array();
 693  
 694          if (empty($limitcourseids) && !empty($areasbylevel[CONTEXT_SYSTEM])) {
 695              // We add system context to all search areas working at this level. Here each area is fully responsible of
 696              // the access control as we can not automate much, we can not even check guest access as some areas might
 697              // want to allow guests to retrieve data from them.
 698  
 699              $systemcontextid = \context_system::instance()->id;
 700              if (!$limitcontextids || in_array($systemcontextid, $limitcontextids)) {
 701                  foreach ($areasbylevel[CONTEXT_SYSTEM] as $areaid => $searchclass) {
 702                      $areascontexts[$areaid][$systemcontextid] = $systemcontextid;
 703                  }
 704              }
 705          }
 706  
 707          if (!empty($areasbylevel[CONTEXT_USER])) {
 708              if ($usercontext = \context_user::instance($USER->id, IGNORE_MISSING)) {
 709                  if (!$limitcontextids || in_array($usercontext->id, $limitcontextids)) {
 710                      // Extra checking although only logged users should reach this point, guest users have a valid context id.
 711                      foreach ($areasbylevel[CONTEXT_USER] as $areaid => $searchclass) {
 712                          $areascontexts[$areaid][$usercontext->id] = $usercontext->id;
 713                      }
 714                  }
 715              }
 716          }
 717  
 718          if (is_siteadmin()) {
 719              $allcourses = $this->get_all_courses($limitcourseids);
 720          } else {
 721              $allcourses = $mycourses = $this->get_my_courses((bool)get_config('core', 'searchallavailablecourses'));
 722  
 723              if (self::include_all_courses()) {
 724                  $allcourses = $this->get_all_courses($limitcourseids);
 725              }
 726          }
 727  
 728          if (empty($limitcourseids) || in_array(SITEID, $limitcourseids)) {
 729              $allcourses[SITEID] = get_course(SITEID);
 730              if (isset($mycourses)) {
 731                  $mycourses[SITEID] = get_course(SITEID);
 732              }
 733          }
 734  
 735          // Keep a list of included course context ids (needed for the block calculation below).
 736          $coursecontextids = [];
 737          $modulecms = [];
 738  
 739          foreach ($allcourses as $course) {
 740              if (!empty($limitcourseids) && !in_array($course->id, $limitcourseids)) {
 741                  // Skip non-included courses.
 742                  continue;
 743              }
 744  
 745              $coursecontext = \context_course::instance($course->id);
 746              $hasgrouprestrictions = false;
 747  
 748              if (!empty($areasbylevel[CONTEXT_COURSE]) &&
 749                      (!$limitcontextids || in_array($coursecontext->id, $limitcontextids))) {
 750                  // Add the course contexts the user can view.
 751                  foreach ($areasbylevel[CONTEXT_COURSE] as $areaid => $searchclass) {
 752                      if (!empty($mycourses[$course->id]) || \core_course_category::can_view_course_info($course)) {
 753                          $areascontexts[$areaid][$coursecontext->id] = $coursecontext->id;
 754                      }
 755                  }
 756              }
 757  
 758              // Skip module context if a user can't access related course.
 759              if (isset($mycourses) && !key_exists($course->id, $mycourses)) {
 760                  continue;
 761              }
 762  
 763              $coursecontextids[] = $coursecontext->id;
 764  
 765              // Info about the course modules.
 766              $modinfo = get_fast_modinfo($course);
 767  
 768              if (!empty($areasbylevel[CONTEXT_MODULE])) {
 769                  // Add the module contexts the user can view (cm_info->uservisible).
 770  
 771                  foreach ($areasbylevel[CONTEXT_MODULE] as $areaid => $searchclass) {
 772  
 773                      // Removing the plugintype 'mod_' prefix.
 774                      $modulename = substr($searchclass->get_component_name(), 4);
 775  
 776                      $modinstances = $modinfo->get_instances_of($modulename);
 777                      foreach ($modinstances as $modinstance) {
 778                          // Skip module context if not included in list of context ids.
 779                          if ($limitcontextids && !in_array($modinstance->context->id, $limitcontextids)) {
 780                              continue;
 781                          }
 782                          if ($modinstance->uservisible) {
 783                              $contextid = $modinstance->context->id;
 784                              $areascontexts[$areaid][$contextid] = $contextid;
 785                              $modulecms[$modinstance->id] = $modinstance;
 786  
 787                              if (!has_capability('moodle/site:accessallgroups', $modinstance->context) &&
 788                                      ($searchclass instanceof base_mod) &&
 789                                      $searchclass->supports_group_restriction()) {
 790                                  if ($searchclass->restrict_cm_access_by_group($modinstance)) {
 791                                      $separategroupscontexts[$contextid] = $contextid;
 792                                      $hasgrouprestrictions = true;
 793                                  } else {
 794                                      // Track a list of anything that has a group id (so might get
 795                                      // filtered) and doesn't want to be, in this context.
 796                                      if (!array_key_exists($contextid, $visiblegroupscontextsareas)) {
 797                                          $visiblegroupscontextsareas[$contextid] = array();
 798                                      }
 799                                      $visiblegroupscontextsareas[$contextid][$areaid] = $areaid;
 800                                  }
 801                              }
 802                          }
 803                      }
 804                  }
 805              }
 806  
 807              // Insert group information for course (unless there aren't any modules restricted by
 808              // group for this user in this course, in which case don't bother).
 809              if ($hasgrouprestrictions) {
 810                  $groups = groups_get_all_groups($course->id, $USER->id, 0, 'g.id');
 811                  foreach ($groups as $group) {
 812                      $usergroups[$group->id] = $group->id;
 813                  }
 814              }
 815          }
 816  
 817          // Chuck away all the 'visible groups contexts' data unless there is actually something
 818          // that does use separate groups in the same context (this data is only used as an
 819          // 'override' in cases where the search is restricting to separate groups).
 820          foreach ($visiblegroupscontextsareas as $contextid => $areas) {
 821              if (!array_key_exists($contextid, $separategroupscontexts)) {
 822                  unset($visiblegroupscontextsareas[$contextid]);
 823              }
 824          }
 825  
 826          // Add all supported block contexts for course contexts that user can access, in a single query for performance.
 827          if (!empty($areasbylevel[CONTEXT_BLOCK]) && !empty($coursecontextids)) {
 828              // Get list of all block types we care about.
 829              $blocklist = [];
 830              foreach ($areasbylevel[CONTEXT_BLOCK] as $areaid => $searchclass) {
 831                  $blocklist[$searchclass->get_block_name()] = true;
 832              }
 833              list ($blocknamesql, $blocknameparams) = $DB->get_in_or_equal(array_keys($blocklist));
 834  
 835              // Get list of course contexts.
 836              list ($contextsql, $contextparams) = $DB->get_in_or_equal($coursecontextids);
 837  
 838              // Get list of block context (if limited).
 839              $blockcontextwhere = '';
 840              $blockcontextparams = [];
 841              if ($limitcontextids) {
 842                  list ($blockcontextsql, $blockcontextparams) = $DB->get_in_or_equal($limitcontextids);
 843                  $blockcontextwhere = 'AND x.id ' . $blockcontextsql;
 844              }
 845  
 846              // Query all blocks that are within an included course, and are set to be visible, and
 847              // in a supported page type (basically just course view). This query could be
 848              // extended (or a second query added) to support blocks that are within a module
 849              // context as well, and we could add more page types if required.
 850              $blockrecs = $DB->get_records_sql("
 851                          SELECT x.*, bi.blockname AS blockname, bi.id AS blockinstanceid
 852                            FROM {block_instances} bi
 853                            JOIN {context} x ON x.instanceid = bi.id AND x.contextlevel = ?
 854                       LEFT JOIN {block_positions} bp ON bp.blockinstanceid = bi.id
 855                                 AND bp.contextid = bi.parentcontextid
 856                                 AND bp.pagetype LIKE 'course-view-%'
 857                                 AND bp.subpage = ''
 858                                 AND bp.visible = 0
 859                           WHERE bi.parentcontextid $contextsql
 860                                 $blockcontextwhere
 861                                 AND bi.blockname $blocknamesql
 862                                 AND bi.subpagepattern IS NULL
 863                                 AND (bi.pagetypepattern = 'site-index'
 864                                     OR bi.pagetypepattern LIKE 'course-view-%'
 865                                     OR bi.pagetypepattern = 'course-*'
 866                                     OR bi.pagetypepattern = '*')
 867                                 AND bp.id IS NULL",
 868                      array_merge([CONTEXT_BLOCK], $contextparams, $blockcontextparams, $blocknameparams));
 869              $blockcontextsbyname = [];
 870              foreach ($blockrecs as $blockrec) {
 871                  if (empty($blockcontextsbyname[$blockrec->blockname])) {
 872                      $blockcontextsbyname[$blockrec->blockname] = [];
 873                  }
 874                  \context_helper::preload_from_record($blockrec);
 875                  $blockcontextsbyname[$blockrec->blockname][] = \context_block::instance(
 876                          $blockrec->blockinstanceid);
 877              }
 878  
 879              // Add the block contexts the user can view.
 880              foreach ($areasbylevel[CONTEXT_BLOCK] as $areaid => $searchclass) {
 881                  if (empty($blockcontextsbyname[$searchclass->get_block_name()])) {
 882                      continue;
 883                  }
 884                  foreach ($blockcontextsbyname[$searchclass->get_block_name()] as $context) {
 885                      if (has_capability('moodle/block:view', $context)) {
 886                          $areascontexts[$areaid][$context->id] = $context->id;
 887                      }
 888                  }
 889              }
 890          }
 891  
 892          // Return all the data.
 893          return (object)array('everything' => false, 'usercontexts' => $areascontexts,
 894                  'separategroupscontexts' => $separategroupscontexts, 'usergroups' => $usergroups,
 895                  'visiblegroupscontextsareas' => $visiblegroupscontextsareas);
 896      }
 897  
 898      /**
 899       * Returns requested page of documents plus additional information for paging.
 900       *
 901       * This function does not perform any kind of security checking for access, the caller code
 902       * should check that the current user have moodle/search:query capability.
 903       *
 904       * If a page is requested that is beyond the last result, the last valid page is returned in
 905       * results, and actualpage indicates which page was returned.
 906       *
 907       * @param stdClass $formdata
 908       * @param int $pagenum The 0 based page number.
 909       * @return object An object with 3 properties:
 910       *                    results    => An array of \core_search\documents for the actual page.
 911       *                    totalcount => Number of records that are possibly available, to base paging on.
 912       *                    actualpage => The actual page returned.
 913       */
 914      public function paged_search(\stdClass $formdata, $pagenum) {
 915          $out = new \stdClass();
 916  
 917          if (self::is_search_area_categories_enabled() && !empty($formdata->cat)) {
 918              $cat = self::get_search_area_category_by_name($formdata->cat);
 919              if (empty($formdata->areaids)) {
 920                  $formdata->areaids = array_keys($cat->get_areas());
 921              } else {
 922                  foreach ($formdata->areaids as $key => $areaid) {
 923                      if (!key_exists($areaid, $cat->get_areas())) {
 924                          unset($formdata->areaids[$key]);
 925                      }
 926                  }
 927              }
 928          }
 929  
 930          $perpage = static::DISPLAY_RESULTS_PER_PAGE;
 931  
 932          // Make sure we only allow request up to max page.
 933          $pagenum = min($pagenum, (static::MAX_RESULTS / $perpage) - 1);
 934  
 935          // Calculate the first and last document number for the current page, 1 based.
 936          $mindoc = ($pagenum * $perpage) + 1;
 937          $maxdoc = ($pagenum + 1) * $perpage;
 938  
 939          // Get engine documents, up to max.
 940          $docs = $this->search($formdata, $maxdoc);
 941  
 942          $resultcount = count($docs);
 943          if ($resultcount < $maxdoc) {
 944              // This means it couldn't give us results to max, so the count must be the max.
 945              $out->totalcount = $resultcount;
 946          } else {
 947              // Get the possible count reported by engine, and limit to our max.
 948              $out->totalcount = $this->engine->get_query_total_count();
 949              if (defined('BEHAT_SITE_RUNNING') && $this->behatresultcount) {
 950                  // Override results when using Behat mock results.
 951                  $out->totalcount = $this->behatresultcount;
 952              }
 953              $out->totalcount = min($out->totalcount, static::MAX_RESULTS);
 954          }
 955  
 956          // Determine the actual page.
 957          if ($resultcount < $mindoc) {
 958              // We couldn't get the min docs for this page, so determine what page we can get.
 959              $out->actualpage = floor(($resultcount - 1) / $perpage);
 960          } else {
 961              $out->actualpage = $pagenum;
 962          }
 963  
 964          // Split the results to only return the page.
 965          $out->results = array_slice($docs, $out->actualpage * $perpage, $perpage, true);
 966  
 967          return $out;
 968      }
 969  
 970      /**
 971       * Returns documents from the engine based on the data provided.
 972       *
 973       * This function does not perform any kind of security checking, the caller code
 974       * should check that the current user have moodle/search:query capability.
 975       *
 976       * It might return the results from the cache instead.
 977       *
 978       * Valid formdata options include:
 979       * - q (query text)
 980       * - courseids (optional list of course ids to restrict)
 981       * - contextids (optional list of context ids to restrict)
 982       * - context (Moodle context object for location user searched from)
 983       * - order (optional ordering, one of the types supported by the search engine e.g. 'relevance')
 984       * - userids (optional list of user ids to restrict)
 985       *
 986       * @param \stdClass $formdata Query input data (usually from search form)
 987       * @param int $limit The maximum number of documents to return
 988       * @return \core_search\document[]
 989       */
 990      public function search(\stdClass $formdata, $limit = 0) {
 991          // For Behat testing, the search results can be faked using a special step.
 992          if (defined('BEHAT_SITE_RUNNING')) {
 993              $fakeresult = get_config('core_search', 'behat_fakeresult');
 994              if ($fakeresult) {
 995                  // Clear config setting.
 996                  unset_config('core_search', 'behat_fakeresult');
 997  
 998                  // Check query matches expected value.
 999                  $details = json_decode($fakeresult);
1000                  if ($formdata->q !== $details->query) {
1001                      throw new \coding_exception('Unexpected search query: ' . $formdata->q);
1002                  }
1003  
1004                  // Create search documents from the JSON data.
1005                  $docs = [];
1006                  foreach ($details->results as $result) {
1007                      $doc = new \core_search\document($result->itemid, $result->componentname,
1008                              $result->areaname);
1009                      foreach ((array)$result->fields as $field => $value) {
1010                          $doc->set($field, $value);
1011                      }
1012                      foreach ((array)$result->extrafields as $field => $value) {
1013                          $doc->set_extra($field, $value);
1014                      }
1015                      $area = $this->get_search_area($doc->get('areaid'));
1016                      $doc->set_doc_url($area->get_doc_url($doc));
1017                      $doc->set_context_url($area->get_context_url($doc));
1018                      $docs[] = $doc;
1019                  }
1020  
1021                  // Store the mock count, and apply the limit to the returned results.
1022                  $this->behatresultcount = count($docs);
1023                  if ($this->behatresultcount > $limit) {
1024                      $docs = array_slice($docs, 0, $limit);
1025                  }
1026  
1027                  return $docs;
1028              }
1029          }
1030  
1031          $limitcourseids = $this->build_limitcourseids($formdata);
1032  
1033          $limitcontextids = false;
1034          if (!empty($formdata->contextids)) {
1035              $limitcontextids = $formdata->contextids;
1036          }
1037  
1038          // Clears previous query errors.
1039          $this->engine->clear_query_error();
1040  
1041          $contextinfo = $this->get_areas_user_accesses($limitcourseids, $limitcontextids);
1042          if (!$contextinfo->everything && !$contextinfo->usercontexts) {
1043              // User can not access any context.
1044              $docs = array();
1045          } else {
1046              // If engine does not support groups, remove group information from the context info -
1047              // use the old format instead (true = admin, array = user contexts).
1048              if (!$this->engine->supports_group_filtering()) {
1049                  $contextinfo = $contextinfo->everything ? true : $contextinfo->usercontexts;
1050              }
1051  
1052              // Execute the actual query.
1053              $docs = $this->engine->execute_query($formdata, $contextinfo, $limit);
1054          }
1055  
1056          return $docs;
1057      }
1058  
1059      /**
1060       * Build a list of course ids to limit the search based on submitted form data.
1061       *
1062       * @param \stdClass $formdata Submitted search form data.
1063       *
1064       * @return array|bool
1065       */
1066      protected function build_limitcourseids(\stdClass $formdata) {
1067          $limitcourseids = false;
1068  
1069          if (!empty($formdata->mycoursesonly)) {
1070              $limitcourseids = array_keys($this->get_my_courses(false));
1071          }
1072  
1073          if (!empty($formdata->courseids)) {
1074              if (empty($limitcourseids)) {
1075                  $limitcourseids = $formdata->courseids;
1076              } else {
1077                  $limitcourseids = array_intersect($limitcourseids, $formdata->courseids);
1078              }
1079          }
1080  
1081          return $limitcourseids;
1082      }
1083  
1084      /**
1085       * Merge separate index segments into one.
1086       */
1087      public function optimize_index() {
1088          $this->engine->optimize();
1089      }
1090  
1091      /**
1092       * Index all documents.
1093       *
1094       * @param bool $fullindex Whether we should reindex everything or not.
1095       * @param float $timelimit Time limit in seconds (0 = no time limit)
1096       * @param \progress_trace|null $progress Optional class for tracking progress
1097       * @throws \moodle_exception
1098       * @return bool Whether there was any updated document or not.
1099       */
1100      public function index($fullindex = false, $timelimit = 0, \progress_trace $progress = null) {
1101          global $DB;
1102  
1103          // Cannot combine time limit with reindex.
1104          if ($timelimit && $fullindex) {
1105              throw new \coding_exception('Cannot apply time limit when reindexing');
1106          }
1107          if (!$progress) {
1108              $progress = new \null_progress_trace();
1109          }
1110  
1111          // Unlimited time.
1112          \core_php_time_limit::raise();
1113  
1114          // Notify the engine that an index starting.
1115          $this->engine->index_starting($fullindex);
1116  
1117          $sumdocs = 0;
1118  
1119          $searchareas = $this->get_search_areas_list(true);
1120  
1121          if ($timelimit) {
1122              // If time is limited (and therefore we're not just indexing everything anyway), select
1123              // an order for search areas. The intention here is to avoid a situation where a new
1124              // large search area is enabled, and this means all our other search areas go out of
1125              // date while that one is being indexed. To do this, we order by the time we spent
1126              // indexing them last time we ran, meaning anything that took a very long time will be
1127              // done last.
1128              uasort($searchareas, function(\core_search\base $area1, \core_search\base $area2) {
1129                  return (int)$area1->get_last_indexing_duration() - (int)$area2->get_last_indexing_duration();
1130              });
1131  
1132              // Decide time to stop.
1133              $stopat = self::get_current_time() + $timelimit;
1134          }
1135  
1136          foreach ($searchareas as $areaid => $searcharea) {
1137  
1138              $progress->output('Processing area: ' . $searcharea->get_visible_name());
1139  
1140              // Notify the engine that an area is starting.
1141              $this->engine->area_index_starting($searcharea, $fullindex);
1142  
1143              $indexingstart = (int)self::get_current_time();
1144              $elapsed = self::get_current_time();
1145  
1146              // This is used to store this component config.
1147              list($componentconfigname, $varname) = $searcharea->get_config_var_name();
1148  
1149              $prevtimestart = intval(get_config($componentconfigname, $varname . '_indexingstart'));
1150  
1151              if ($fullindex === true) {
1152                  $referencestarttime = 0;
1153  
1154                  // For full index, we delete any queued context index requests, as those will
1155                  // obviously be met by the full index.
1156                  $DB->delete_records('search_index_requests');
1157              } else {
1158                  $partial = get_config($componentconfigname, $varname . '_partial');
1159                  if ($partial) {
1160                      // When the previous index did not complete all data, we start from the time of the
1161                      // last document that was successfully indexed. (Note this will result in
1162                      // re-indexing that one document, but we can't avoid that because there may be
1163                      // other documents in the same second.)
1164                      $referencestarttime = intval(get_config($componentconfigname, $varname . '_lastindexrun'));
1165                  } else {
1166                      $referencestarttime = $prevtimestart;
1167                  }
1168              }
1169  
1170              // Getting the recordset from the area.
1171              $recordset = $searcharea->get_recordset_by_timestamp($referencestarttime);
1172              $initialquerytime = self::get_current_time() - $elapsed;
1173              if ($initialquerytime > self::DISPLAY_LONG_QUERY_TIME) {
1174                  $progress->output('Initial query took ' . round($initialquerytime, 1) .
1175                          ' seconds.', 1);
1176              }
1177  
1178              // Pass get_document as callback.
1179              $fileindexing = $this->engine->file_indexing_enabled() && $searcharea->uses_file_indexing();
1180              $options = array('indexfiles' => $fileindexing, 'lastindexedtime' => $prevtimestart);
1181              if ($timelimit) {
1182                  $options['stopat'] = $stopat;
1183              }
1184              $options['progress'] = $progress;
1185              $iterator = new skip_future_documents_iterator(new \core\dml\recordset_walk(
1186                      $recordset, array($searcharea, 'get_document'), $options));
1187              $result = $this->engine->add_documents($iterator, $searcharea, $options);
1188              $recordset->close();
1189              $batchinfo = '';
1190              if (count($result) === 6) {
1191                  [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial, $batches] = $result;
1192                  // Only show the batch count if we actually batched any requests.
1193                  if ($batches !== $numdocs + $numdocsignored) {
1194                      $batchinfo = ' (' . $batches . ' batch' . ($batches === 1 ? '' : 'es') . ')';
1195                  }
1196              } else if (count($result) === 5) {
1197                  // Backward compatibility for engines that don't return a batch count.
1198                  [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial] = $result;
1199                  // Deprecated since Moodle 3.10 MDL-68690.
1200                  // TODO: MDL-68776 This will be deleted in Moodle 4.2.
1201                  debugging('engine::add_documents() should return $batches (5-value return is deprecated)',
1202                          DEBUG_DEVELOPER);
1203              } else {
1204                  throw new coding_exception('engine::add_documents() should return $partial (4-value return is deprecated)');
1205              }
1206  
1207              if ($numdocs > 0) {
1208                  $elapsed = round((self::get_current_time() - $elapsed), 1);
1209  
1210                  $partialtext = '';
1211                  if ($partial) {
1212                      $partialtext = ' (not complete; done to ' . userdate($lastindexeddoc,
1213                              get_string('strftimedatetimeshort', 'langconfig')) . ')';
1214                  }
1215  
1216                  $progress->output('Processed ' . $numrecords . ' records containing ' . $numdocs .
1217                          ' documents' . $batchinfo . ', in ' . $elapsed . ' seconds' . $partialtext . '.', 1);
1218              } else {
1219                  $progress->output('No new documents to index.', 1);
1220              }
1221  
1222              // Notify the engine this area is complete, and only mark times if true.
1223              if ($this->engine->area_index_complete($searcharea, $numdocs, $fullindex)) {
1224                  $sumdocs += $numdocs;
1225  
1226                  // Store last index run once documents have been committed to the search engine.
1227                  set_config($varname . '_indexingstart', $indexingstart, $componentconfigname);
1228                  set_config($varname . '_indexingend', (int)self::get_current_time(), $componentconfigname);
1229                  set_config($varname . '_docsignored', $numdocsignored, $componentconfigname);
1230                  set_config($varname . '_docsprocessed', $numdocs, $componentconfigname);
1231                  set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname);
1232                  if ($lastindexeddoc > 0) {
1233                      set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname);
1234                  }
1235                  if ($partial) {
1236                      set_config($varname . '_partial', 1, $componentconfigname);
1237                  } else {
1238                      unset_config($varname . '_partial', $componentconfigname);
1239                  }
1240              } else {
1241                  $progress->output('Engine reported error.');
1242              }
1243  
1244              if ($timelimit && (self::get_current_time() >= $stopat)) {
1245                  $progress->output('Stopping indexing due to time limit.');
1246                  break;
1247              }
1248          }
1249  
1250          if ($sumdocs > 0) {
1251              $event = \core\event\search_indexed::create(
1252                      array('context' => \context_system::instance()));
1253              $event->trigger();
1254          }
1255  
1256          $this->engine->index_complete($sumdocs, $fullindex);
1257  
1258          return (bool)$sumdocs;
1259      }
1260  
1261      /**
1262       * Indexes or reindexes a specific context of the system, e.g. one course.
1263       *
1264       * The function returns an object with field 'complete' (true or false).
1265       *
1266       * This function supports partial indexing via the time limit parameter. If the time limit
1267       * expires, it will return values for $startfromarea and $startfromtime which can be passed
1268       * next time to continue indexing.
1269       *
1270       * @param \context $context Context to restrict index.
1271       * @param string $singleareaid If specified, indexes only the given area.
1272       * @param float $timelimit Time limit in seconds (0 = no time limit)
1273       * @param \progress_trace|null $progress Optional class for tracking progress
1274       * @param string $startfromarea Area to start from
1275       * @param int $startfromtime Timestamp to start from
1276       * @return \stdClass Object indicating success
1277       */
1278      public function index_context($context, $singleareaid = '', $timelimit = 0,
1279              \progress_trace $progress = null, $startfromarea = '', $startfromtime = 0) {
1280          if (!$progress) {
1281              $progress = new \null_progress_trace();
1282          }
1283  
1284          // Work out time to stop, if limited.
1285          if ($timelimit) {
1286              // Decide time to stop.
1287              $stopat = self::get_current_time() + $timelimit;
1288          }
1289  
1290          // No PHP time limit.
1291          \core_php_time_limit::raise();
1292  
1293          // Notify the engine that an index starting.
1294          $this->engine->index_starting(false);
1295  
1296          $sumdocs = 0;
1297  
1298          // Get all search areas, in consistent order.
1299          $searchareas = $this->get_search_areas_list(true);
1300          ksort($searchareas);
1301  
1302          // Are we skipping past some that were handled previously?
1303          $skipping = $startfromarea ? true : false;
1304  
1305          foreach ($searchareas as $areaid => $searcharea) {
1306              // If we're only processing one area id, skip all the others.
1307              if ($singleareaid && $singleareaid !== $areaid) {
1308                  continue;
1309              }
1310  
1311              // If we're skipping to a later area, continue through the loop.
1312              $referencestarttime = 0;
1313              if ($skipping) {
1314                  if ($areaid !== $startfromarea) {
1315                      continue;
1316                  }
1317                  // Stop skipping and note the reference start time.
1318                  $skipping = false;
1319                  $referencestarttime = $startfromtime;
1320              }
1321  
1322              $progress->output('Processing area: ' . $searcharea->get_visible_name());
1323  
1324              $elapsed = self::get_current_time();
1325  
1326              // Get the recordset of all documents from the area for this context.
1327              $recordset = $searcharea->get_document_recordset($referencestarttime, $context);
1328              if (!$recordset) {
1329                  if ($recordset === null) {
1330                      $progress->output('Skipping (not relevant to context).', 1);
1331                  } else {
1332                      $progress->output('Skipping (does not support context indexing).', 1);
1333                  }
1334                  continue;
1335              }
1336  
1337              // Notify the engine that an area is starting.
1338              $this->engine->area_index_starting($searcharea, false);
1339  
1340              // Work out search options.
1341              $options = [];
1342              $options['indexfiles'] = $this->engine->file_indexing_enabled() &&
1343                      $searcharea->uses_file_indexing();
1344              if ($timelimit) {
1345                  $options['stopat'] = $stopat;
1346              }
1347  
1348              // Construct iterator which will use get_document on the recordset results.
1349              $iterator = new \core\dml\recordset_walk($recordset,
1350                      array($searcharea, 'get_document'), $options);
1351  
1352              // Use this iterator to add documents.
1353              $result = $this->engine->add_documents($iterator, $searcharea, $options);
1354              $batchinfo = '';
1355              if (count($result) === 6) {
1356                  [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial, $batches] = $result;
1357                  // Only show the batch count if we actually batched any requests.
1358                  if ($batches !== $numdocs + $numdocsignored) {
1359                      $batchinfo = ' (' . $batches . ' batch' . ($batches === 1 ? '' : 'es') . ')';
1360                  }
1361              } else if (count($result) === 5) {
1362                  // Backward compatibility for engines that don't return a batch count.
1363                  [$numrecords, $numdocs, $numdocsignored, $lastindexeddoc, $partial] = $result;
1364                  // Deprecated since Moodle 3.10 MDL-68690.
1365                  // TODO: MDL-68776 This will be deleted in Moodle 4.2 (as should the below bit).
1366                  debugging('engine::add_documents() should return $batches (5-value return is deprecated)',
1367                          DEBUG_DEVELOPER);
1368              } else {
1369                  // Backward compatibility for engines that don't support partial adding.
1370                  list($numrecords, $numdocs, $numdocsignored, $lastindexeddoc) = $result;
1371                  debugging('engine::add_documents() should return $partial (4-value return is deprecated)',
1372                          DEBUG_DEVELOPER);
1373                  $partial = false;
1374              }
1375  
1376              if ($numdocs > 0) {
1377                  $elapsed = round((self::get_current_time() - $elapsed), 3);
1378                  $progress->output('Processed ' . $numrecords . ' records containing ' . $numdocs .
1379                          ' documents' . $batchinfo . ', in ' . $elapsed . ' seconds' .
1380                          ($partial ? ' (not complete)' : '') . '.', 1);
1381              } else {
1382                  $progress->output('No documents to index.', 1);
1383              }
1384  
1385              // Notify the engine this area is complete, but don't store any times as this is not
1386              // part of the 'normal' search index.
1387              if (!$this->engine->area_index_complete($searcharea, $numdocs, false)) {
1388                  $progress->output('Engine reported error.', 1);
1389              }
1390  
1391              if ($partial && $timelimit && (self::get_current_time() >= $stopat)) {
1392                  $progress->output('Stopping indexing due to time limit.');
1393                  break;
1394              }
1395          }
1396  
1397          if ($sumdocs > 0) {
1398              $event = \core\event\search_indexed::create(
1399                      array('context' => $context));
1400              $event->trigger();
1401          }
1402  
1403          $this->engine->index_complete($sumdocs, false);
1404  
1405          // Indicate in result whether we completed indexing, or only part of it.
1406          $result = new \stdClass();
1407          if ($partial) {
1408              $result->complete = false;
1409              $result->startfromarea = $areaid;
1410              $result->startfromtime = $lastindexeddoc;
1411          } else {
1412              $result->complete = true;
1413          }
1414          return $result;
1415      }
1416  
1417      /**
1418       * Resets areas config.
1419       *
1420       * @throws \moodle_exception
1421       * @param string $areaid
1422       * @return void
1423       */
1424      public function reset_config($areaid = false) {
1425  
1426          if (!empty($areaid)) {
1427              $searchareas = array();
1428              if (!$searchareas[$areaid] = static::get_search_area($areaid)) {
1429                  throw new \moodle_exception('errorareanotavailable', 'search', '', $areaid);
1430              }
1431          } else {
1432              // Only the enabled ones.
1433              $searchareas = static::get_search_areas_list(true);
1434          }
1435  
1436          foreach ($searchareas as $searcharea) {
1437              list($componentname, $varname) = $searcharea->get_config_var_name();
1438              $config = $searcharea->get_config();
1439  
1440              foreach ($config as $key => $value) {
1441                  // We reset them all but the enable/disabled one.
1442                  if ($key !== $varname . '_enabled') {
1443                      set_config($key, 0, $componentname);
1444                  }
1445              }
1446          }
1447      }
1448  
1449      /**
1450       * Deletes an area's documents or all areas documents.
1451       *
1452       * @param string $areaid The area id or false for all
1453       * @return void
1454       */
1455      public function delete_index($areaid = false) {
1456          if (!empty($areaid)) {
1457              $this->engine->delete($areaid);
1458              $this->reset_config($areaid);
1459          } else {
1460              $this->engine->delete();
1461              $this->reset_config();
1462          }
1463      }
1464  
1465      /**
1466       * Deletes index by id.
1467       *
1468       * @param int Solr Document string $id
1469       */
1470      public function delete_index_by_id($id) {
1471          $this->engine->delete_by_id($id);
1472      }
1473  
1474      /**
1475       * Returns search areas configuration.
1476       *
1477       * @param \core_search\base[] $searchareas
1478       * @return \stdClass[] $configsettings
1479       */
1480      public function get_areas_config($searchareas) {
1481  
1482          $vars = array('indexingstart', 'indexingend', 'lastindexrun', 'docsignored',
1483                  'docsprocessed', 'recordsprocessed', 'partial');
1484  
1485          $configsettings = [];
1486          foreach ($searchareas as $searcharea) {
1487  
1488              $areaid = $searcharea->get_area_id();
1489  
1490              $configsettings[$areaid] = new \stdClass();
1491              list($componentname, $varname) = $searcharea->get_config_var_name();
1492  
1493              if (!$searcharea->is_enabled()) {
1494                  // We delete all indexed data on disable so no info.
1495                  foreach ($vars as $var) {
1496                      $configsettings[$areaid]->{$var} = 0;
1497                  }
1498              } else {
1499                  foreach ($vars as $var) {
1500                      $configsettings[$areaid]->{$var} = get_config($componentname, $varname .'_' . $var);
1501                  }
1502              }
1503  
1504              // Formatting the time.
1505              if (!empty($configsettings[$areaid]->lastindexrun)) {
1506                  $configsettings[$areaid]->lastindexrun = userdate($configsettings[$areaid]->lastindexrun);
1507              } else {
1508                  $configsettings[$areaid]->lastindexrun = get_string('never');
1509              }
1510          }
1511          return $configsettings;
1512      }
1513  
1514      /**
1515       * Triggers search_results_viewed event
1516       *
1517       * Other data required:
1518       * - q: The query string
1519       * - page: The page number
1520       * - title: Title filter
1521       * - areaids: Search areas filter
1522       * - courseids: Courses filter
1523       * - timestart: Time start filter
1524       * - timeend: Time end filter
1525       *
1526       * @since Moodle 3.2
1527       * @param array $other Other info for the event.
1528       * @return \core\event\search_results_viewed
1529       */
1530      public static function trigger_search_results_viewed($other) {
1531          $event = \core\event\search_results_viewed::create([
1532              'context' => \context_system::instance(),
1533              'other' => $other
1534          ]);
1535          $event->trigger();
1536  
1537          return $event;
1538      }
1539  
1540      /**
1541       * Checks whether a classname is of an actual search area.
1542       *
1543       * @param string $classname
1544       * @return bool
1545       */
1546      protected static function is_search_area($classname) {
1547          if (is_subclass_of($classname, 'core_search\base')) {
1548              return (new \ReflectionClass($classname))->isInstantiable();
1549          }
1550  
1551          return false;
1552      }
1553  
1554      /**
1555       * Requests that a specific context is indexed by the scheduled task. The context will be
1556       * added to a queue which is processed by the task.
1557       *
1558       * This is used after a restore to ensure that restored items are indexed, even though their
1559       * modified time will be older than the latest indexed. It is also used by the 'Gradual reindex'
1560       * admin feature from the search areas screen.
1561       *
1562       * @param \context $context Context to index within
1563       * @param string $areaid Area to index, '' = all areas
1564       * @param int $priority Priority (INDEX_PRIORITY_xx constant)
1565       */
1566      public static function request_index(\context $context, $areaid = '',
1567              $priority = self::INDEX_PRIORITY_NORMAL) {
1568          global $DB;
1569  
1570          // Check through existing requests for this context or any parent context.
1571          list ($contextsql, $contextparams) = $DB->get_in_or_equal(
1572                  $context->get_parent_context_ids(true));
1573          $existing = $DB->get_records_select('search_index_requests',
1574                  'contextid ' . $contextsql, $contextparams, '',
1575                  'id, searcharea, partialarea, indexpriority');
1576          foreach ($existing as $rec) {
1577              // If we haven't started processing the existing request yet, and it covers the same
1578              // area (or all areas) then that will be sufficient so don't add anything else.
1579              if ($rec->partialarea === '' && ($rec->searcharea === $areaid || $rec->searcharea === '')) {
1580                  // If the existing request has the same (or higher) priority, no need to add anything.
1581                  if ($rec->indexpriority >= $priority) {
1582                      return;
1583                  }
1584                  // The existing request has lower priority. If it is exactly the same, then just
1585                  // adjust the priority of the existing request.
1586                  if ($rec->searcharea === $areaid) {
1587                      $DB->set_field('search_index_requests', 'indexpriority', $priority,
1588                              ['id' => $rec->id]);
1589                      return;
1590                  }
1591                  // The existing request would cover this area but is a lower priority. We need to
1592                  // add the new request even though that means we will index part of it twice.
1593              }
1594          }
1595  
1596          // No suitable existing request, so add a new one.
1597          $newrecord = [ 'contextid' => $context->id, 'searcharea' => $areaid,
1598                  'timerequested' => (int)self::get_current_time(),
1599                  'partialarea' => '', 'partialtime' => 0,
1600                  'indexpriority' => $priority ];
1601          $DB->insert_record('search_index_requests', $newrecord);
1602      }
1603  
1604      /**
1605       * Processes outstanding index requests. This will take the first item from the queue (taking
1606       * account the indexing priority) and process it, continuing until an optional time limit is
1607       * reached.
1608       *
1609       * If there are no index requests, the function will do nothing.
1610       *
1611       * @param float $timelimit Time limit (0 = none)
1612       * @param \progress_trace|null $progress Optional progress indicator
1613       */
1614      public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) {
1615          global $DB;
1616  
1617          if (!$progress) {
1618              $progress = new \null_progress_trace();
1619          }
1620  
1621          $before = self::get_current_time();
1622          if ($timelimit) {
1623              $stopat = $before + $timelimit;
1624          }
1625          while (true) {
1626              // Retrieve first request, using fully defined ordering.
1627              $requests = $DB->get_records('search_index_requests', null,
1628                      'indexpriority DESC, timerequested, contextid, searcharea',
1629                      'id, contextid, searcharea, partialarea, partialtime', 0, 1);
1630              if (!$requests) {
1631                  // If there are no more requests, stop.
1632                  break;
1633              }
1634              $request = reset($requests);
1635  
1636              // Calculate remaining time.
1637              $remainingtime = 0;
1638              $beforeindex = self::get_current_time();
1639              if ($timelimit) {
1640                  $remainingtime = $stopat - $beforeindex;
1641  
1642                  // If the time limit expired already, stop now. (Otherwise we might accidentally
1643                  // index with no time limit or a negative time limit.)
1644                  if ($remainingtime <= 0) {
1645                      break;
1646                  }
1647              }
1648  
1649              // Show a message before each request, indicating what will be indexed.
1650              $context = \context::instance_by_id($request->contextid, IGNORE_MISSING);
1651              if (!$context) {
1652                  $DB->delete_records('search_index_requests', ['id' => $request->id]);
1653                  $progress->output('Skipped deleted context: ' . $request->contextid);
1654                  continue;
1655              }
1656              $contextname = $context->get_context_name();
1657              if ($request->searcharea) {
1658                  $contextname .= ' (search area: ' . $request->searcharea . ')';
1659              }
1660              $progress->output('Indexing requested context: ' . $contextname);
1661  
1662              // Actually index the context.
1663              $result = $this->index_context($context, $request->searcharea, $remainingtime,
1664                      $progress, $request->partialarea, $request->partialtime);
1665  
1666              // Work out shared part of message.
1667              $endmessage = $contextname . ' (' . round(self::get_current_time() - $beforeindex, 1) . 's)';
1668  
1669              // Update database table and continue/stop as appropriate.
1670              if ($result->complete) {
1671                  // If we completed the request, remove it from the table.
1672                  $DB->delete_records('search_index_requests', ['id' => $request->id]);
1673                  $progress->output('Completed requested context: ' . $endmessage);
1674              } else {
1675                  // If we didn't complete the request, store the partial details (how far it got).
1676                  $DB->update_record('search_index_requests', ['id' => $request->id,
1677                          'partialarea' => $result->startfromarea,
1678                          'partialtime' => $result->startfromtime]);
1679                  $progress->output('Ending requested context: ' . $endmessage);
1680  
1681                  // The time limit must have expired, so stop looping.
1682                  break;
1683              }
1684          }
1685      }
1686  
1687      /**
1688       * Gets information about the request queue, in the form of a plain object suitable for passing
1689       * to a template for rendering.
1690       *
1691       * @return \stdClass Information about queued index requests
1692       */
1693      public function get_index_requests_info() {
1694          global $DB;
1695  
1696          $result = new \stdClass();
1697  
1698          $result->total = $DB->count_records('search_index_requests');
1699          $result->topten = $DB->get_records('search_index_requests', null,
1700                  'indexpriority DESC, timerequested, contextid, searcharea',
1701                  'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority',
1702                  0, 10);
1703          foreach ($result->topten as $item) {
1704              $context = \context::instance_by_id($item->contextid);
1705              $item->contextlink = \html_writer::link($context->get_url(),
1706                      s($context->get_context_name()));
1707              if ($item->searcharea) {
1708                  $item->areaname = $this->get_search_area($item->searcharea)->get_visible_name();
1709              }
1710              if ($item->partialarea) {
1711                  $item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name();
1712              }
1713              switch ($item->indexpriority) {
1714                  case self::INDEX_PRIORITY_REINDEXING :
1715                      $item->priorityname = get_string('priority_reindexing', 'search');
1716                      break;
1717                  case self::INDEX_PRIORITY_NORMAL :
1718                      $item->priorityname = get_string('priority_normal', 'search');
1719                      break;
1720              }
1721          }
1722  
1723          // Normalise array indices.
1724          $result->topten = array_values($result->topten);
1725  
1726          if ($result->total > 10) {
1727              $result->ellipsis = true;
1728          }
1729  
1730          return $result;
1731      }
1732  
1733      /**
1734       * Gets current time for use in search system.
1735       *
1736       * Note: This should be replaced with generic core functionality once possible (see MDL-60644).
1737       *
1738       * @return float Current time in seconds (with decimals)
1739       */
1740      public static function get_current_time() {
1741          if (PHPUNIT_TEST && self::$phpunitfaketime) {
1742              return self::$phpunitfaketime;
1743          }
1744          return microtime(true);
1745      }
1746  
1747      /**
1748       * Check if search area categories functionality is enabled.
1749       *
1750       * @return bool
1751       */
1752      public static function is_search_area_categories_enabled() {
1753          return !empty(get_config('core', 'searchenablecategories'));
1754      }
1755  
1756      /**
1757       * Check if all results category should be hidden.
1758       *
1759       * @return bool
1760       */
1761      public static function should_hide_all_results_category() {
1762          return get_config('core', 'searchhideallcategory');
1763      }
1764  
1765      /**
1766       * Returns default search area category name.
1767       *
1768       * @return string
1769       */
1770      public static function get_default_area_category_name() {
1771          $default = get_config('core', 'searchdefaultcategory');
1772  
1773          if (empty($default)) {
1774              $default = self::SEARCH_AREA_CATEGORY_ALL;
1775          }
1776  
1777          if ($default == self::SEARCH_AREA_CATEGORY_ALL && self::should_hide_all_results_category()) {
1778              $default = self::SEARCH_AREA_CATEGORY_COURSE_CONTENT;
1779          }
1780  
1781          return $default;
1782      }
1783  
1784      /**
1785       * Get a list of all courses limited by ids if required.
1786       *
1787       * @param array|false $limitcourseids An array of course ids to limit the search to. False for no limiting.
1788       * @return array
1789       */
1790      protected function get_all_courses($limitcourseids) {
1791          global $DB;
1792  
1793          if ($limitcourseids) {
1794              list ($coursesql, $courseparams) = $DB->get_in_or_equal($limitcourseids);
1795              $coursesql = 'id ' . $coursesql;
1796          } else {
1797              $coursesql = '';
1798              $courseparams = [];
1799          }
1800  
1801          // Get courses using the same list of fields from enrol_get_my_courses.
1802          return $DB->get_records_select('course', $coursesql, $courseparams, '',
1803              'id, category, sortorder, shortname, fullname, idnumber, startdate, visible, ' .
1804              'groupmode, groupmodeforce, cacherev');
1805      }
1806  
1807      /**
1808       * Get a list of courses as user can access.
1809       *
1810       * @param bool $allaccessible Include courses user is not enrolled in, but can access.
1811       * @return array
1812       */
1813      protected function get_my_courses($allaccessible) {
1814          return enrol_get_my_courses(array('id', 'cacherev'), 'id', 0, [], $allaccessible);
1815      }
1816  
1817      /**
1818       * Check if search all courses setting is enabled.
1819       *
1820       * @return bool
1821       */
1822      public static function include_all_courses() {
1823          return !empty(get_config('core', 'searchincludeallcourses'));
1824      }
1825  
1826      /**
1827       * Cleans up non existing search area.
1828       *
1829       * 1. Remove all configs from {config_plugins} table.
1830       * 2. Delete all related indexed documents.
1831       *
1832       * @param string $areaid Search area id.
1833       */
1834      public static function clean_up_non_existing_area($areaid) {
1835          global $DB;
1836  
1837          if (!empty(self::get_search_area($areaid))) {
1838              throw new \coding_exception("Area $areaid exists. Please use appropriate search area class to manipulate the data.");
1839          }
1840  
1841          $parts = self::parse_areaid($areaid);
1842  
1843          $plugin = $parts[0];
1844          $configprefix = $parts[1];
1845  
1846          foreach (base::get_settingnames() as $settingname) {
1847              $name = $configprefix. $settingname;
1848              $DB->delete_records('config_plugins', ['name' => $name, 'plugin' => $plugin]);
1849          }
1850  
1851          $engine = self::instance()->get_engine();
1852          $engine->delete($areaid);
1853      }
1854  
1855      /**
1856       * Informs the search system that a context has been deleted.
1857       *
1858       * This will clear the data from the search index, where the search engine supports that.
1859       *
1860       * This function does not usually throw an exception (so as not to get in the way of the
1861       * context deletion finishing).
1862       *
1863       * This is called for all types of context deletion.
1864       *
1865       * @param \context $context Context object that has just been deleted
1866       */
1867      public static function context_deleted(\context $context) {
1868          if (self::is_indexing_enabled()) {
1869              try {
1870                  // Hold on, are we deleting a course? If so, and this context is part of the course,
1871                  // then don't bother to send a delete because we delete the whole course at once
1872                  // later.
1873                  if (!empty(self::$coursedeleting)) {
1874                      $coursecontext = $context->get_course_context(false);
1875                      if ($coursecontext && array_key_exists($coursecontext->instanceid, self::$coursedeleting)) {
1876                          // Skip further processing.
1877                          return;
1878                      }
1879                  }
1880  
1881                  $engine = self::instance()->get_engine();
1882                  $engine->delete_index_for_context($context->id);
1883              } catch (\moodle_exception $e) {
1884                  debugging('Error deleting search index data for context ' . $context->id . ': ' . $e->getMessage());
1885              }
1886          }
1887      }
1888  
1889      /**
1890       * Informs the search system that a course is about to be deleted.
1891       *
1892       * This prevents it from sending hundreds of 'delete context' updates for all the individual
1893       * contexts that are deleted.
1894       *
1895       * If you call this, you must call course_deleting_finish().
1896       *
1897       * @param int $courseid Course id that is being deleted
1898       */
1899      public static function course_deleting_start(int $courseid) {
1900          self::$coursedeleting[$courseid] = true;
1901      }
1902  
1903      /**
1904       * Informs the search engine that a course has now been deleted.
1905       *
1906       * This causes the search engine to actually delete the index for the whole course.
1907       *
1908       * @param int $courseid Course id that no longer exists
1909       */
1910      public static function course_deleting_finish(int $courseid) {
1911          if (!array_key_exists($courseid, self::$coursedeleting)) {
1912              // Show a debug warning. It doesn't actually matter very much, as we will now delete
1913              // the course data anyhow.
1914              debugging('course_deleting_start not called before deletion of ' . $courseid, DEBUG_DEVELOPER);
1915          }
1916          unset(self::$coursedeleting[$courseid]);
1917  
1918          if (self::is_indexing_enabled()) {
1919              try {
1920                  $engine = self::instance()->get_engine();
1921                  $engine->delete_index_for_course($courseid);
1922              } catch (\moodle_exception $e) {
1923                  debugging('Error deleting search index data for course ' . $courseid . ': ' . $e->getMessage());
1924              }
1925          }
1926      }
1927  }