Search moodle.org's
Developer Documentation

See Release Notes
Long Term Support Release

  • Bug fixes for general core bugs in 4.1.x will end 13 November 2023 (12 months).
  • Bug fixes for security issues in 4.1.x will end 10 November 2025 (36 months).
  • PHP version: minimum PHP 7.4.0 Note: minimum PHP version has increased since Moodle 4.0. PHP 8.0.x is supported too.

Differences Between: [Versions 310 and 401] [Versions 311 and 401] [Versions 39 and 401] [Versions 400 and 401] [Versions 401 and 402] [Versions 401 and 403]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  namespace search_solr;
  18  
  19  defined('MOODLE_INTERNAL') || die();
  20  
  21  global $CFG;
  22  require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
  23  require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php');
  24  require_once($CFG->dirroot . '/search/engine/solr/tests/fixtures/testable_engine.php');
  25  
  26  /**
  27   * Solr search engine base unit tests.
  28   *
  29   * Required params:
  30   * - define('TEST_SEARCH_SOLR_HOSTNAME', '127.0.0.1');
  31   * - define('TEST_SEARCH_SOLR_PORT', '8983');
  32   * - define('TEST_SEARCH_SOLR_INDEXNAME', 'unittest');
  33   *
  34   * Optional params:
  35   * - define('TEST_SEARCH_SOLR_USERNAME', '');
  36   * - define('TEST_SEARCH_SOLR_PASSWORD', '');
  37   * - define('TEST_SEARCH_SOLR_SSLCERT', '');
  38   * - define('TEST_SEARCH_SOLR_SSLKEY', '');
  39   * - define('TEST_SEARCH_SOLR_KEYPASSWORD', '');
  40   * - define('TEST_SEARCH_SOLR_CAINFOCERT', '');
  41   *
  42   * @package     search_solr
  43   * @category    test
  44   * @copyright   2015 David Monllao {@link http://www.davidmonllao.com}
  45   * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  46   *
  47   * @runTestsInSeparateProcesses
  48   */
  49  class engine_test extends \advanced_testcase {
  50  
  51      /**
  52       * @var \core_search\manager
  53       */
  54      protected $search = null;
  55  
  56      /**
  57       * @var Instace of core_search_generator.
  58       */
  59      protected $generator = null;
  60  
  61      /**
  62       * @var Instace of testable_engine.
  63       */
  64      protected $engine = null;
  65  
  66      public function setUp(): void {
  67          $this->resetAfterTest();
  68          set_config('enableglobalsearch', true);
  69          set_config('searchengine', 'solr');
  70  
  71          if (!function_exists('solr_get_version')) {
  72              $this->markTestSkipped('Solr extension is not loaded.');
  73          }
  74  
  75          if (!defined('TEST_SEARCH_SOLR_HOSTNAME') || !defined('TEST_SEARCH_SOLR_INDEXNAME') ||
  76                  !defined('TEST_SEARCH_SOLR_PORT')) {
  77              $this->markTestSkipped('Solr extension test server not set.');
  78          }
  79  
  80          set_config('server_hostname', TEST_SEARCH_SOLR_HOSTNAME, 'search_solr');
  81          set_config('server_port', TEST_SEARCH_SOLR_PORT, 'search_solr');
  82          set_config('indexname', TEST_SEARCH_SOLR_INDEXNAME, 'search_solr');
  83  
  84          if (defined('TEST_SEARCH_SOLR_USERNAME')) {
  85              set_config('server_username', TEST_SEARCH_SOLR_USERNAME, 'search_solr');
  86          }
  87  
  88          if (defined('TEST_SEARCH_SOLR_PASSWORD')) {
  89              set_config('server_password', TEST_SEARCH_SOLR_PASSWORD, 'search_solr');
  90          }
  91  
  92          if (defined('TEST_SEARCH_SOLR_SSLCERT')) {
  93              set_config('secure', true, 'search_solr');
  94              set_config('ssl_cert', TEST_SEARCH_SOLR_SSLCERT, 'search_solr');
  95          }
  96  
  97          if (defined('TEST_SEARCH_SOLR_SSLKEY')) {
  98              set_config('ssl_key', TEST_SEARCH_SOLR_SSLKEY, 'search_solr');
  99          }
 100  
 101          if (defined('TEST_SEARCH_SOLR_KEYPASSWORD')) {
 102              set_config('ssl_keypassword', TEST_SEARCH_SOLR_KEYPASSWORD, 'search_solr');
 103          }
 104  
 105          if (defined('TEST_SEARCH_SOLR_CAINFOCERT')) {
 106              set_config('ssl_cainfo', TEST_SEARCH_SOLR_CAINFOCERT, 'search_solr');
 107          }
 108  
 109          set_config('fileindexing', 1, 'search_solr');
 110  
 111          // We are only test indexing small string files, so setting this as low as we can.
 112          set_config('maxindexfilekb', 1, 'search_solr');
 113  
 114          $this->generator = self::getDataGenerator()->get_plugin_generator('core_search');
 115          $this->generator->setup();
 116  
 117          // Inject search solr engine into the testable core search as we need to add the mock
 118          // search component to it.
 119          $this->engine = new \search_solr\testable_engine();
 120          $this->search = \testable_core_search::instance($this->engine);
 121          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 122          $this->search->add_search_area($areaid, new \core_mocksearch\search\mock_search_area());
 123  
 124          $this->setAdminUser();
 125  
 126          // Cleanup before doing anything on it as the index it is out of this test control.
 127          $this->search->delete_index();
 128  
 129          // Add moodle fields if they don't exist.
 130          $schema = new \search_solr\schema($this->engine);
 131          $schema->setup(false);
 132      }
 133  
 134      public function tearDown(): void {
 135          // For unit tests before PHP 7, teardown is called even on skip. So only do our teardown if we did setup.
 136          if ($this->generator) {
 137              // Moodle DML freaks out if we don't teardown the temp table after each run.
 138              $this->generator->teardown();
 139              $this->generator = null;
 140          }
 141      }
 142  
 143      /**
 144       * Simple data provider to allow tests to be run with file indexing on and off.
 145       */
 146      public function file_indexing_provider() {
 147          return array(
 148              'file-indexing-on' => array(1),
 149              'file-indexing-off' => array(0)
 150          );
 151      }
 152  
 153      public function test_connection() {
 154          $this->assertTrue($this->engine->is_server_ready());
 155      }
 156  
 157      /**
 158       * Tests that the alternate settings are used when configured.
 159       */
 160      public function test_alternate_settings() {
 161          // Index a couple of things.
 162          $this->generator->create_record();
 163          $this->generator->create_record();
 164          $this->search->index();
 165  
 166          // By default settings, alternates are not set.
 167          $this->assertFalse($this->engine->has_alternate_configuration());
 168  
 169          // Set up all the config the same as normal.
 170          foreach (['server_hostname', 'indexname', 'secure', 'server_port',
 171                  'server_username', 'server_password'] as $setting) {
 172              set_config('alternate' . $setting, get_config('search_solr', $setting), 'search_solr');
 173          }
 174          // Also mess up the normal config.
 175          set_config('indexname', 'not_the_right_index_name', 'search_solr');
 176  
 177          // Construct a new engine using normal settings.
 178          $engine = new engine();
 179  
 180          // Now alternates are available.
 181          $this->assertTrue($engine->has_alternate_configuration());
 182  
 183          // But it won't actually work because of the bogus index name.
 184          $this->assertFalse($engine->is_server_ready() === true);
 185          $this->assertDebuggingCalled();
 186  
 187          // But if we construct one using alternate settings, it will work as normal.
 188          $engine = new engine(true);
 189          $this->assertTrue($engine->is_server_ready());
 190  
 191          // Including finding the search results.
 192          $this->assertCount(2, $engine->execute_query(
 193                  (object)['q' => 'message'], (object)['everything' => true]));
 194      }
 195  
 196      /**
 197       * @dataProvider file_indexing_provider
 198       */
 199      public function test_index($fileindexing) {
 200          global $DB;
 201  
 202          $this->engine->test_set_config('fileindexing', $fileindexing);
 203  
 204          $record = new \stdClass();
 205          $record->timemodified = time() - 1;
 206          $this->generator->create_record($record);
 207  
 208          // Data gets into the search engine.
 209          $this->assertTrue($this->search->index());
 210  
 211          // Not anymore as everything was already added.
 212          sleep(1);
 213          $this->assertFalse($this->search->index());
 214  
 215          $this->generator->create_record();
 216  
 217          // Indexing again once there is new data.
 218          $this->assertTrue($this->search->index());
 219      }
 220  
 221      /**
 222       * Better keep this not very strict about which or how many results are returned as may depend on solr engine config.
 223       *
 224       * @dataProvider file_indexing_provider
 225       *
 226       * @return void
 227       */
 228      public function test_search($fileindexing) {
 229          global $USER, $DB;
 230  
 231          $this->engine->test_set_config('fileindexing', $fileindexing);
 232  
 233          $this->generator->create_record();
 234          $record = new \stdClass();
 235          $record->title = "Special title";
 236          $this->generator->create_record($record);
 237  
 238          $this->search->index();
 239  
 240          $querydata = new \stdClass();
 241          $querydata->q = 'message';
 242          $results = $this->search->search($querydata);
 243          $this->assertCount(2, $results);
 244  
 245          // Based on core_mocksearch\search\indexer.
 246          $this->assertEquals($USER->id, $results[0]->get('userid'));
 247          $this->assertEquals(\context_course::instance(SITEID)->id, $results[0]->get('contextid'));
 248  
 249          // Do a test to make sure we aren't searching non-query fields, like areaid.
 250          $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 251          $this->assertCount(0, $this->search->search($querydata));
 252          $querydata->q = 'message';
 253  
 254          sleep(1);
 255          $beforeadding = time();
 256          sleep(1);
 257          $this->generator->create_record();
 258          $this->search->index();
 259  
 260          // Timestart.
 261          $querydata->timestart = $beforeadding;
 262          $this->assertCount(1, $this->search->search($querydata));
 263  
 264          // Timeend.
 265          unset($querydata->timestart);
 266          $querydata->timeend = $beforeadding;
 267          $this->assertCount(2, $this->search->search($querydata));
 268  
 269          // Title.
 270          unset($querydata->timeend);
 271          $querydata->title = 'Special title';
 272          $this->assertCount(1, $this->search->search($querydata));
 273  
 274          // Course IDs.
 275          unset($querydata->title);
 276          $querydata->courseids = array(SITEID + 1);
 277          $this->assertCount(0, $this->search->search($querydata));
 278  
 279          $querydata->courseids = array(SITEID);
 280          $this->assertCount(3, $this->search->search($querydata));
 281  
 282          // Now try some area-id combinations.
 283          unset($querydata->courseids);
 284          $forumpostareaid = \core_search\manager::generate_areaid('mod_forum', 'post');
 285          $mockareaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 286  
 287          $querydata->areaids = array($forumpostareaid);
 288          $this->assertCount(0, $this->search->search($querydata));
 289  
 290          $querydata->areaids = array($forumpostareaid, $mockareaid);
 291          $this->assertCount(3, $this->search->search($querydata));
 292  
 293          $querydata->areaids = array($mockareaid);
 294          $this->assertCount(3, $this->search->search($querydata));
 295  
 296          $querydata->areaids = array();
 297          $this->assertCount(3, $this->search->search($querydata));
 298  
 299          // Check that index contents get updated.
 300          $this->generator->delete_all();
 301          $this->search->index(true);
 302          unset($querydata->title);
 303          $querydata->q = '*';
 304          $this->assertCount(0, $this->search->search($querydata));
 305      }
 306  
 307      /**
 308       * @dataProvider file_indexing_provider
 309       */
 310      public function test_delete($fileindexing) {
 311          $this->engine->test_set_config('fileindexing', $fileindexing);
 312  
 313          $this->generator->create_record();
 314          $this->generator->create_record();
 315          $this->search->index();
 316  
 317          $querydata = new \stdClass();
 318          $querydata->q = 'message';
 319  
 320          $this->assertCount(2, $this->search->search($querydata));
 321  
 322          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 323          $this->search->delete_index($areaid);
 324          $this->assertCount(0, $this->search->search($querydata));
 325      }
 326  
 327      /**
 328       * @dataProvider file_indexing_provider
 329       */
 330      public function test_alloweduserid($fileindexing) {
 331          $this->engine->test_set_config('fileindexing', $fileindexing);
 332  
 333          $area = new \core_mocksearch\search\mock_search_area();
 334  
 335          $record = $this->generator->create_record();
 336  
 337          // Get the doc and insert the default doc.
 338          $doc = $area->get_document($record);
 339          $this->engine->add_document($doc);
 340  
 341          $users = array();
 342          $users[] = $this->getDataGenerator()->create_user();
 343          $users[] = $this->getDataGenerator()->create_user();
 344          $users[] = $this->getDataGenerator()->create_user();
 345  
 346          // Add a record that only user 100 can see.
 347          $originalid = $doc->get('id');
 348  
 349          // Now add a custom doc for each user.
 350          foreach ($users as $user) {
 351              $doc = $area->get_document($record);
 352              $doc->set('id', $originalid.'-'.$user->id);
 353              $doc->set('owneruserid', $user->id);
 354              $this->engine->add_document($doc);
 355          }
 356  
 357          $this->engine->area_index_complete($area->get_area_id());
 358  
 359          $querydata = new \stdClass();
 360          $querydata->q = 'message';
 361          $querydata->title = $doc->get('title');
 362  
 363          // We are going to go through each user and see if they get the original and the owned doc.
 364          foreach ($users as $user) {
 365              $this->setUser($user);
 366  
 367              $results = $this->search->search($querydata);
 368              $this->assertCount(2, $results);
 369  
 370              $owned = 0;
 371              $notowned = 0;
 372  
 373              // We don't know what order we will get the results in, so we are doing this.
 374              foreach ($results as $result) {
 375                  $owneruserid = $result->get('owneruserid');
 376                  if (empty($owneruserid)) {
 377                      $notowned++;
 378                      $this->assertEquals(0, $owneruserid);
 379                      $this->assertEquals($originalid, $result->get('id'));
 380                  } else {
 381                      $owned++;
 382                      $this->assertEquals($user->id, $owneruserid);
 383                      $this->assertEquals($originalid.'-'.$user->id, $result->get('id'));
 384                  }
 385              }
 386  
 387              $this->assertEquals(1, $owned);
 388              $this->assertEquals(1, $notowned);
 389          }
 390  
 391          // Now test a user with no owned results.
 392          $otheruser = $this->getDataGenerator()->create_user();
 393          $this->setUser($otheruser);
 394  
 395          $results = $this->search->search($querydata);
 396          $this->assertCount(1, $results);
 397  
 398          $this->assertEquals(0, $results[0]->get('owneruserid'));
 399          $this->assertEquals($originalid, $results[0]->get('id'));
 400      }
 401  
 402      /**
 403       * @dataProvider file_indexing_provider
 404       */
 405      public function test_highlight($fileindexing) {
 406          global $PAGE;
 407  
 408          $this->engine->test_set_config('fileindexing', $fileindexing);
 409  
 410          $this->generator->create_record();
 411          $this->search->index();
 412  
 413          $querydata = new \stdClass();
 414          $querydata->q = 'message';
 415  
 416          $results = $this->search->search($querydata);
 417          $this->assertCount(1, $results);
 418  
 419          $result = reset($results);
 420  
 421          $regex = '|'.\search_solr\engine::HIGHLIGHT_START.'message'.\search_solr\engine::HIGHLIGHT_END.'|';
 422          $this->assertMatchesRegularExpression($regex, $result->get('content'));
 423  
 424          $searchrenderer = $PAGE->get_renderer('core_search');
 425          $exported = $result->export_for_template($searchrenderer);
 426  
 427          $regex = '|<span class="highlight">message</span>|';
 428          $this->assertMatchesRegularExpression($regex, $exported['content']);
 429      }
 430  
 431      public function test_export_file_for_engine() {
 432          // Get area to work with.
 433          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 434          $area = \core_search\manager::get_search_area($areaid);
 435  
 436          $record = $this->generator->create_record();
 437  
 438          $doc = $area->get_document($record);
 439          $filerecord = new \stdClass();
 440          $filerecord->timemodified  = 978310800;
 441          $file = $this->generator->create_file($filerecord);
 442          $doc->add_stored_file($file);
 443  
 444          $filearray = $doc->export_file_for_engine($file);
 445  
 446          $this->assertEquals(\core_search\manager::TYPE_FILE, $filearray['type']);
 447          $this->assertEquals($file->get_id(), $filearray['solr_fileid']);
 448          $this->assertEquals($file->get_contenthash(), $filearray['solr_filecontenthash']);
 449          $this->assertEquals(\search_solr\document::INDEXED_FILE_TRUE, $filearray['solr_fileindexstatus']);
 450          $this->assertEquals($file->get_filename(), $filearray['title']);
 451          $this->assertEquals(978310800, \search_solr\document::import_time_from_engine($filearray['modified']));
 452      }
 453  
 454      public function test_index_file() {
 455          // Very simple test.
 456          $file = $this->generator->create_file();
 457  
 458          $record = new \stdClass();
 459          $record->attachfileids = array($file->get_id());
 460          $this->generator->create_record($record);
 461  
 462          $this->search->index();
 463          $querydata = new \stdClass();
 464          $querydata->q = '"File contents"';
 465  
 466          $this->assertCount(1, $this->search->search($querydata));
 467      }
 468  
 469      public function test_reindexing_files() {
 470          // Get area to work with.
 471          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 472          $area = \core_search\manager::get_search_area($areaid);
 473  
 474          $record = $this->generator->create_record();
 475  
 476          $doc = $area->get_document($record);
 477  
 478          // Now we are going to make some files.
 479          $fs = get_file_storage();
 480          $syscontext = \context_system::instance();
 481  
 482          $files = array();
 483  
 484          $filerecord = new \stdClass();
 485          // We make enough so that we pass the 500 files threashold. That is the boundary when getting files.
 486          $boundary = 500;
 487          $top = (int)($boundary * 1.1);
 488          for ($i = 0; $i < $top; $i++) {
 489              $filerecord->filename  = 'searchfile'.$i;
 490              $filerecord->content = 'Some FileContents'.$i;
 491              $file = $this->generator->create_file($filerecord);
 492              $doc->add_stored_file($file);
 493              $files[] = $file;
 494          }
 495  
 496          // Add the doc with lots of files, then commit.
 497          $this->engine->add_document($doc, true);
 498          $this->engine->area_index_complete($area->get_area_id());
 499  
 500          // Indexes we are going to check. 0 means we will delete, 1 means we will keep.
 501          $checkfiles = array(
 502              0 => 0,                        // Check the begining of the set.
 503              1 => 1,
 504              2 => 0,
 505              ($top - 3) => 0,               // Check the end of the set.
 506              ($top - 2) => 1,
 507              ($top - 1) => 0,
 508              ($boundary - 2) => 0,          // Check at the boundary between fetch groups.
 509              ($boundary - 1) => 0,
 510              $boundary => 0,
 511              ($boundary + 1) => 0,
 512              ((int)($boundary * 0.5)) => 1, // Make sure we keep some middle ones.
 513              ((int)($boundary * 1.05)) => 1
 514          );
 515  
 516          $querydata = new \stdClass();
 517  
 518          // First, check that all the files are currently there.
 519          foreach ($checkfiles as $key => $unused) {
 520              $querydata->q = 'FileContents'.$key;
 521              $this->assertCount(1, $this->search->search($querydata));
 522              $querydata->q = 'searchfile'.$key;
 523              $this->assertCount(1, $this->search->search($querydata));
 524          }
 525  
 526          // Remove the files we want removed from the files array.
 527          foreach ($checkfiles as $key => $keep) {
 528              if (!$keep) {
 529                  unset($files[$key]);
 530              }
 531          }
 532  
 533          // And make us a new file to add.
 534          $filerecord->filename  = 'searchfileNew';
 535          $filerecord->content  = 'Some FileContentsNew';
 536          $files[] = $this->generator->create_file($filerecord);
 537          $checkfiles['New'] = 1;
 538  
 539          $doc = $area->get_document($record);
 540          foreach($files as $file) {
 541              $doc->add_stored_file($file);
 542          }
 543  
 544          // Reindex the document with the changed files.
 545          $this->engine->add_document($doc, true);
 546          $this->engine->area_index_complete($area->get_area_id());
 547  
 548          // Go through our check array, and see if the file is there or not.
 549          foreach ($checkfiles as $key => $keep) {
 550              $querydata->q = 'FileContents'.$key;
 551              $this->assertCount($keep, $this->search->search($querydata));
 552              $querydata->q = 'searchfile'.$key;
 553              $this->assertCount($keep, $this->search->search($querydata));
 554          }
 555  
 556          // Now check that we get one result when we search from something in all of them.
 557          $querydata->q = 'Some';
 558          $this->assertCount(1, $this->search->search($querydata));
 559      }
 560  
 561      /**
 562       * Test indexing a file we don't consider indexable.
 563       */
 564      public function test_index_filtered_file() {
 565          // Get area to work with.
 566          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
 567          $area = \core_search\manager::get_search_area($areaid);
 568  
 569          // Get a single record to make a doc from.
 570          $record = $this->generator->create_record();
 571  
 572          $doc = $area->get_document($record);
 573  
 574          // Now we are going to make some files.
 575          $fs = get_file_storage();
 576          $syscontext = \context_system::instance();
 577  
 578          // We need to make a file greater than 1kB in size, which is the lowest filter size.
 579          $filerecord = new \stdClass();
 580          $filerecord->filename = 'largefile';
 581          $filerecord->content = 'Some LargeFindContent to find.';
 582          for ($i = 0; $i < 200; $i++) {
 583              $filerecord->content .= ' The quick brown fox jumps over the lazy dog.';
 584          }
 585  
 586          $this->assertGreaterThan(1024, strlen($filerecord->content));
 587  
 588          $file = $this->generator->create_file($filerecord);
 589          $doc->add_stored_file($file);
 590  
 591          $filerecord->filename = 'smallfile';
 592          $filerecord->content = 'Some SmallFindContent to find.';
 593          $file = $this->generator->create_file($filerecord);
 594          $doc->add_stored_file($file);
 595  
 596          $this->engine->add_document($doc, true);
 597          $this->engine->area_index_complete($area->get_area_id());
 598  
 599          $querydata = new \stdClass();
 600          // We shouldn't be able to find the large file contents.
 601          $querydata->q = 'LargeFindContent';
 602          $this->assertCount(0, $this->search->search($querydata));
 603  
 604          // But we should be able to find the filename.
 605          $querydata->q = 'largefile';
 606          $this->assertCount(1, $this->search->search($querydata));
 607  
 608          // We should be able to find the small file contents.
 609          $querydata->q = 'SmallFindContent';
 610          $this->assertCount(1, $this->search->search($querydata));
 611  
 612          // And we should be able to find the filename.
 613          $querydata->q = 'smallfile';
 614          $this->assertCount(1, $this->search->search($querydata));
 615      }
 616  
 617      public function test_delete_by_id() {
 618          // First get files in the index.
 619          $file = $this->generator->create_file();
 620          $record = new \stdClass();
 621          $record->attachfileids = array($file->get_id());
 622          $this->generator->create_record($record);
 623          $this->generator->create_record($record);
 624          $this->search->index();
 625  
 626          $querydata = new \stdClass();
 627  
 628          // Then search to make sure they are there.
 629          $querydata->q = '"File contents"';
 630          $results = $this->search->search($querydata);
 631          $this->assertCount(2, $results);
 632  
 633          $first = reset($results);
 634          $deleteid = $first->get('id');
 635  
 636          $this->engine->delete_by_id($deleteid);
 637  
 638          // Check that we don't get a result for it anymore.
 639          $results = $this->search->search($querydata);
 640          $this->assertCount(1, $results);
 641          $result = reset($results);
 642          $this->assertNotEquals($deleteid, $result->get('id'));
 643      }
 644  
 645      /**
 646       * Test that expected results are returned, even with low check_access success rate.
 647       *
 648       * @dataProvider file_indexing_provider
 649       */
 650      public function test_solr_filling($fileindexing) {
 651          $this->engine->test_set_config('fileindexing', $fileindexing);
 652  
 653          $user1 = self::getDataGenerator()->create_user();
 654          $user2 = self::getDataGenerator()->create_user();
 655  
 656          // We are going to create a bunch of records that user 1 can see with 2 keywords.
 657          // Then we are going to create a bunch for user 2 with only 1 of the keywords.
 658          // If user 2 searches for both keywords, solr will return all of the user 1 results, then the user 2 results.
 659          // This is because the user 1 results will match 2 keywords, while the others will match only 1.
 660  
 661          $record = new \stdClass();
 662  
 663          // First create a bunch of records for user 1 to see.
 664          $record->denyuserids = array($user2->id);
 665          $record->content = 'Something1 Something2';
 666          $maxresults = (int)(\core_search\manager::MAX_RESULTS * .75);
 667          for ($i = 0; $i < $maxresults; $i++) {
 668              $this->generator->create_record($record);
 669          }
 670  
 671          // Then create a bunch of records for user 2 to see.
 672          $record->denyuserids = array($user1->id);
 673          $record->content = 'Something1';
 674          for ($i = 0; $i < $maxresults; $i++) {
 675              $this->generator->create_record($record);
 676          }
 677  
 678          $this->search->index();
 679  
 680          // Check that user 1 sees all their results.
 681          $this->setUser($user1);
 682          $querydata = new \stdClass();
 683          $querydata->q = 'Something1 Something2';
 684          $results = $this->search->search($querydata);
 685          $this->assertCount($maxresults, $results);
 686  
 687          // Check that user 2 will see theirs, even though they may be crouded out.
 688          $this->setUser($user2);
 689          $results = $this->search->search($querydata);
 690          $this->assertCount($maxresults, $results);
 691      }
 692  
 693      /**
 694       * Create 40 docs, that will be return from Solr in 10 hidden, 10 visible, 10 hidden, 10 visible if you query for:
 695       * Something1 Something2 Something3 Something4, with the specified user set.
 696       */
 697      protected function setup_user_hidden_docs($user) {
 698          // These results will come first, and will not be visible by the user.
 699          $record = new \stdClass();
 700          $record->denyuserids = array($user->id);
 701          $record->content = 'Something1 Something2 Something3 Something4';
 702          for ($i = 0; $i < 10; $i++) {
 703              $this->generator->create_record($record);
 704          }
 705  
 706          // These results will come second, and will  be visible by the user.
 707          unset($record->denyuserids);
 708          $record->content = 'Something1 Something2 Something3';
 709          for ($i = 0; $i < 10; $i++) {
 710              $this->generator->create_record($record);
 711          }
 712  
 713          // These results will come third, and will not be visible by the user.
 714          $record->denyuserids = array($user->id);
 715          $record->content = 'Something1 Something2';
 716          for ($i = 0; $i < 10; $i++) {
 717              $this->generator->create_record($record);
 718          }
 719  
 720          // These results will come fourth, and will be visible by the user.
 721          unset($record->denyuserids);
 722          $record->content = 'Something1 ';
 723          for ($i = 0; $i < 10; $i++) {
 724              $this->generator->create_record($record);
 725          }
 726      }
 727  
 728      /**
 729       * Test that counts are what we expect.
 730       *
 731       * @dataProvider file_indexing_provider
 732       */
 733      public function test_get_query_total_count($fileindexing) {
 734          $this->engine->test_set_config('fileindexing', $fileindexing);
 735  
 736          $user = self::getDataGenerator()->create_user();
 737          $this->setup_user_hidden_docs($user);
 738          $this->search->index();
 739  
 740          $this->setUser($user);
 741          $querydata = new \stdClass();
 742          $querydata->q = 'Something1 Something2 Something3 Something4';
 743  
 744          // In this first set, it should have determined the first 10 of 40 are bad, so there could be up to 30 left.
 745          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 5);
 746          $this->assertEquals(30, $this->engine->get_query_total_count());
 747          $this->assertCount(5, $results);
 748  
 749          // To get to 15, it has to process the first 10 that are bad, 10 that are good, 10 that are bad, then 5 that are good.
 750          // So we now know 20 are bad out of 40.
 751          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 15);
 752          $this->assertEquals(20, $this->engine->get_query_total_count());
 753          $this->assertCount(15, $results);
 754  
 755          // Try to get more then all, make sure we still see 20 count and 20 returned.
 756          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 30);
 757          $this->assertEquals(20, $this->engine->get_query_total_count());
 758          $this->assertCount(20, $results);
 759      }
 760  
 761      /**
 762       * Test that paged results are what we expect.
 763       *
 764       * @dataProvider file_indexing_provider
 765       */
 766      public function test_manager_paged_search($fileindexing) {
 767          $this->engine->test_set_config('fileindexing', $fileindexing);
 768  
 769          $user = self::getDataGenerator()->create_user();
 770          $this->setup_user_hidden_docs($user);
 771          $this->search->index();
 772  
 773          // Check that user 1 sees all their results.
 774          $this->setUser($user);
 775          $querydata = new \stdClass();
 776          $querydata->q = 'Something1 Something2 Something3 Something4';
 777  
 778          // On this first page, it should have determined the first 10 of 40 are bad, so there could be up to 30 left.
 779          $results = $this->search->paged_search($querydata, 0);
 780          $this->assertEquals(30, $results->totalcount);
 781          $this->assertCount(10, $results->results);
 782          $this->assertEquals(0, $results->actualpage);
 783  
 784          // On the second page, it should have found the next 10 bad ones, so we no know there are only 20 total.
 785          $results = $this->search->paged_search($querydata, 1);
 786          $this->assertEquals(20, $results->totalcount);
 787          $this->assertCount(10, $results->results);
 788          $this->assertEquals(1, $results->actualpage);
 789  
 790          // Try to get an additional page - we should get back page 1 results, since that is the last page with valid results.
 791          $results = $this->search->paged_search($querydata, 2);
 792          $this->assertEquals(20, $results->totalcount);
 793          $this->assertCount(10, $results->results);
 794          $this->assertEquals(1, $results->actualpage);
 795      }
 796  
 797      /**
 798       * Tests searching for results restricted to context id.
 799       */
 800      public function test_context_restriction() {
 801          // Use real search areas.
 802          $this->search->clear_static();
 803          $this->search->add_core_search_areas();
 804  
 805          // Create 2 courses and some forums.
 806          $generator = $this->getDataGenerator();
 807          $course1 = $generator->create_course(['fullname' => 'Course 1', 'summary' => 'xyzzy']);
 808          $contextc1 = \context_course::instance($course1->id);
 809          $course1forum1 = $generator->create_module('forum', ['course' => $course1,
 810                  'name' => 'C1F1', 'intro' => 'xyzzy']);
 811          $contextc1f1 = \context_module::instance($course1forum1->cmid);
 812          $course1forum2 = $generator->create_module('forum', ['course' => $course1,
 813                  'name' => 'C1F2', 'intro' => 'xyzzy']);
 814          $contextc1f2 = \context_module::instance($course1forum2->cmid);
 815          $course2 = $generator->create_course(['fullname' => 'Course 2', 'summary' => 'xyzzy']);
 816          $contextc2 = \context_course::instance($course1->id);
 817          $course2forum = $generator->create_module('forum', ['course' => $course2,
 818                  'name' => 'C2F', 'intro' => 'xyzzy']);
 819          $contextc2f = \context_module::instance($course2forum->cmid);
 820  
 821          // Index the courses and forums.
 822          $this->search->index();
 823  
 824          // Search as admin user should find everything.
 825          $querydata = new \stdClass();
 826          $querydata->q = 'xyzzy';
 827          $results = $this->search->search($querydata);
 828          $this->assert_result_titles(
 829                  ['Course 1', 'Course 2', 'C1F1', 'C1F2', 'C2F'], $results);
 830  
 831          // Admin user manually restricts results by context id to include one course and one forum.
 832          $querydata->contextids = [$contextc2f->id, $contextc1->id];
 833          $results = $this->search->search($querydata);
 834          $this->assert_result_titles(['Course 1', 'C2F'], $results);
 835  
 836          // Student enrolled in only one course, same restriction, only has the available results.
 837          $student2 = $generator->create_user();
 838          $generator->enrol_user($student2->id, $course2->id, 'student');
 839          $this->setUser($student2);
 840          $results = $this->search->search($querydata);
 841          $this->assert_result_titles(['C2F'], $results);
 842  
 843          // Student enrolled in both courses, same restriction, same results as admin.
 844          $student1 = $generator->create_user();
 845          $generator->enrol_user($student1->id, $course1->id, 'student');
 846          $generator->enrol_user($student1->id, $course2->id, 'student');
 847          $this->setUser($student1);
 848          $results = $this->search->search($querydata);
 849          $this->assert_result_titles(['Course 1', 'C2F'], $results);
 850  
 851          // Restrict both course and context.
 852          $querydata->courseids = [$course2->id];
 853          $results = $this->search->search($querydata);
 854          $this->assert_result_titles(['C2F'], $results);
 855          unset($querydata->courseids);
 856  
 857          // Restrict both area and context.
 858          $querydata->areaids = ['core_course-course'];
 859          $results = $this->search->search($querydata);
 860          $this->assert_result_titles(['Course 1'], $results);
 861  
 862          // Restrict area and context, incompatibly - this has no results (and doesn't do a query).
 863          $querydata->contextids = [$contextc2f->id];
 864          $results = $this->search->search($querydata);
 865          $this->assert_result_titles([], $results);
 866      }
 867  
 868      /**
 869       * Tests searching for results in groups, either by specified group ids or based on user
 870       * access permissions.
 871       */
 872      public function test_groups() {
 873          global $USER;
 874  
 875          // Use real search areas.
 876          $this->search->clear_static();
 877          $this->search->add_core_search_areas();
 878  
 879          // Create 2 courses and a selection of forums with different group mode.
 880          $generator = $this->getDataGenerator();
 881          $course1 = $generator->create_course(['fullname' => 'Course 1']);
 882          $forum1nogroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => NOGROUPS]);
 883          $forum1separategroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => SEPARATEGROUPS]);
 884          $forum1visiblegroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => VISIBLEGROUPS]);
 885          $course2 = $generator->create_course(['fullname' => 'Course 2']);
 886          $forum2separategroups = $generator->create_module('forum', ['course' => $course2, 'groupmode' => SEPARATEGROUPS]);
 887  
 888          // Create two groups on each course.
 889          $group1a = $generator->create_group(['courseid' => $course1->id]);
 890          $group1b = $generator->create_group(['courseid' => $course1->id]);
 891          $group2a = $generator->create_group(['courseid' => $course2->id]);
 892          $group2b = $generator->create_group(['courseid' => $course2->id]);
 893  
 894          // Create search records in each activity and (where relevant) in each group.
 895          $forumgenerator = $generator->get_plugin_generator('mod_forum');
 896          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
 897                  'forum' => $forum1nogroups->id, 'name' => 'F1NG', 'message' => 'xyzzy']);
 898          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
 899                  'forum' => $forum1separategroups->id, 'name' => 'F1SG-A',  'message' => 'xyzzy',
 900                  'groupid' => $group1a->id]);
 901          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
 902                  'forum' => $forum1separategroups->id, 'name' => 'F1SG-B', 'message' => 'xyzzy',
 903                  'groupid' => $group1b->id]);
 904          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
 905                  'forum' => $forum1visiblegroups->id, 'name' => 'F1VG-A', 'message' => 'xyzzy',
 906                  'groupid' => $group1a->id]);
 907          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
 908                  'forum' => $forum1visiblegroups->id, 'name' => 'F1VG-B', 'message' => 'xyzzy',
 909                  'groupid' => $group1b->id]);
 910          $forumgenerator->create_discussion(['course' => $course2->id, 'userid' => $USER->id,
 911                  'forum' => $forum2separategroups->id, 'name' => 'F2SG-A', 'message' => 'xyzzy',
 912                  'groupid' => $group2a->id]);
 913          $forumgenerator->create_discussion(['course' => $course2->id, 'userid' => $USER->id,
 914                  'forum' => $forum2separategroups->id, 'name' => 'F2SG-B', 'message' => 'xyzzy',
 915                  'groupid' => $group2b->id]);
 916  
 917          $this->search->index();
 918  
 919          // Search as admin user should find everything.
 920          $querydata = new \stdClass();
 921          $querydata->q = 'xyzzy';
 922          $results = $this->search->search($querydata);
 923          $this->assert_result_titles(
 924                  ['F1NG', 'F1SG-A', 'F1SG-B', 'F1VG-A', 'F1VG-B', 'F2SG-A', 'F2SG-B'], $results);
 925  
 926          // Admin user manually restricts results by groups.
 927          $querydata->groupids = [$group1b->id, $group2a->id];
 928          $results = $this->search->search($querydata);
 929          $this->assert_result_titles(['F1SG-B', 'F1VG-B', 'F2SG-A'], $results);
 930  
 931          // Student enrolled in both courses but no groups.
 932          $student1 = $generator->create_user();
 933          $generator->enrol_user($student1->id, $course1->id, 'student');
 934          $generator->enrol_user($student1->id, $course2->id, 'student');
 935          $this->setUser($student1);
 936  
 937          unset($querydata->groupids);
 938          $results = $this->search->search($querydata);
 939          $this->assert_result_titles(['F1NG', 'F1VG-A', 'F1VG-B'], $results);
 940  
 941          // Student enrolled in both courses and group A in both cases.
 942          $student2 = $generator->create_user();
 943          $generator->enrol_user($student2->id, $course1->id, 'student');
 944          $generator->enrol_user($student2->id, $course2->id, 'student');
 945          groups_add_member($group1a, $student2);
 946          groups_add_member($group2a, $student2);
 947          $this->setUser($student2);
 948  
 949          $results = $this->search->search($querydata);
 950          $this->assert_result_titles(['F1NG', 'F1SG-A', 'F1VG-A', 'F1VG-B', 'F2SG-A'], $results);
 951  
 952          // Manually restrict results to group B in course 1.
 953          $querydata->groupids = [$group1b->id];
 954          $results = $this->search->search($querydata);
 955          $this->assert_result_titles(['F1VG-B'], $results);
 956  
 957          // Manually restrict results to group A in course 1.
 958          $querydata->groupids = [$group1a->id];
 959          $results = $this->search->search($querydata);
 960          $this->assert_result_titles(['F1SG-A', 'F1VG-A'], $results);
 961  
 962          // Manager enrolled in both courses (has access all groups).
 963          $manager = $generator->create_user();
 964          $generator->enrol_user($manager->id, $course1->id, 'manager');
 965          $generator->enrol_user($manager->id, $course2->id, 'manager');
 966          $this->setUser($manager);
 967          unset($querydata->groupids);
 968          $results = $this->search->search($querydata);
 969          $this->assert_result_titles(
 970                  ['F1NG', 'F1SG-A', 'F1SG-B', 'F1VG-A', 'F1VG-B', 'F2SG-A', 'F2SG-B'], $results);
 971      }
 972  
 973      /**
 974       * Tests searching for results restricted to specific user id(s).
 975       */
 976      public function test_user_restriction() {
 977          // Use real search areas.
 978          $this->search->clear_static();
 979          $this->search->add_core_search_areas();
 980  
 981          // Create a course, a forum, and a glossary.
 982          $generator = $this->getDataGenerator();
 983          $course = $generator->create_course();
 984          $forum = $generator->create_module('forum', ['course' => $course->id]);
 985          $glossary = $generator->create_module('glossary', ['course' => $course->id]);
 986  
 987          // Create 3 user accounts, all enrolled as students on the course.
 988          $user1 = $generator->create_user();
 989          $user2 = $generator->create_user();
 990          $user3 = $generator->create_user();
 991          $generator->enrol_user($user1->id, $course->id, 'student');
 992          $generator->enrol_user($user2->id, $course->id, 'student');
 993          $generator->enrol_user($user3->id, $course->id, 'student');
 994  
 995          // All users create a forum discussion.
 996          $forumgen = $generator->get_plugin_generator('mod_forum');
 997          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
 998              'userid' => $user1->id, 'name' => 'Post1', 'message' => 'plugh']);
 999          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1000                  'userid' => $user2->id, 'name' => 'Post2', 'message' => 'plugh']);
1001          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1002                  'userid' => $user3->id, 'name' => 'Post3', 'message' => 'plugh']);
1003  
1004          // Two of the users create entries in the glossary.
1005          $glossarygen = $generator->get_plugin_generator('mod_glossary');
1006          $glossarygen->create_content($glossary, ['concept' => 'Entry1', 'definition' => 'plugh',
1007                  'userid' => $user1->id]);
1008          $glossarygen->create_content($glossary, ['concept' => 'Entry3', 'definition' => 'plugh',
1009                  'userid' => $user3->id]);
1010  
1011          // Index the data.
1012          $this->search->index();
1013  
1014          // Search without user restriction should find everything.
1015          $querydata = new \stdClass();
1016          $querydata->q = 'plugh';
1017          $results = $this->search->search($querydata);
1018          $this->assert_result_titles(
1019                  ['Entry1', 'Entry3', 'Post1', 'Post2', 'Post3'], $results);
1020  
1021          // Restriction to user 3 only.
1022          $querydata->userids = [$user3->id];
1023          $results = $this->search->search($querydata);
1024          $this->assert_result_titles(
1025                  ['Entry3', 'Post3'], $results);
1026  
1027          // Restriction to users 1 and 2.
1028          $querydata->userids = [$user1->id, $user2->id];
1029          $results = $this->search->search($querydata);
1030          $this->assert_result_titles(
1031                  ['Entry1', 'Post1', 'Post2'], $results);
1032  
1033          // Restriction to users 1 and 2 combined with context restriction.
1034          $querydata->contextids = [\context_module::instance($glossary->cmid)->id];
1035          $results = $this->search->search($querydata);
1036          $this->assert_result_titles(
1037                  ['Entry1'], $results);
1038  
1039          // Restriction to users 1 and 2 combined with area restriction.
1040          unset($querydata->contextids);
1041          $querydata->areaids = [\core_search\manager::generate_areaid('mod_forum', 'post')];
1042          $results = $this->search->search($querydata);
1043          $this->assert_result_titles(
1044                  ['Post1', 'Post2'], $results);
1045      }
1046  
1047      /**
1048       * Tests searching for results containing words in italic text. (This used to fail.)
1049       */
1050      public function test_italics() {
1051          global $USER;
1052  
1053          // Use real search areas.
1054          $this->search->clear_static();
1055          $this->search->add_core_search_areas();
1056  
1057          // Create a course and a forum.
1058          $generator = $this->getDataGenerator();
1059          $course = $generator->create_course();
1060          $forum = $generator->create_module('forum', ['course' => $course->id]);
1061  
1062          // As admin user, create forum discussions with various words in italics or with underlines.
1063          $this->setAdminUser();
1064          $forumgen = $generator->get_plugin_generator('mod_forum');
1065          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1066                  'userid' => $USER->id, 'name' => 'Post1',
1067                  'message' => '<p>This is a post about <i>frogs</i>.</p>']);
1068          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1069                  'userid' => $USER->id, 'name' => 'Post2',
1070                  'message' => '<p>This is a post about <i>toads and zombies</i>.</p>']);
1071          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1072                  'userid' => $USER->id, 'name' => 'Post3',
1073                  'message' => '<p>This is a post about toads_and_zombies.</p>']);
1074          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
1075                  'userid' => $USER->id, 'name' => 'Post4',
1076                  'message' => '<p>This is a post about _leading and trailing_ underlines.</p>']);
1077  
1078          // Index the data.
1079          $this->search->index();
1080  
1081          // Search for 'frogs' should find the post.
1082          $querydata = new \stdClass();
1083          $querydata->q = 'frogs';
1084          $results = $this->search->search($querydata);
1085          $this->assert_result_titles(['Post1'], $results);
1086  
1087          // Search for 'toads' or 'zombies' should find post 2 (and not 3)...
1088          $querydata->q = 'toads';
1089          $results = $this->search->search($querydata);
1090          $this->assert_result_titles(['Post2'], $results);
1091          $querydata->q = 'zombies';
1092          $results = $this->search->search($querydata);
1093          $this->assert_result_titles(['Post2'], $results);
1094  
1095          // Search for 'toads_and_zombies' should find post 3.
1096          $querydata->q = 'toads_and_zombies';
1097          $results = $this->search->search($querydata);
1098          $this->assert_result_titles(['Post3'], $results);
1099  
1100          // Search for '_leading' or 'trailing_' should find post 4.
1101          $querydata->q = '_leading';
1102          $results = $this->search->search($querydata);
1103          $this->assert_result_titles(['Post4'], $results);
1104          $querydata->q = 'trailing_';
1105          $results = $this->search->search($querydata);
1106          $this->assert_result_titles(['Post4'], $results);
1107      }
1108  
1109      /**
1110       * Asserts that the returned documents have the expected titles (regardless of order).
1111       *
1112       * @param string[] $expected List of expected document titles
1113       * @param \core_search\document[] $results List of returned documents
1114       */
1115      protected function assert_result_titles(array $expected, array $results) {
1116          $titles = [];
1117          foreach ($results as $result) {
1118              $titles[] = $result->get('title');
1119          }
1120          sort($titles);
1121          sort($expected);
1122          $this->assertEquals($expected, $titles);
1123      }
1124  
1125      /**
1126       * Tests the get_supported_orders function for contexts where we can only use relevance
1127       * (system, category).
1128       */
1129      public function test_get_supported_orders_relevance_only() {
1130          global $DB;
1131  
1132          // System or category context: relevance only.
1133          $orders = $this->engine->get_supported_orders(\context_system::instance());
1134          $this->assertCount(1, $orders);
1135          $this->assertArrayHasKey('relevance', $orders);
1136  
1137          $categoryid = $DB->get_field_sql('SELECT MIN(id) FROM {course_categories}');
1138          $orders = $this->engine->get_supported_orders(\context_coursecat::instance($categoryid));
1139          $this->assertCount(1, $orders);
1140          $this->assertArrayHasKey('relevance', $orders);
1141      }
1142  
1143      /**
1144       * Tests the get_supported_orders function for contexts where we support location as well
1145       * (course, activity, block).
1146       */
1147      public function test_get_supported_orders_relevance_and_location() {
1148          global $DB;
1149  
1150          // Test with course context.
1151          $generator = $this->getDataGenerator();
1152          $course = $generator->create_course(['fullname' => 'Frogs']);
1153          $coursecontext = \context_course::instance($course->id);
1154  
1155          $orders = $this->engine->get_supported_orders($coursecontext);
1156          $this->assertCount(2, $orders);
1157          $this->assertArrayHasKey('relevance', $orders);
1158          $this->assertArrayHasKey('location', $orders);
1159          $this->assertStringContainsString('Course: Frogs', $orders['location']);
1160  
1161          // Test with activity context.
1162          $page = $generator->create_module('page', ['course' => $course->id, 'name' => 'Toads']);
1163  
1164          $orders = $this->engine->get_supported_orders(\context_module::instance($page->cmid));
1165          $this->assertCount(2, $orders);
1166          $this->assertArrayHasKey('relevance', $orders);
1167          $this->assertArrayHasKey('location', $orders);
1168          $this->assertStringContainsString('Page: Toads', $orders['location']);
1169  
1170          // Test with block context.
1171          $instance = (object)['blockname' => 'html', 'parentcontextid' => $coursecontext->id,
1172                  'showinsubcontexts' => 0, 'pagetypepattern' => 'course-view-*',
1173                  'defaultweight' => 0, 'timecreated' => 1, 'timemodified' => 1,
1174                  'configdata' => ''];
1175          $blockid = $DB->insert_record('block_instances', $instance);
1176          $blockcontext = \context_block::instance($blockid);
1177  
1178          $orders = $this->engine->get_supported_orders($blockcontext);
1179          $this->assertCount(2, $orders);
1180          $this->assertArrayHasKey('relevance', $orders);
1181          $this->assertArrayHasKey('location', $orders);
1182          $this->assertStringContainsString('Block: Text', $orders['location']);
1183      }
1184  
1185      /**
1186       * Tests ordering by relevance vs location.
1187       */
1188      public function test_ordering() {
1189          // Create 2 courses and 2 activities.
1190          $generator = $this->getDataGenerator();
1191          $course1 = $generator->create_course(['fullname' => 'Course 1']);
1192          $course1context = \context_course::instance($course1->id);
1193          $course1page = $generator->create_module('page', ['course' => $course1]);
1194          $course1pagecontext = \context_module::instance($course1page->cmid);
1195          $course2 = $generator->create_course(['fullname' => 'Course 2']);
1196          $course2context = \context_course::instance($course2->id);
1197          $course2page = $generator->create_module('page', ['course' => $course2]);
1198          $course2pagecontext = \context_module::instance($course2page->cmid);
1199  
1200          // Create one search record in each activity and course.
1201          $this->create_search_record($course1->id, $course1context->id, 'C1', 'Xyzzy');
1202          $this->create_search_record($course1->id, $course1pagecontext->id, 'C1P', 'Xyzzy');
1203          $this->create_search_record($course2->id, $course2context->id, 'C2', 'Xyzzy');
1204          $this->create_search_record($course2->id, $course2pagecontext->id, 'C2P', 'Xyzzy plugh');
1205          $this->search->index();
1206  
1207          // Default search works by relevance so the one with both words should be top.
1208          $querydata = new \stdClass();
1209          $querydata->q = 'xyzzy plugh';
1210          $results = $this->search->search($querydata);
1211          $this->assertCount(4, $results);
1212          $this->assertEquals('C2P', $results[0]->get('title'));
1213  
1214          // Same if you explicitly specify relevance.
1215          $querydata->order = 'relevance';
1216          $results = $this->search->search($querydata);
1217          $this->assertEquals('C2P', $results[0]->get('title'));
1218  
1219          // If you specify order by location and you are in C2 or C2P then results are the same.
1220          $querydata->order = 'location';
1221          $querydata->context = $course2context;
1222          $results = $this->search->search($querydata);
1223          $this->assertEquals('C2P', $results[0]->get('title'));
1224          $querydata->context = $course2pagecontext;
1225          $results = $this->search->search($querydata);
1226          $this->assertEquals('C2P', $results[0]->get('title'));
1227  
1228          // But if you are in C1P then you get different results (C1P first).
1229          $querydata->context = $course1pagecontext;
1230          $results = $this->search->search($querydata);
1231          $this->assertEquals('C1P', $results[0]->get('title'));
1232      }
1233  
1234      /**
1235       * Tests with bogus content (that can be entered into Moodle) to see if it crashes.
1236       */
1237      public function test_bogus_content() {
1238          $generator = $this->getDataGenerator();
1239          $course1 = $generator->create_course(['fullname' => 'Course 1']);
1240          $course1context = \context_course::instance($course1->id);
1241  
1242          // It is possible to enter into a Moodle database content containing these characters,
1243          // which are Unicode non-characters / byte order marks. If sent to Solr, these cause
1244          // failures.
1245          $boguscontent = html_entity_decode('&#xfffe;', ENT_COMPAT) . 'frog';
1246          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1247          $boguscontent = html_entity_decode('&#xffff;', ENT_COMPAT) . 'frog';
1248          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1249  
1250          // Unicode Standard Version 9.0 - Core Specification, section 23.7, lists 66 non-characters
1251          // in total. Here are some of them - these work OK for me but it may depend on platform.
1252          $boguscontent = html_entity_decode('&#xfdd0;', ENT_COMPAT) . 'frog';
1253          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1254          $boguscontent = html_entity_decode('&#xfdef;', ENT_COMPAT) . 'frog';
1255          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1256          $boguscontent = html_entity_decode('&#x1fffe;', ENT_COMPAT) . 'frog';
1257          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1258          $boguscontent = html_entity_decode('&#x10ffff;', ENT_COMPAT) . 'frog';
1259          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
1260  
1261          // Do the indexing (this will check it doesn't throw warnings).
1262          $this->search->index();
1263  
1264          // Confirm that all 6 documents are found in search.
1265          $querydata = new \stdClass();
1266          $querydata->q = 'frog';
1267          $results = $this->search->search($querydata);
1268          $this->assertCount(6, $results);
1269      }
1270  
1271      /**
1272       * Adds a record to the mock search area, so that the search engine can find it later.
1273       *
1274       * @param int $courseid Course id
1275       * @param int $contextid Context id
1276       * @param string $title Title for search index
1277       * @param string $content Content for search index
1278       */
1279      protected function create_search_record($courseid, $contextid, $title, $content) {
1280          $record = new \stdClass();
1281          $record->content = $content;
1282          $record->title = $title;
1283          $record->courseid = $courseid;
1284          $record->contextid = $contextid;
1285          $this->generator->create_record($record);
1286      }
1287  
1288      /**
1289       * Tries out deleting data for a context or a course.
1290       *
1291       * @throws coding_exception
1292       * @throws moodle_exception
1293       */
1294      public function test_deleted_contexts_and_courses() {
1295          // Create some courses and activities.
1296          $generator = $this->getDataGenerator();
1297          $course1 = $generator->create_course(['fullname' => 'Course 1']);
1298          $course1context = \context_course::instance($course1->id);
1299          $course1page1 = $generator->create_module('page', ['course' => $course1]);
1300          $course1page1context = \context_module::instance($course1page1->cmid);
1301          $course1page2 = $generator->create_module('page', ['course' => $course1]);
1302          $course1page2context = \context_module::instance($course1page2->cmid);
1303          $course2 = $generator->create_course(['fullname' => 'Course 2']);
1304          $course2context = \context_course::instance($course2->id);
1305          $course2page = $generator->create_module('page', ['course' => $course2]);
1306          $course2pagecontext = \context_module::instance($course2page->cmid);
1307  
1308          // Create one search record in each activity and course.
1309          $this->create_search_record($course1->id, $course1context->id, 'C1', 'Xyzzy');
1310          $this->create_search_record($course1->id, $course1page1context->id, 'C1P1', 'Xyzzy');
1311          $this->create_search_record($course1->id, $course1page2context->id, 'C1P2', 'Xyzzy');
1312          $this->create_search_record($course2->id, $course2context->id, 'C2', 'Xyzzy');
1313          $this->create_search_record($course2->id, $course2pagecontext->id, 'C2P', 'Xyzzy plugh');
1314          $this->search->index();
1315  
1316          // By default we have all results.
1317          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2', 'C2', 'C2P']);
1318  
1319          // Say we delete the course2pagecontext...
1320          $this->engine->delete_index_for_context($course2pagecontext->id);
1321          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2', 'C2']);
1322  
1323          // Now delete the second course...
1324          $this->engine->delete_index_for_course($course2->id);
1325          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2']);
1326  
1327          // Finally let's delete using Moodle functions to check that works. Single context first.
1328          course_delete_module($course1page1->cmid);
1329          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P2']);
1330          delete_course($course1, false);
1331          $this->assert_raw_solr_query_result('content:xyzzy', []);
1332      }
1333  
1334      /**
1335       * Specific test of the add_document_batch function (also used in many other tests).
1336       */
1337      public function test_add_document_batch() {
1338          // Get a default document.
1339          $area = new \core_mocksearch\search\mock_search_area();
1340          $record = $this->generator->create_record();
1341          $doc = $area->get_document($record);
1342          $originalid = $doc->get('id');
1343  
1344          // Now create 5 similar documents.
1345          $docs = [];
1346          for ($i = 1; $i <= 5; $i++) {
1347              $doc = $area->get_document($record);
1348              $doc->set('id', $originalid . '-' . $i);
1349              $doc->set('title', 'Batch ' . $i);
1350              $docs[$i] = $doc;
1351          }
1352  
1353          // Document 3 has a file attached.
1354          $fs = get_file_storage();
1355          $filerecord = new \stdClass();
1356          $filerecord->content = 'Some FileContents';
1357          $file = $this->generator->create_file($filerecord);
1358          $docs[3]->add_stored_file($file);
1359  
1360          // Add all these documents to the search engine.
1361          $this->assertEquals([5, 0, 1], $this->engine->add_document_batch($docs, true));
1362          $this->engine->area_index_complete($area->get_area_id());
1363  
1364          // Check all documents were indexed.
1365          $querydata = new \stdClass();
1366          $querydata->q = 'Batch';
1367          $results = $this->search->search($querydata);
1368          $this->assertCount(5, $results);
1369  
1370          // Check it also finds based on the file.
1371          $querydata->q = 'FileContents';
1372          $results = $this->search->search($querydata);
1373          $this->assertCount(1, $results);
1374      }
1375  
1376      /**
1377       * Tests the batching logic, specifically the limit to 100 documents per
1378       * batch, and not batching very large documents.
1379       */
1380      public function test_batching() {
1381          $area = new \core_mocksearch\search\mock_search_area();
1382          $record = $this->generator->create_record();
1383          $doc = $area->get_document($record);
1384          $originalid = $doc->get('id');
1385  
1386          // Up to 100 documents in 1 batch.
1387          $docs = [];
1388          for ($i = 1; $i <= 100; $i++) {
1389              $doc = $area->get_document($record);
1390              $doc->set('id', $originalid . '-' . $i);
1391              $docs[$i] = $doc;
1392          }
1393          [, , , , , $batches] = $this->engine->add_documents(
1394                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
1395          $this->assertEquals(1, $batches);
1396  
1397          // More than 100 needs 2 batches.
1398          $docs = [];
1399          for ($i = 1; $i <= 101; $i++) {
1400              $doc = $area->get_document($record);
1401              $doc->set('id', $originalid . '-' . $i);
1402              $docs[$i] = $doc;
1403          }
1404          [, , , , , $batches] = $this->engine->add_documents(
1405                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
1406          $this->assertEquals(2, $batches);
1407  
1408          // Small number but with some large documents that aren't batched.
1409          $docs = [];
1410          for ($i = 1; $i <= 10; $i++) {
1411              $doc = $area->get_document($record);
1412              $doc->set('id', $originalid . '-' . $i);
1413              $docs[$i] = $doc;
1414          }
1415          // This one is just small enough to fit.
1416          $docs[3]->set('content', str_pad('xyzzy ', 1024 * 1024, 'x'));
1417          // These two don't fit.
1418          $docs[5]->set('content', str_pad('xyzzy ', 1024 * 1024 + 1, 'x'));
1419          $docs[6]->set('content', str_pad('xyzzy ', 1024 * 1024 + 1, 'x'));
1420          [, , , , , $batches] = $this->engine->add_documents(
1421                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
1422          $this->assertEquals(3, $batches);
1423  
1424          // Check that all 3 of the large documents (added as batch or not) show up in results.
1425          $this->engine->area_index_complete($area->get_area_id());
1426          $querydata = new \stdClass();
1427          $querydata->q = 'xyzzy';
1428          $results = $this->search->search($querydata);
1429          $this->assertCount(3, $results);
1430      }
1431  
1432      /**
1433       * Tests with large documents. The point of this test is that we stop batching
1434       * documents if they are bigger than 1MB, and the maximum batch count is 100,
1435       * so the maximum size batch will be about 100 1MB documents.
1436       */
1437      public function test_add_document_batch_large() {
1438          // This test is a bit slow and not that important to run every time...
1439          if (!PHPUNIT_LONGTEST) {
1440              $this->markTestSkipped('PHPUNIT_LONGTEST is not defined');
1441          }
1442  
1443          // Get a default document.
1444          $area = new \core_mocksearch\search\mock_search_area();
1445          $record = $this->generator->create_record();
1446          $doc = $area->get_document($record);
1447          $originalid = $doc->get('id');
1448  
1449          // Now create 100 large documents.
1450          $size = 1024 * 1024;
1451          $docs = [];
1452          for ($i = 1; $i <= 100; $i++) {
1453              $doc = $area->get_document($record);
1454              $doc->set('id', $originalid . '-' . $i);
1455              $doc->set('title', 'Batch ' . $i);
1456              $doc->set('content', str_pad('', $size, 'Long text ' . $i . '. ', STR_PAD_RIGHT) . ' xyzzy');
1457              $docs[$i] = $doc;
1458          }
1459  
1460          // Add all these documents to the search engine.
1461          $this->engine->add_document_batch($docs, true);
1462          $this->engine->area_index_complete($area->get_area_id());
1463  
1464          // Check all documents were indexed, searching for text at end.
1465          $querydata = new \stdClass();
1466          $querydata->q = 'xyzzy';
1467          $results = $this->search->search($querydata);
1468          $this->assertCount(100, $results);
1469  
1470          // Search for specific text that's only in one.
1471          $querydata->q = '42';
1472          $results = $this->search->search($querydata);
1473          $this->assertCount(1, $results);
1474      }
1475  
1476      /**
1477       * Carries out a raw Solr query using the Solr basic query syntax.
1478       *
1479       * This is used to test data contained in the index without going through Moodle processing.
1480       *
1481       * @param string $q Search query
1482       * @param string[] $expected Expected titles of results, in alphabetical order
1483       */
1484      protected function assert_raw_solr_query_result(string $q, array $expected) {
1485          $solr = $this->engine->get_search_client_public();
1486          $query = new \SolrQuery($q);
1487          $results = $solr->query($query)->getResponse()->response->docs;
1488          if ($results) {
1489              $titles = array_map(function($x) {
1490                  return $x->title;
1491              }, $results);
1492              sort($titles);
1493          } else {
1494              $titles = [];
1495          }
1496          $this->assertEquals($expected, $titles);
1497      }
1498  }