Search moodle.org's
Developer Documentation

  • Bug fixes for general core bugs in 3.11.x will end 14 Nov 2022 (12 months plus 6 months extension).
  • Bug fixes for security issues in 3.11.x will end 13 Nov 2023 (18 months plus 12 months extension).
  • PHP version: minimum PHP 7.3.0 Note: minimum PHP version has increased since Moodle 3.10. PHP 7.4.x is supported too.
  • Differences Between: [Versions 310 and 311] [Versions 311 and 400] [Versions 37 and 311] [Versions 38 and 311] [Versions 39 and 311]

       1  <?php
       2  // This file is part of Moodle - http://moodle.org/
       3  //
       4  // Moodle is free software: you can redistribute it and/or modify
       5  // it under the terms of the GNU General Public License as published by
       6  // the Free Software Foundation, either version 3 of the License, or
       7  // (at your option) any later version.
       8  //
       9  // Moodle is distributed in the hope that it will be useful,
      10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
      11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12  // GNU General Public License for more details.
      13  //
      14  // You should have received a copy of the GNU General Public License
      15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
      16  
      17  namespace search_solr;
      18  
      19  defined('MOODLE_INTERNAL') || die();
      20  
      21  global $CFG;
      22  require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
      23  require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php');
      24  require_once($CFG->dirroot . '/search/engine/solr/tests/fixtures/testable_engine.php');
      25  
      26  /**
      27   * Solr search engine base unit tests.
      28   *
      29   * Required params:
      30   * - define('TEST_SEARCH_SOLR_HOSTNAME', '127.0.0.1');
      31   * - define('TEST_SEARCH_SOLR_PORT', '8983');
      32   * - define('TEST_SEARCH_SOLR_INDEXNAME', 'unittest');
      33   *
      34   * Optional params:
      35   * - define('TEST_SEARCH_SOLR_USERNAME', '');
      36   * - define('TEST_SEARCH_SOLR_PASSWORD', '');
      37   * - define('TEST_SEARCH_SOLR_SSLCERT', '');
      38   * - define('TEST_SEARCH_SOLR_SSLKEY', '');
      39   * - define('TEST_SEARCH_SOLR_KEYPASSWORD', '');
      40   * - define('TEST_SEARCH_SOLR_CAINFOCERT', '');
      41   *
      42   * @package     search_solr
      43   * @category    test
      44   * @copyright   2015 David Monllao {@link http://www.davidmonllao.com}
      45   * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
      46   *
      47   * @runTestsInSeparateProcesses
      48   */
      49  class engine_test extends \advanced_testcase {
      50  
      51      /**
      52       * @var \core_search\manager
      53       */
      54      protected $search = null;
      55  
      56      /**
      57       * @var Instace of core_search_generator.
      58       */
      59      protected $generator = null;
      60  
      61      /**
      62       * @var Instace of testable_engine.
      63       */
      64      protected $engine = null;
      65  
      66      public function setUp(): void {
      67          $this->resetAfterTest();
      68          set_config('enableglobalsearch', true);
      69          set_config('searchengine', 'solr');
      70  
      71          if (!function_exists('solr_get_version')) {
      72              $this->markTestSkipped('Solr extension is not loaded.');
      73          }
      74  
      75          if (!defined('TEST_SEARCH_SOLR_HOSTNAME') || !defined('TEST_SEARCH_SOLR_INDEXNAME') ||
      76                  !defined('TEST_SEARCH_SOLR_PORT')) {
      77              $this->markTestSkipped('Solr extension test server not set.');
      78          }
      79  
      80          set_config('server_hostname', TEST_SEARCH_SOLR_HOSTNAME, 'search_solr');
      81          set_config('server_port', TEST_SEARCH_SOLR_PORT, 'search_solr');
      82          set_config('indexname', TEST_SEARCH_SOLR_INDEXNAME, 'search_solr');
      83  
      84          if (defined('TEST_SEARCH_SOLR_USERNAME')) {
      85              set_config('server_username', TEST_SEARCH_SOLR_USERNAME, 'search_solr');
      86          }
      87  
      88          if (defined('TEST_SEARCH_SOLR_PASSWORD')) {
      89              set_config('server_password', TEST_SEARCH_SOLR_PASSWORD, 'search_solr');
      90          }
      91  
      92          if (defined('TEST_SEARCH_SOLR_SSLCERT')) {
      93              set_config('secure', true, 'search_solr');
      94              set_config('ssl_cert', TEST_SEARCH_SOLR_SSLCERT, 'search_solr');
      95          }
      96  
      97          if (defined('TEST_SEARCH_SOLR_SSLKEY')) {
      98              set_config('ssl_key', TEST_SEARCH_SOLR_SSLKEY, 'search_solr');
      99          }
     100  
     101          if (defined('TEST_SEARCH_SOLR_KEYPASSWORD')) {
     102              set_config('ssl_keypassword', TEST_SEARCH_SOLR_KEYPASSWORD, 'search_solr');
     103          }
     104  
     105          if (defined('TEST_SEARCH_SOLR_CAINFOCERT')) {
     106              set_config('ssl_cainfo', TEST_SEARCH_SOLR_CAINFOCERT, 'search_solr');
     107          }
     108  
     109          set_config('fileindexing', 1, 'search_solr');
     110  
     111          // We are only test indexing small string files, so setting this as low as we can.
     112          set_config('maxindexfilekb', 1, 'search_solr');
     113  
     114          $this->generator = self::getDataGenerator()->get_plugin_generator('core_search');
     115          $this->generator->setup();
     116  
     117          // Inject search solr engine into the testable core search as we need to add the mock
     118          // search component to it.
     119          $this->engine = new \search_solr\testable_engine();
     120          $this->search = \testable_core_search::instance($this->engine);
     121          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     122          $this->search->add_search_area($areaid, new \core_mocksearch\search\mock_search_area());
     123  
     124          $this->setAdminUser();
     125  
     126          // Cleanup before doing anything on it as the index it is out of this test control.
     127          $this->search->delete_index();
     128  
     129          // Add moodle fields if they don't exist.
     130          $schema = new \search_solr\schema($this->engine);
     131          $schema->setup(false);
     132      }
     133  
     134      public function tearDown(): void {
     135          // For unit tests before PHP 7, teardown is called even on skip. So only do our teardown if we did setup.
     136          if ($this->generator) {
     137              // Moodle DML freaks out if we don't teardown the temp table after each run.
     138              $this->generator->teardown();
     139              $this->generator = null;
     140          }
     141      }
     142  
     143      /**
     144       * Simple data provider to allow tests to be run with file indexing on and off.
     145       */
     146      public function file_indexing_provider() {
     147          return array(
     148              'file-indexing-on' => array(1),
     149              'file-indexing-off' => array(0)
     150          );
     151      }
     152  
     153      public function test_connection() {
     154          $this->assertTrue($this->engine->is_server_ready());
     155      }
     156  
     157      /**
     158       * Tests that the alternate settings are used when configured.
     159       */
     160      public function test_alternate_settings() {
     161          // Index a couple of things.
     162          $this->generator->create_record();
     163          $this->generator->create_record();
     164          $this->search->index();
     165  
     166          // By default settings, alternates are not set.
     167          $this->assertFalse($this->engine->has_alternate_configuration());
     168  
     169          // Set up all the config the same as normal.
     170          foreach (['server_hostname', 'indexname', 'secure', 'server_port',
     171                  'server_username', 'server_password'] as $setting) {
     172              set_config('alternate' . $setting, get_config('search_solr', $setting), 'search_solr');
     173          }
     174          // Also mess up the normal config.
     175          set_config('indexname', 'not_the_right_index_name', 'search_solr');
     176  
     177          // Construct a new engine using normal settings.
     178          $engine = new engine();
     179  
     180          // Now alternates are available.
     181          $this->assertTrue($engine->has_alternate_configuration());
     182  
     183          // But it won't actually work because of the bogus index name.
     184          $this->assertFalse($engine->is_server_ready() === true);
     185          $this->assertDebuggingCalled();
     186  
     187          // But if we construct one using alternate settings, it will work as normal.
     188          $engine = new engine(true);
     189          $this->assertTrue($engine->is_server_ready());
     190  
     191          // Including finding the search results.
     192          $this->assertCount(2, $engine->execute_query(
     193                  (object)['q' => 'message'], (object)['everything' => true]));
     194      }
     195  
     196      /**
     197       * @dataProvider file_indexing_provider
     198       */
     199      public function test_index($fileindexing) {
     200          global $DB;
     201  
     202          $this->engine->test_set_config('fileindexing', $fileindexing);
     203  
     204          $record = new \stdClass();
     205          $record->timemodified = time() - 1;
     206          $this->generator->create_record($record);
     207  
     208          // Data gets into the search engine.
     209          $this->assertTrue($this->search->index());
     210  
     211          // Not anymore as everything was already added.
     212          sleep(1);
     213          $this->assertFalse($this->search->index());
     214  
     215          $this->generator->create_record();
     216  
     217          // Indexing again once there is new data.
     218          $this->assertTrue($this->search->index());
     219      }
     220  
     221      /**
     222       * Better keep this not very strict about which or how many results are returned as may depend on solr engine config.
     223       *
     224       * @dataProvider file_indexing_provider
     225       *
     226       * @return void
     227       */
     228      public function test_search($fileindexing) {
     229          global $USER, $DB;
     230  
     231          $this->engine->test_set_config('fileindexing', $fileindexing);
     232  
     233          $this->generator->create_record();
     234          $record = new \stdClass();
     235          $record->title = "Special title";
     236          $this->generator->create_record($record);
     237  
     238          $this->search->index();
     239  
     240          $querydata = new \stdClass();
     241          $querydata->q = 'message';
     242          $results = $this->search->search($querydata);
     243          $this->assertCount(2, $results);
     244  
     245          // Based on core_mocksearch\search\indexer.
     246          $this->assertEquals($USER->id, $results[0]->get('userid'));
     247          $this->assertEquals(\context_course::instance(SITEID)->id, $results[0]->get('contextid'));
     248  
     249          // Do a test to make sure we aren't searching non-query fields, like areaid.
     250          $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     251          $this->assertCount(0, $this->search->search($querydata));
     252          $querydata->q = 'message';
     253  
     254          sleep(1);
     255          $beforeadding = time();
     256          sleep(1);
     257          $this->generator->create_record();
     258          $this->search->index();
     259  
     260          // Timestart.
     261          $querydata->timestart = $beforeadding;
     262          $this->assertCount(1, $this->search->search($querydata));
     263  
     264          // Timeend.
     265          unset($querydata->timestart);
     266          $querydata->timeend = $beforeadding;
     267          $this->assertCount(2, $this->search->search($querydata));
     268  
     269          // Title.
     270          unset($querydata->timeend);
     271          $querydata->title = 'Special title';
     272          $this->assertCount(1, $this->search->search($querydata));
     273  
     274          // Course IDs.
     275          unset($querydata->title);
     276          $querydata->courseids = array(SITEID + 1);
     277          $this->assertCount(0, $this->search->search($querydata));
     278  
     279          $querydata->courseids = array(SITEID);
     280          $this->assertCount(3, $this->search->search($querydata));
     281  
     282          // Now try some area-id combinations.
     283          unset($querydata->courseids);
     284          $forumpostareaid = \core_search\manager::generate_areaid('mod_forum', 'post');
     285          $mockareaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     286  
     287          $querydata->areaids = array($forumpostareaid);
     288          $this->assertCount(0, $this->search->search($querydata));
     289  
     290          $querydata->areaids = array($forumpostareaid, $mockareaid);
     291          $this->assertCount(3, $this->search->search($querydata));
     292  
     293          $querydata->areaids = array($mockareaid);
     294          $this->assertCount(3, $this->search->search($querydata));
     295  
     296          $querydata->areaids = array();
     297          $this->assertCount(3, $this->search->search($querydata));
     298  
     299          // Check that index contents get updated.
     300          $this->generator->delete_all();
     301          $this->search->index(true);
     302          unset($querydata->title);
     303          $querydata->q = '*';
     304          $this->assertCount(0, $this->search->search($querydata));
     305      }
     306  
     307      /**
     308       * @dataProvider file_indexing_provider
     309       */
     310      public function test_delete($fileindexing) {
     311          $this->engine->test_set_config('fileindexing', $fileindexing);
     312  
     313          $this->generator->create_record();
     314          $this->generator->create_record();
     315          $this->search->index();
     316  
     317          $querydata = new \stdClass();
     318          $querydata->q = 'message';
     319  
     320          $this->assertCount(2, $this->search->search($querydata));
     321  
     322          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     323          $this->search->delete_index($areaid);
     324          $this->assertCount(0, $this->search->search($querydata));
     325      }
     326  
     327      /**
     328       * @dataProvider file_indexing_provider
     329       */
     330      public function test_alloweduserid($fileindexing) {
     331          $this->engine->test_set_config('fileindexing', $fileindexing);
     332  
     333          $area = new \core_mocksearch\search\mock_search_area();
     334  
     335          $record = $this->generator->create_record();
     336  
     337          // Get the doc and insert the default doc.
     338          $doc = $area->get_document($record);
     339          $this->engine->add_document($doc);
     340  
     341          $users = array();
     342          $users[] = $this->getDataGenerator()->create_user();
     343          $users[] = $this->getDataGenerator()->create_user();
     344          $users[] = $this->getDataGenerator()->create_user();
     345  
     346          // Add a record that only user 100 can see.
     347          $originalid = $doc->get('id');
     348  
     349          // Now add a custom doc for each user.
     350          foreach ($users as $user) {
     351              $doc = $area->get_document($record);
     352              $doc->set('id', $originalid.'-'.$user->id);
     353              $doc->set('owneruserid', $user->id);
     354              $this->engine->add_document($doc);
     355          }
     356  
     357          $this->engine->area_index_complete($area->get_area_id());
     358  
     359          $querydata = new \stdClass();
     360          $querydata->q = 'message';
     361          $querydata->title = $doc->get('title');
     362  
     363          // We are going to go through each user and see if they get the original and the owned doc.
     364          foreach ($users as $user) {
     365              $this->setUser($user);
     366  
     367              $results = $this->search->search($querydata);
     368              $this->assertCount(2, $results);
     369  
     370              $owned = 0;
     371              $notowned = 0;
     372  
     373              // We don't know what order we will get the results in, so we are doing this.
     374              foreach ($results as $result) {
     375                  $owneruserid = $result->get('owneruserid');
     376                  if (empty($owneruserid)) {
     377                      $notowned++;
     378                      $this->assertEquals(0, $owneruserid);
     379                      $this->assertEquals($originalid, $result->get('id'));
     380                  } else {
     381                      $owned++;
     382                      $this->assertEquals($user->id, $owneruserid);
     383                      $this->assertEquals($originalid.'-'.$user->id, $result->get('id'));
     384                  }
     385              }
     386  
     387              $this->assertEquals(1, $owned);
     388              $this->assertEquals(1, $notowned);
     389          }
     390  
     391          // Now test a user with no owned results.
     392          $otheruser = $this->getDataGenerator()->create_user();
     393          $this->setUser($otheruser);
     394  
     395          $results = $this->search->search($querydata);
     396          $this->assertCount(1, $results);
     397  
     398          $this->assertEquals(0, $results[0]->get('owneruserid'));
     399          $this->assertEquals($originalid, $results[0]->get('id'));
     400      }
     401  
     402      /**
     403       * @dataProvider file_indexing_provider
     404       */
     405      public function test_highlight($fileindexing) {
     406          global $PAGE;
     407  
     408          $this->engine->test_set_config('fileindexing', $fileindexing);
     409  
     410          $this->generator->create_record();
     411          $this->search->index();
     412  
     413          $querydata = new \stdClass();
     414          $querydata->q = 'message';
     415  
     416          $results = $this->search->search($querydata);
     417          $this->assertCount(1, $results);
     418  
     419          $result = reset($results);
     420  
     421          $regex = '|'.\search_solr\engine::HIGHLIGHT_START.'message'.\search_solr\engine::HIGHLIGHT_END.'|';
     422          $this->assertMatchesRegularExpression($regex, $result->get('content'));
     423  
     424          $searchrenderer = $PAGE->get_renderer('core_search');
     425          $exported = $result->export_for_template($searchrenderer);
     426  
     427          $regex = '|<span class="highlight">message</span>|';
     428          $this->assertMatchesRegularExpression($regex, $exported['content']);
     429      }
     430  
     431      public function test_export_file_for_engine() {
     432          // Get area to work with.
     433          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     434          $area = \core_search\manager::get_search_area($areaid);
     435  
     436          $record = $this->generator->create_record();
     437  
     438          $doc = $area->get_document($record);
     439          $filerecord = new \stdClass();
     440          $filerecord->timemodified  = 978310800;
     441          $file = $this->generator->create_file($filerecord);
     442          $doc->add_stored_file($file);
     443  
     444          $filearray = $doc->export_file_for_engine($file);
     445  
     446          $this->assertEquals(\core_search\manager::TYPE_FILE, $filearray['type']);
     447          $this->assertEquals($file->get_id(), $filearray['solr_fileid']);
     448          $this->assertEquals($file->get_contenthash(), $filearray['solr_filecontenthash']);
     449          $this->assertEquals(\search_solr\document::INDEXED_FILE_TRUE, $filearray['solr_fileindexstatus']);
     450          $this->assertEquals($file->get_filename(), $filearray['title']);
     451          $this->assertEquals(978310800, \search_solr\document::import_time_from_engine($filearray['modified']));
     452      }
     453  
     454      public function test_index_file() {
     455          // Very simple test.
     456          $file = $this->generator->create_file();
     457  
     458          $record = new \stdClass();
     459          $record->attachfileids = array($file->get_id());
     460          $this->generator->create_record($record);
     461  
     462          $this->search->index();
     463          $querydata = new \stdClass();
     464          $querydata->q = '"File contents"';
     465  
     466          $this->assertCount(1, $this->search->search($querydata));
     467      }
     468  
     469      public function test_reindexing_files() {
     470          // Get area to work with.
     471          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     472          $area = \core_search\manager::get_search_area($areaid);
     473  
     474          $record = $this->generator->create_record();
     475  
     476          $doc = $area->get_document($record);
     477  
     478          // Now we are going to make some files.
     479          $fs = get_file_storage();
     480          $syscontext = \context_system::instance();
     481  
     482          $files = array();
     483  
     484          $filerecord = new \stdClass();
     485          // We make enough so that we pass the 500 files threashold. That is the boundary when getting files.
     486          $boundary = 500;
     487          $top = (int)($boundary * 1.1);
     488          for ($i = 0; $i < $top; $i++) {
     489              $filerecord->filename  = 'searchfile'.$i;
     490              $filerecord->content = 'Some FileContents'.$i;
     491              $file = $this->generator->create_file($filerecord);
     492              $doc->add_stored_file($file);
     493              $files[] = $file;
     494          }
     495  
     496          // Add the doc with lots of files, then commit.
     497          $this->engine->add_document($doc, true);
     498          $this->engine->area_index_complete($area->get_area_id());
     499  
     500          // Indexes we are going to check. 0 means we will delete, 1 means we will keep.
     501          $checkfiles = array(
     502              0 => 0,                        // Check the begining of the set.
     503              1 => 1,
     504              2 => 0,
     505              ($top - 3) => 0,               // Check the end of the set.
     506              ($top - 2) => 1,
     507              ($top - 1) => 0,
     508              ($boundary - 2) => 0,          // Check at the boundary between fetch groups.
     509              ($boundary - 1) => 0,
     510              $boundary => 0,
     511              ($boundary + 1) => 0,
     512              ((int)($boundary * 0.5)) => 1, // Make sure we keep some middle ones.
     513              ((int)($boundary * 1.05)) => 1
     514          );
     515  
     516          $querydata = new \stdClass();
     517  
     518          // First, check that all the files are currently there.
     519          foreach ($checkfiles as $key => $unused) {
     520              $querydata->q = 'FileContents'.$key;
     521              $this->assertCount(1, $this->search->search($querydata));
     522              $querydata->q = 'searchfile'.$key;
     523              $this->assertCount(1, $this->search->search($querydata));
     524          }
     525  
     526          // Remove the files we want removed from the files array.
     527          foreach ($checkfiles as $key => $keep) {
     528              if (!$keep) {
     529                  unset($files[$key]);
     530              }
     531          }
     532  
     533          // And make us a new file to add.
     534          $filerecord->filename  = 'searchfileNew';
     535          $filerecord->content  = 'Some FileContentsNew';
     536          $files[] = $this->generator->create_file($filerecord);
     537          $checkfiles['New'] = 1;
     538  
     539          $doc = $area->get_document($record);
     540          foreach($files as $file) {
     541              $doc->add_stored_file($file);
     542          }
     543  
     544          // Reindex the document with the changed files.
     545          $this->engine->add_document($doc, true);
     546          $this->engine->area_index_complete($area->get_area_id());
     547  
     548          // Go through our check array, and see if the file is there or not.
     549          foreach ($checkfiles as $key => $keep) {
     550              $querydata->q = 'FileContents'.$key;
     551              $this->assertCount($keep, $this->search->search($querydata));
     552              $querydata->q = 'searchfile'.$key;
     553              $this->assertCount($keep, $this->search->search($querydata));
     554          }
     555  
     556          // Now check that we get one result when we search from something in all of them.
     557          $querydata->q = 'Some';
     558          $this->assertCount(1, $this->search->search($querydata));
     559      }
     560  
     561      /**
     562       * Test indexing a file we don't consider indexable.
     563       */
     564      public function test_index_filtered_file() {
     565          // Get area to work with.
     566          $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'mock_search_area');
     567          $area = \core_search\manager::get_search_area($areaid);
     568  
     569          // Get a single record to make a doc from.
     570          $record = $this->generator->create_record();
     571  
     572          $doc = $area->get_document($record);
     573  
     574          // Now we are going to make some files.
     575          $fs = get_file_storage();
     576          $syscontext = \context_system::instance();
     577  
     578          // We need to make a file greater than 1kB in size, which is the lowest filter size.
     579          $filerecord = new \stdClass();
     580          $filerecord->filename = 'largefile';
     581          $filerecord->content = 'Some LargeFindContent to find.';
     582          for ($i = 0; $i < 200; $i++) {
     583              $filerecord->content .= ' The quick brown fox jumps over the lazy dog.';
     584          }
     585  
     586          $this->assertGreaterThan(1024, strlen($filerecord->content));
     587  
     588          $file = $this->generator->create_file($filerecord);
     589          $doc->add_stored_file($file);
     590  
     591          $filerecord->filename = 'smallfile';
     592          $filerecord->content = 'Some SmallFindContent to find.';
     593          $file = $this->generator->create_file($filerecord);
     594          $doc->add_stored_file($file);
     595  
     596          $this->engine->add_document($doc, true);
     597          $this->engine->area_index_complete($area->get_area_id());
     598  
     599          $querydata = new \stdClass();
     600          // We shouldn't be able to find the large file contents.
     601          $querydata->q = 'LargeFindContent';
     602          $this->assertCount(0, $this->search->search($querydata));
     603  
     604          // But we should be able to find the filename.
     605          $querydata->q = 'largefile';
     606          $this->assertCount(1, $this->search->search($querydata));
     607  
     608          // We should be able to find the small file contents.
     609          $querydata->q = 'SmallFindContent';
     610          $this->assertCount(1, $this->search->search($querydata));
     611  
     612          // And we should be able to find the filename.
     613          $querydata->q = 'smallfile';
     614          $this->assertCount(1, $this->search->search($querydata));
     615      }
     616  
     617      public function test_delete_by_id() {
     618          // First get files in the index.
     619          $file = $this->generator->create_file();
     620          $record = new \stdClass();
     621          $record->attachfileids = array($file->get_id());
     622          $this->generator->create_record($record);
     623          $this->generator->create_record($record);
     624          $this->search->index();
     625  
     626          $querydata = new \stdClass();
     627  
     628          // Then search to make sure they are there.
     629          $querydata->q = '"File contents"';
     630          $results = $this->search->search($querydata);
     631          $this->assertCount(2, $results);
     632  
     633          $first = reset($results);
     634          $deleteid = $first->get('id');
     635  
     636          $this->engine->delete_by_id($deleteid);
     637  
     638          // Check that we don't get a result for it anymore.
     639          $results = $this->search->search($querydata);
     640          $this->assertCount(1, $results);
     641          $result = reset($results);
     642          $this->assertNotEquals($deleteid, $result->get('id'));
     643      }
     644  
     645      /**
     646       * Test that expected results are returned, even with low check_access success rate.
     647       *
     648       * @dataProvider file_indexing_provider
     649       */
     650      public function test_solr_filling($fileindexing) {
     651          $this->engine->test_set_config('fileindexing', $fileindexing);
     652  
     653          $user1 = self::getDataGenerator()->create_user();
     654          $user2 = self::getDataGenerator()->create_user();
     655  
     656          // We are going to create a bunch of records that user 1 can see with 2 keywords.
     657          // Then we are going to create a bunch for user 2 with only 1 of the keywords.
     658          // If user 2 searches for both keywords, solr will return all of the user 1 results, then the user 2 results.
     659          // This is because the user 1 results will match 2 keywords, while the others will match only 1.
     660  
     661          $record = new \stdClass();
     662  
     663          // First create a bunch of records for user 1 to see.
     664          $record->denyuserids = array($user2->id);
     665          $record->content = 'Something1 Something2';
     666          $maxresults = (int)(\core_search\manager::MAX_RESULTS * .75);
     667          for ($i = 0; $i < $maxresults; $i++) {
     668              $this->generator->create_record($record);
     669          }
     670  
     671          // Then create a bunch of records for user 2 to see.
     672          $record->denyuserids = array($user1->id);
     673          $record->content = 'Something1';
     674          for ($i = 0; $i < $maxresults; $i++) {
     675              $this->generator->create_record($record);
     676          }
     677  
     678          $this->search->index();
     679  
     680          // Check that user 1 sees all their results.
     681          $this->setUser($user1);
     682          $querydata = new \stdClass();
     683          $querydata->q = 'Something1 Something2';
     684          $results = $this->search->search($querydata);
     685          $this->assertCount($maxresults, $results);
     686  
     687          // Check that user 2 will see theirs, even though they may be crouded out.
     688          $this->setUser($user2);
     689          $results = $this->search->search($querydata);
     690          $this->assertCount($maxresults, $results);
     691      }
     692  
     693      /**
     694       * Create 40 docs, that will be return from Solr in 10 hidden, 10 visible, 10 hidden, 10 visible if you query for:
     695       * Something1 Something2 Something3 Something4, with the specified user set.
     696       */
     697      protected function setup_user_hidden_docs($user) {
     698          // These results will come first, and will not be visible by the user.
     699          $record = new \stdClass();
     700          $record->denyuserids = array($user->id);
     701          $record->content = 'Something1 Something2 Something3 Something4';
     702          for ($i = 0; $i < 10; $i++) {
     703              $this->generator->create_record($record);
     704          }
     705  
     706          // These results will come second, and will  be visible by the user.
     707          unset($record->denyuserids);
     708          $record->content = 'Something1 Something2 Something3';
     709          for ($i = 0; $i < 10; $i++) {
     710              $this->generator->create_record($record);
     711          }
     712  
     713          // These results will come third, and will not be visible by the user.
     714          $record->denyuserids = array($user->id);
     715          $record->content = 'Something1 Something2';
     716          for ($i = 0; $i < 10; $i++) {
     717              $this->generator->create_record($record);
     718          }
     719  
     720          // These results will come fourth, and will be visible by the user.
     721          unset($record->denyuserids);
     722          $record->content = 'Something1 ';
     723          for ($i = 0; $i < 10; $i++) {
     724              $this->generator->create_record($record);
     725          }
     726      }
     727  
     728      /**
     729       * Test that counts are what we expect.
     730       *
     731       * @dataProvider file_indexing_provider
     732       */
     733      public function test_get_query_total_count($fileindexing) {
     734          $this->engine->test_set_config('fileindexing', $fileindexing);
     735  
     736          $user = self::getDataGenerator()->create_user();
     737          $this->setup_user_hidden_docs($user);
     738          $this->search->index();
     739  
     740          $this->setUser($user);
     741          $querydata = new \stdClass();
     742          $querydata->q = 'Something1 Something2 Something3 Something4';
     743  
     744          // In this first set, it should have determined the first 10 of 40 are bad, so there could be up to 30 left.
     745          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 5);
     746          $this->assertEquals(30, $this->engine->get_query_total_count());
     747          $this->assertCount(5, $results);
     748  
     749          // To get to 15, it has to process the first 10 that are bad, 10 that are good, 10 that are bad, then 5 that are good.
     750          // So we now know 20 are bad out of 40.
     751          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 15);
     752          $this->assertEquals(20, $this->engine->get_query_total_count());
     753          $this->assertCount(15, $results);
     754  
     755          // Try to get more then all, make sure we still see 20 count and 20 returned.
     756          $results = $this->engine->execute_query($querydata, (object)['everything' => true], 30);
     757          $this->assertEquals(20, $this->engine->get_query_total_count());
     758          $this->assertCount(20, $results);
     759      }
     760  
     761      /**
     762       * Test that paged results are what we expect.
     763       *
     764       * @dataProvider file_indexing_provider
     765       */
     766      public function test_manager_paged_search($fileindexing) {
     767          $this->engine->test_set_config('fileindexing', $fileindexing);
     768  
     769          $user = self::getDataGenerator()->create_user();
     770          $this->setup_user_hidden_docs($user);
     771          $this->search->index();
     772  
     773          // Check that user 1 sees all their results.
     774          $this->setUser($user);
     775          $querydata = new \stdClass();
     776          $querydata->q = 'Something1 Something2 Something3 Something4';
     777  
     778          // On this first page, it should have determined the first 10 of 40 are bad, so there could be up to 30 left.
     779          $results = $this->search->paged_search($querydata, 0);
     780          $this->assertEquals(30, $results->totalcount);
     781          $this->assertCount(10, $results->results);
     782          $this->assertEquals(0, $results->actualpage);
     783  
     784          // On the second page, it should have found the next 10 bad ones, so we no know there are only 20 total.
     785          $results = $this->search->paged_search($querydata, 1);
     786          $this->assertEquals(20, $results->totalcount);
     787          $this->assertCount(10, $results->results);
     788          $this->assertEquals(1, $results->actualpage);
     789  
     790          // Try to get an additional page - we should get back page 1 results, since that is the last page with valid results.
     791          $results = $this->search->paged_search($querydata, 2);
     792          $this->assertEquals(20, $results->totalcount);
     793          $this->assertCount(10, $results->results);
     794          $this->assertEquals(1, $results->actualpage);
     795      }
     796  
     797      /**
     798       * Tests searching for results restricted to context id.
     799       */
     800      public function test_context_restriction() {
     801          // Use real search areas.
     802          $this->search->clear_static();
     803          $this->search->add_core_search_areas();
     804  
     805          // Create 2 courses and some forums.
     806          $generator = $this->getDataGenerator();
     807          $course1 = $generator->create_course(['fullname' => 'Course 1', 'summary' => 'xyzzy']);
     808          $contextc1 = \context_course::instance($course1->id);
     809          $course1forum1 = $generator->create_module('forum', ['course' => $course1,
     810                  'name' => 'C1F1', 'intro' => 'xyzzy']);
     811          $contextc1f1 = \context_module::instance($course1forum1->cmid);
     812          $course1forum2 = $generator->create_module('forum', ['course' => $course1,
     813                  'name' => 'C1F2', 'intro' => 'xyzzy']);
     814          $contextc1f2 = \context_module::instance($course1forum2->cmid);
     815          $course2 = $generator->create_course(['fullname' => 'Course 2', 'summary' => 'xyzzy']);
     816          $contextc2 = \context_course::instance($course1->id);
     817          $course2forum = $generator->create_module('forum', ['course' => $course2,
     818                  'name' => 'C2F', 'intro' => 'xyzzy']);
     819          $contextc2f = \context_module::instance($course2forum->cmid);
     820  
     821          // Index the courses and forums.
     822          $this->search->index();
     823  
     824          // Search as admin user should find everything.
     825          $querydata = new \stdClass();
     826          $querydata->q = 'xyzzy';
     827          $results = $this->search->search($querydata);
     828          $this->assert_result_titles(
     829                  ['Course 1', 'Course 2', 'C1F1', 'C1F2', 'C2F'], $results);
     830  
     831          // Admin user manually restricts results by context id to include one course and one forum.
     832          $querydata->contextids = [$contextc2f->id, $contextc1->id];
     833          $results = $this->search->search($querydata);
     834          $this->assert_result_titles(['Course 1', 'C2F'], $results);
     835  
     836          // Student enrolled in only one course, same restriction, only has the available results.
     837          $student2 = $generator->create_user();
     838          $generator->enrol_user($student2->id, $course2->id, 'student');
     839          $this->setUser($student2);
     840          $results = $this->search->search($querydata);
     841          $this->assert_result_titles(['C2F'], $results);
     842  
     843          // Student enrolled in both courses, same restriction, same results as admin.
     844          $student1 = $generator->create_user();
     845          $generator->enrol_user($student1->id, $course1->id, 'student');
     846          $generator->enrol_user($student1->id, $course2->id, 'student');
     847          $this->setUser($student1);
     848          $results = $this->search->search($querydata);
     849          $this->assert_result_titles(['Course 1', 'C2F'], $results);
     850  
     851          // Restrict both course and context.
     852          $querydata->courseids = [$course2->id];
     853          $results = $this->search->search($querydata);
     854          $this->assert_result_titles(['C2F'], $results);
     855          unset($querydata->courseids);
     856  
     857          // Restrict both area and context.
     858          $querydata->areaids = ['core_course-course'];
     859          $results = $this->search->search($querydata);
     860          $this->assert_result_titles(['Course 1'], $results);
     861  
     862          // Restrict area and context, incompatibly - this has no results (and doesn't do a query).
     863          $querydata->contextids = [$contextc2f->id];
     864          $results = $this->search->search($querydata);
     865          $this->assert_result_titles([], $results);
     866      }
     867  
     868      /**
     869       * Tests searching for results in groups, either by specified group ids or based on user
     870       * access permissions.
     871       */
     872      public function test_groups() {
     873          global $USER;
     874  
     875          // Use real search areas.
     876          $this->search->clear_static();
     877          $this->search->add_core_search_areas();
     878  
     879          // Create 2 courses and a selection of forums with different group mode.
     880          $generator = $this->getDataGenerator();
     881          $course1 = $generator->create_course(['fullname' => 'Course 1']);
     882          $forum1nogroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => NOGROUPS]);
     883          $forum1separategroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => SEPARATEGROUPS]);
     884          $forum1visiblegroups = $generator->create_module('forum', ['course' => $course1, 'groupmode' => VISIBLEGROUPS]);
     885          $course2 = $generator->create_course(['fullname' => 'Course 2']);
     886          $forum2separategroups = $generator->create_module('forum', ['course' => $course2, 'groupmode' => SEPARATEGROUPS]);
     887  
     888          // Create two groups on each course.
     889          $group1a = $generator->create_group(['courseid' => $course1->id]);
     890          $group1b = $generator->create_group(['courseid' => $course1->id]);
     891          $group2a = $generator->create_group(['courseid' => $course2->id]);
     892          $group2b = $generator->create_group(['courseid' => $course2->id]);
     893  
     894          // Create search records in each activity and (where relevant) in each group.
     895          $forumgenerator = $generator->get_plugin_generator('mod_forum');
     896          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
     897                  'forum' => $forum1nogroups->id, 'name' => 'F1NG', 'message' => 'xyzzy']);
     898          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
     899                  'forum' => $forum1separategroups->id, 'name' => 'F1SG-A',  'message' => 'xyzzy',
     900                  'groupid' => $group1a->id]);
     901          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
     902                  'forum' => $forum1separategroups->id, 'name' => 'F1SG-B', 'message' => 'xyzzy',
     903                  'groupid' => $group1b->id]);
     904          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
     905                  'forum' => $forum1visiblegroups->id, 'name' => 'F1VG-A', 'message' => 'xyzzy',
     906                  'groupid' => $group1a->id]);
     907          $forumgenerator->create_discussion(['course' => $course1->id, 'userid' => $USER->id,
     908                  'forum' => $forum1visiblegroups->id, 'name' => 'F1VG-B', 'message' => 'xyzzy',
     909                  'groupid' => $group1b->id]);
     910          $forumgenerator->create_discussion(['course' => $course2->id, 'userid' => $USER->id,
     911                  'forum' => $forum2separategroups->id, 'name' => 'F2SG-A', 'message' => 'xyzzy',
     912                  'groupid' => $group2a->id]);
     913          $forumgenerator->create_discussion(['course' => $course2->id, 'userid' => $USER->id,
     914                  'forum' => $forum2separategroups->id, 'name' => 'F2SG-B', 'message' => 'xyzzy',
     915                  'groupid' => $group2b->id]);
     916  
     917          $this->search->index();
     918  
     919          // Search as admin user should find everything.
     920          $querydata = new \stdClass();
     921          $querydata->q = 'xyzzy';
     922          $results = $this->search->search($querydata);
     923          $this->assert_result_titles(
     924                  ['F1NG', 'F1SG-A', 'F1SG-B', 'F1VG-A', 'F1VG-B', 'F2SG-A', 'F2SG-B'], $results);
     925  
     926          // Admin user manually restricts results by groups.
     927          $querydata->groupids = [$group1b->id, $group2a->id];
     928          $results = $this->search->search($querydata);
     929          $this->assert_result_titles(['F1SG-B', 'F1VG-B', 'F2SG-A'], $results);
     930  
     931          // Student enrolled in both courses but no groups.
     932          $student1 = $generator->create_user();
     933          $generator->enrol_user($student1->id, $course1->id, 'student');
     934          $generator->enrol_user($student1->id, $course2->id, 'student');
     935          $this->setUser($student1);
     936  
     937          unset($querydata->groupids);
     938          $results = $this->search->search($querydata);
     939          $this->assert_result_titles(['F1NG', 'F1VG-A', 'F1VG-B'], $results);
     940  
     941          // Student enrolled in both courses and group A in both cases.
     942          $student2 = $generator->create_user();
     943          $generator->enrol_user($student2->id, $course1->id, 'student');
     944          $generator->enrol_user($student2->id, $course2->id, 'student');
     945          groups_add_member($group1a, $student2);
     946          groups_add_member($group2a, $student2);
     947          $this->setUser($student2);
     948  
     949          $results = $this->search->search($querydata);
     950          $this->assert_result_titles(['F1NG', 'F1SG-A', 'F1VG-A', 'F1VG-B', 'F2SG-A'], $results);
     951  
     952          // Manually restrict results to group B in course 1.
     953          $querydata->groupids = [$group1b->id];
     954          $results = $this->search->search($querydata);
     955          $this->assert_result_titles(['F1VG-B'], $results);
     956  
     957          // Manually restrict results to group A in course 1.
     958          $querydata->groupids = [$group1a->id];
     959          $results = $this->search->search($querydata);
     960          $this->assert_result_titles(['F1SG-A', 'F1VG-A'], $results);
     961  
     962          // Manager enrolled in both courses (has access all groups).
     963          $manager = $generator->create_user();
     964          $generator->enrol_user($manager->id, $course1->id, 'manager');
     965          $generator->enrol_user($manager->id, $course2->id, 'manager');
     966          $this->setUser($manager);
     967          unset($querydata->groupids);
     968          $results = $this->search->search($querydata);
     969          $this->assert_result_titles(
     970                  ['F1NG', 'F1SG-A', 'F1SG-B', 'F1VG-A', 'F1VG-B', 'F2SG-A', 'F2SG-B'], $results);
     971      }
     972  
     973      /**
     974       * Tests searching for results restricted to specific user id(s).
     975       */
     976      public function test_user_restriction() {
     977          // Use real search areas.
     978          $this->search->clear_static();
     979          $this->search->add_core_search_areas();
     980  
     981          // Create a course, a forum, and a glossary.
     982          $generator = $this->getDataGenerator();
     983          $course = $generator->create_course();
     984          $forum = $generator->create_module('forum', ['course' => $course->id]);
     985          $glossary = $generator->create_module('glossary', ['course' => $course->id]);
     986  
     987          // Create 3 user accounts, all enrolled as students on the course.
     988          $user1 = $generator->create_user();
     989          $user2 = $generator->create_user();
     990          $user3 = $generator->create_user();
     991          $generator->enrol_user($user1->id, $course->id, 'student');
     992          $generator->enrol_user($user2->id, $course->id, 'student');
     993          $generator->enrol_user($user3->id, $course->id, 'student');
     994  
     995          // All users create a forum discussion.
     996          $forumgen = $generator->get_plugin_generator('mod_forum');
     997          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
     998              'userid' => $user1->id, 'name' => 'Post1', 'message' => 'plugh']);
     999          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1000                  'userid' => $user2->id, 'name' => 'Post2', 'message' => 'plugh']);
    1001          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1002                  'userid' => $user3->id, 'name' => 'Post3', 'message' => 'plugh']);
    1003  
    1004          // Two of the users create entries in the glossary.
    1005          $glossarygen = $generator->get_plugin_generator('mod_glossary');
    1006          $glossarygen->create_content($glossary, ['concept' => 'Entry1', 'definition' => 'plugh',
    1007                  'userid' => $user1->id]);
    1008          $glossarygen->create_content($glossary, ['concept' => 'Entry3', 'definition' => 'plugh',
    1009                  'userid' => $user3->id]);
    1010  
    1011          // Index the data.
    1012          $this->search->index();
    1013  
    1014          // Search without user restriction should find everything.
    1015          $querydata = new \stdClass();
    1016          $querydata->q = 'plugh';
    1017          $results = $this->search->search($querydata);
    1018          $this->assert_result_titles(
    1019                  ['Entry1', 'Entry3', 'Post1', 'Post2', 'Post3'], $results);
    1020  
    1021          // Restriction to user 3 only.
    1022          $querydata->userids = [$user3->id];
    1023          $results = $this->search->search($querydata);
    1024          $this->assert_result_titles(
    1025                  ['Entry3', 'Post3'], $results);
    1026  
    1027          // Restriction to users 1 and 2.
    1028          $querydata->userids = [$user1->id, $user2->id];
    1029          $results = $this->search->search($querydata);
    1030          $this->assert_result_titles(
    1031                  ['Entry1', 'Post1', 'Post2'], $results);
    1032  
    1033          // Restriction to users 1 and 2 combined with context restriction.
    1034          $querydata->contextids = [\context_module::instance($glossary->cmid)->id];
    1035          $results = $this->search->search($querydata);
    1036          $this->assert_result_titles(
    1037                  ['Entry1'], $results);
    1038  
    1039          // Restriction to users 1 and 2 combined with area restriction.
    1040          unset($querydata->contextids);
    1041          $querydata->areaids = [\core_search\manager::generate_areaid('mod_forum', 'post')];
    1042          $results = $this->search->search($querydata);
    1043          $this->assert_result_titles(
    1044                  ['Post1', 'Post2'], $results);
    1045      }
    1046  
    1047      /**
    1048       * Tests searching for results containing words in italic text. (This used to fail.)
    1049       */
    1050      public function test_italics() {
    1051          global $USER;
    1052  
    1053          // Use real search areas.
    1054          $this->search->clear_static();
    1055          $this->search->add_core_search_areas();
    1056  
    1057          // Create a course and a forum.
    1058          $generator = $this->getDataGenerator();
    1059          $course = $generator->create_course();
    1060          $forum = $generator->create_module('forum', ['course' => $course->id]);
    1061  
    1062          // As admin user, create forum discussions with various words in italics or with underlines.
    1063          $this->setAdminUser();
    1064          $forumgen = $generator->get_plugin_generator('mod_forum');
    1065          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1066                  'userid' => $USER->id, 'name' => 'Post1',
    1067                  'message' => '<p>This is a post about <i>frogs</i>.</p>']);
    1068          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1069                  'userid' => $USER->id, 'name' => 'Post2',
    1070                  'message' => '<p>This is a post about <i>toads and zombies</i>.</p>']);
    1071          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1072                  'userid' => $USER->id, 'name' => 'Post3',
    1073                  'message' => '<p>This is a post about toads_and_zombies.</p>']);
    1074          $forumgen->create_discussion(['course' => $course->id, 'forum' => $forum->id,
    1075                  'userid' => $USER->id, 'name' => 'Post4',
    1076                  'message' => '<p>This is a post about _leading and trailing_ underlines.</p>']);
    1077  
    1078          // Index the data.
    1079          $this->search->index();
    1080  
    1081          // Search for 'frogs' should find the post.
    1082          $querydata = new \stdClass();
    1083          $querydata->q = 'frogs';
    1084          $results = $this->search->search($querydata);
    1085          $this->assert_result_titles(['Post1'], $results);
    1086  
    1087          // Search for 'toads' or 'zombies' should find post 2 (and not 3)...
    1088          $querydata->q = 'toads';
    1089          $results = $this->search->search($querydata);
    1090          $this->assert_result_titles(['Post2'], $results);
    1091          $querydata->q = 'zombies';
    1092          $results = $this->search->search($querydata);
    1093          $this->assert_result_titles(['Post2'], $results);
    1094  
    1095          // Search for 'toads_and_zombies' should find post 3.
    1096          $querydata->q = 'toads_and_zombies';
    1097          $results = $this->search->search($querydata);
    1098          $this->assert_result_titles(['Post3'], $results);
    1099  
    1100          // Search for '_leading' or 'trailing_' should find post 4.
    1101          $querydata->q = '_leading';
    1102          $results = $this->search->search($querydata);
    1103          $this->assert_result_titles(['Post4'], $results);
    1104          $querydata->q = 'trailing_';
    1105          $results = $this->search->search($querydata);
    1106          $this->assert_result_titles(['Post4'], $results);
    1107      }
    1108  
    1109      /**
    1110       * Asserts that the returned documents have the expected titles (regardless of order).
    1111       *
    1112       * @param string[] $expected List of expected document titles
    1113       * @param \core_search\document[] $results List of returned documents
    1114       */
    1115      protected function assert_result_titles(array $expected, array $results) {
    1116          $titles = [];
    1117          foreach ($results as $result) {
    1118              $titles[] = $result->get('title');
    1119          }
    1120          sort($titles);
    1121          sort($expected);
    1122          $this->assertEquals($expected, $titles);
    1123      }
    1124  
    1125      /**
    1126       * Tests the get_supported_orders function for contexts where we can only use relevance
    1127       * (system, category).
    1128       */
    1129      public function test_get_supported_orders_relevance_only() {
    1130          global $DB;
    1131  
    1132          // System or category context: relevance only.
    1133          $orders = $this->engine->get_supported_orders(\context_system::instance());
    1134          $this->assertCount(1, $orders);
    1135          $this->assertArrayHasKey('relevance', $orders);
    1136  
    1137          $categoryid = $DB->get_field_sql('SELECT MIN(id) FROM {course_categories}');
    1138          $orders = $this->engine->get_supported_orders(\context_coursecat::instance($categoryid));
    1139          $this->assertCount(1, $orders);
    1140          $this->assertArrayHasKey('relevance', $orders);
    1141      }
    1142  
    1143      /**
    1144       * Tests the get_supported_orders function for contexts where we support location as well
    1145       * (course, activity, block).
    1146       */
    1147      public function test_get_supported_orders_relevance_and_location() {
    1148          global $DB;
    1149  
    1150          // Test with course context.
    1151          $generator = $this->getDataGenerator();
    1152          $course = $generator->create_course(['fullname' => 'Frogs']);
    1153          $coursecontext = \context_course::instance($course->id);
    1154  
    1155          $orders = $this->engine->get_supported_orders($coursecontext);
    1156          $this->assertCount(2, $orders);
    1157          $this->assertArrayHasKey('relevance', $orders);
    1158          $this->assertArrayHasKey('location', $orders);
    1159          $this->assertStringContainsString('Course: Frogs', $orders['location']);
    1160  
    1161          // Test with activity context.
    1162          $page = $generator->create_module('page', ['course' => $course->id, 'name' => 'Toads']);
    1163  
    1164          $orders = $this->engine->get_supported_orders(\context_module::instance($page->cmid));
    1165          $this->assertCount(2, $orders);
    1166          $this->assertArrayHasKey('relevance', $orders);
    1167          $this->assertArrayHasKey('location', $orders);
    1168          $this->assertStringContainsString('Page: Toads', $orders['location']);
    1169  
    1170          // Test with block context.
    1171          $instance = (object)['blockname' => 'html', 'parentcontextid' => $coursecontext->id,
    1172                  'showinsubcontexts' => 0, 'pagetypepattern' => 'course-view-*',
    1173                  'defaultweight' => 0, 'timecreated' => 1, 'timemodified' => 1,
    1174                  'configdata' => ''];
    1175          $blockid = $DB->insert_record('block_instances', $instance);
    1176          $blockcontext = \context_block::instance($blockid);
    1177  
    1178          $orders = $this->engine->get_supported_orders($blockcontext);
    1179          $this->assertCount(2, $orders);
    1180          $this->assertArrayHasKey('relevance', $orders);
    1181          $this->assertArrayHasKey('location', $orders);
    1182          $this->assertStringContainsString('Block: HTML', $orders['location']);
    1183      }
    1184  
    1185      /**
    1186       * Tests ordering by relevance vs location.
    1187       */
    1188      public function test_ordering() {
    1189          // Create 2 courses and 2 activities.
    1190          $generator = $this->getDataGenerator();
    1191          $course1 = $generator->create_course(['fullname' => 'Course 1']);
    1192          $course1context = \context_course::instance($course1->id);
    1193          $course1page = $generator->create_module('page', ['course' => $course1]);
    1194          $course1pagecontext = \context_module::instance($course1page->cmid);
    1195          $course2 = $generator->create_course(['fullname' => 'Course 2']);
    1196          $course2context = \context_course::instance($course2->id);
    1197          $course2page = $generator->create_module('page', ['course' => $course2]);
    1198          $course2pagecontext = \context_module::instance($course2page->cmid);
    1199  
    1200          // Create one search record in each activity and course.
    1201          $this->create_search_record($course1->id, $course1context->id, 'C1', 'Xyzzy');
    1202          $this->create_search_record($course1->id, $course1pagecontext->id, 'C1P', 'Xyzzy');
    1203          $this->create_search_record($course2->id, $course2context->id, 'C2', 'Xyzzy');
    1204          $this->create_search_record($course2->id, $course2pagecontext->id, 'C2P', 'Xyzzy plugh');
    1205          $this->search->index();
    1206  
    1207          // Default search works by relevance so the one with both words should be top.
    1208          $querydata = new \stdClass();
    1209          $querydata->q = 'xyzzy plugh';
    1210          $results = $this->search->search($querydata);
    1211          $this->assertCount(4, $results);
    1212          $this->assertEquals('C2P', $results[0]->get('title'));
    1213  
    1214          // Same if you explicitly specify relevance.
    1215          $querydata->order = 'relevance';
    1216          $results = $this->search->search($querydata);
    1217          $this->assertEquals('C2P', $results[0]->get('title'));
    1218  
    1219          // If you specify order by location and you are in C2 or C2P then results are the same.
    1220          $querydata->order = 'location';
    1221          $querydata->context = $course2context;
    1222          $results = $this->search->search($querydata);
    1223          $this->assertEquals('C2P', $results[0]->get('title'));
    1224          $querydata->context = $course2pagecontext;
    1225          $results = $this->search->search($querydata);
    1226          $this->assertEquals('C2P', $results[0]->get('title'));
    1227  
    1228          // But if you are in C1P then you get different results (C1P first).
    1229          $querydata->context = $course1pagecontext;
    1230          $results = $this->search->search($querydata);
    1231          $this->assertEquals('C1P', $results[0]->get('title'));
    1232      }
    1233  
    1234      /**
    1235       * Tests with bogus content (that can be entered into Moodle) to see if it crashes.
    1236       */
    1237      public function test_bogus_content() {
    1238          $generator = $this->getDataGenerator();
    1239          $course1 = $generator->create_course(['fullname' => 'Course 1']);
    1240          $course1context = \context_course::instance($course1->id);
    1241  
    1242          // It is possible to enter into a Moodle database content containing these characters,
    1243          // which are Unicode non-characters / byte order marks. If sent to Solr, these cause
    1244          // failures.
    1245          $boguscontent = html_entity_decode('&#xfffe;') . 'frog';
    1246          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1247          $boguscontent = html_entity_decode('&#xffff;') . 'frog';
    1248          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1249  
    1250          // Unicode Standard Version 9.0 - Core Specification, section 23.7, lists 66 non-characters
    1251          // in total. Here are some of them - these work OK for me but it may depend on platform.
    1252          $boguscontent = html_entity_decode('&#xfdd0;') . 'frog';
    1253          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1254          $boguscontent = html_entity_decode('&#xfdef;') . 'frog';
    1255          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1256          $boguscontent = html_entity_decode('&#x1fffe;') . 'frog';
    1257          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1258          $boguscontent = html_entity_decode('&#x10ffff;') . 'frog';
    1259          $this->create_search_record($course1->id, $course1context->id, 'C1', $boguscontent);
    1260  
    1261          // Do the indexing (this will check it doesn't throw warnings).
    1262          $this->search->index();
    1263  
    1264          // Confirm that all 6 documents are found in search.
    1265          $querydata = new \stdClass();
    1266          $querydata->q = 'frog';
    1267          $results = $this->search->search($querydata);
    1268          $this->assertCount(6, $results);
    1269      }
    1270  
    1271      /**
    1272       * Adds a record to the mock search area, so that the search engine can find it later.
    1273       *
    1274       * @param int $courseid Course id
    1275       * @param int $contextid Context id
    1276       * @param string $title Title for search index
    1277       * @param string $content Content for search index
    1278       */
    1279      protected function create_search_record($courseid, $contextid, $title, $content) {
    1280          $record = new \stdClass();
    1281          $record->content = $content;
    1282          $record->title = $title;
    1283          $record->courseid = $courseid;
    1284          $record->contextid = $contextid;
    1285          $this->generator->create_record($record);
    1286      }
    1287  
    1288      /**
    1289       * Tries out deleting data for a context or a course.
    1290       *
    1291       * @throws coding_exception
    1292       * @throws moodle_exception
    1293       */
    1294      public function test_deleted_contexts_and_courses() {
    1295          // Create some courses and activities.
    1296          $generator = $this->getDataGenerator();
    1297          $course1 = $generator->create_course(['fullname' => 'Course 1']);
    1298          $course1context = \context_course::instance($course1->id);
    1299          $course1page1 = $generator->create_module('page', ['course' => $course1]);
    1300          $course1page1context = \context_module::instance($course1page1->cmid);
    1301          $course1page2 = $generator->create_module('page', ['course' => $course1]);
    1302          $course1page2context = \context_module::instance($course1page2->cmid);
    1303          $course2 = $generator->create_course(['fullname' => 'Course 2']);
    1304          $course2context = \context_course::instance($course2->id);
    1305          $course2page = $generator->create_module('page', ['course' => $course2]);
    1306          $course2pagecontext = \context_module::instance($course2page->cmid);
    1307  
    1308          // Create one search record in each activity and course.
    1309          $this->create_search_record($course1->id, $course1context->id, 'C1', 'Xyzzy');
    1310          $this->create_search_record($course1->id, $course1page1context->id, 'C1P1', 'Xyzzy');
    1311          $this->create_search_record($course1->id, $course1page2context->id, 'C1P2', 'Xyzzy');
    1312          $this->create_search_record($course2->id, $course2context->id, 'C2', 'Xyzzy');
    1313          $this->create_search_record($course2->id, $course2pagecontext->id, 'C2P', 'Xyzzy plugh');
    1314          $this->search->index();
    1315  
    1316          // By default we have all results.
    1317          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2', 'C2', 'C2P']);
    1318  
    1319          // Say we delete the course2pagecontext...
    1320          $this->engine->delete_index_for_context($course2pagecontext->id);
    1321          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2', 'C2']);
    1322  
    1323          // Now delete the second course...
    1324          $this->engine->delete_index_for_course($course2->id);
    1325          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P1', 'C1P2']);
    1326  
    1327          // Finally let's delete using Moodle functions to check that works. Single context first.
    1328          course_delete_module($course1page1->cmid);
    1329          $this->assert_raw_solr_query_result('content:xyzzy', ['C1', 'C1P2']);
    1330          delete_course($course1, false);
    1331          $this->assert_raw_solr_query_result('content:xyzzy', []);
    1332      }
    1333  
    1334      /**
    1335       * Specific test of the add_document_batch function (also used in many other tests).
    1336       */
    1337      public function test_add_document_batch() {
    1338          // Get a default document.
    1339          $area = new \core_mocksearch\search\mock_search_area();
    1340          $record = $this->generator->create_record();
    1341          $doc = $area->get_document($record);
    1342          $originalid = $doc->get('id');
    1343  
    1344          // Now create 5 similar documents.
    1345          $docs = [];
    1346          for ($i = 1; $i <= 5; $i++) {
    1347              $doc = $area->get_document($record);
    1348              $doc->set('id', $originalid . '-' . $i);
    1349              $doc->set('title', 'Batch ' . $i);
    1350              $docs[$i] = $doc;
    1351          }
    1352  
    1353          // Document 3 has a file attached.
    1354          $fs = get_file_storage();
    1355          $filerecord = new \stdClass();
    1356          $filerecord->content = 'Some FileContents';
    1357          $file = $this->generator->create_file($filerecord);
    1358          $docs[3]->add_stored_file($file);
    1359  
    1360          // Add all these documents to the search engine.
    1361          $this->assertEquals([5, 0, 1], $this->engine->add_document_batch($docs, true));
    1362          $this->engine->area_index_complete($area->get_area_id());
    1363  
    1364          // Check all documents were indexed.
    1365          $querydata = new \stdClass();
    1366          $querydata->q = 'Batch';
    1367          $results = $this->search->search($querydata);
    1368          $this->assertCount(5, $results);
    1369  
    1370          // Check it also finds based on the file.
    1371          $querydata->q = 'FileContents';
    1372          $results = $this->search->search($querydata);
    1373          $this->assertCount(1, $results);
    1374      }
    1375  
    1376      /**
    1377       * Tests the batching logic, specifically the limit to 100 documents per
    1378       * batch, and not batching very large documents.
    1379       */
    1380      public function test_batching() {
    1381          $area = new \core_mocksearch\search\mock_search_area();
    1382          $record = $this->generator->create_record();
    1383          $doc = $area->get_document($record);
    1384          $originalid = $doc->get('id');
    1385  
    1386          // Up to 100 documents in 1 batch.
    1387          $docs = [];
    1388          for ($i = 1; $i <= 100; $i++) {
    1389              $doc = $area->get_document($record);
    1390              $doc->set('id', $originalid . '-' . $i);
    1391              $docs[$i] = $doc;
    1392          }
    1393          [, , , , , $batches] = $this->engine->add_documents(
    1394                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
    1395          $this->assertEquals(1, $batches);
    1396  
    1397          // More than 100 needs 2 batches.
    1398          $docs = [];
    1399          for ($i = 1; $i <= 101; $i++) {
    1400              $doc = $area->get_document($record);
    1401              $doc->set('id', $originalid . '-' . $i);
    1402              $docs[$i] = $doc;
    1403          }
    1404          [, , , , , $batches] = $this->engine->add_documents(
    1405                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
    1406          $this->assertEquals(2, $batches);
    1407  
    1408          // Small number but with some large documents that aren't batched.
    1409          $docs = [];
    1410          for ($i = 1; $i <= 10; $i++) {
    1411              $doc = $area->get_document($record);
    1412              $doc->set('id', $originalid . '-' . $i);
    1413              $docs[$i] = $doc;
    1414          }
    1415          // This one is just small enough to fit.
    1416          $docs[3]->set('content', str_pad('xyzzy ', 1024 * 1024, 'x'));
    1417          // These two don't fit.
    1418          $docs[5]->set('content', str_pad('xyzzy ', 1024 * 1024 + 1, 'x'));
    1419          $docs[6]->set('content', str_pad('xyzzy ', 1024 * 1024 + 1, 'x'));
    1420          [, , , , , $batches] = $this->engine->add_documents(
    1421                  new \ArrayIterator($docs), $area, ['indexfiles' => true]);
    1422          $this->assertEquals(3, $batches);
    1423  
    1424          // Check that all 3 of the large documents (added as batch or not) show up in results.
    1425          $this->engine->area_index_complete($area->get_area_id());
    1426          $querydata = new \stdClass();
    1427          $querydata->q = 'xyzzy';
    1428          $results = $this->search->search($querydata);
    1429          $this->assertCount(3, $results);
    1430      }
    1431  
    1432      /**
    1433       * Tests with large documents. The point of this test is that we stop batching
    1434       * documents if they are bigger than 1MB, and the maximum batch count is 100,
    1435       * so the maximum size batch will be about 100 1MB documents.
    1436       */
    1437      public function test_add_document_batch_large() {
    1438          // This test is a bit slow and not that important to run every time...
    1439          if (!PHPUNIT_LONGTEST) {
    1440              $this->markTestSkipped('PHPUNIT_LONGTEST is not defined');
    1441          }
    1442  
    1443          // Get a default document.
    1444          $area = new \core_mocksearch\search\mock_search_area();
    1445          $record = $this->generator->create_record();
    1446          $doc = $area->get_document($record);
    1447          $originalid = $doc->get('id');
    1448  
    1449          // Now create 100 large documents.
    1450          $size = 1024 * 1024;
    1451          $docs = [];
    1452          for ($i = 1; $i <= 100; $i++) {
    1453              $doc = $area->get_document($record);
    1454              $doc->set('id', $originalid . '-' . $i);
    1455              $doc->set('title', 'Batch ' . $i);
    1456              $doc->set('content', str_pad('', $size, 'Long text ' . $i . '. ', STR_PAD_RIGHT) . ' xyzzy');
    1457              $docs[$i] = $doc;
    1458          }
    1459  
    1460          // Add all these documents to the search engine.
    1461          $this->engine->add_document_batch($docs, true);
    1462          $this->engine->area_index_complete($area->get_area_id());
    1463  
    1464          // Check all documents were indexed, searching for text at end.
    1465          $querydata = new \stdClass();
    1466          $querydata->q = 'xyzzy';
    1467          $results = $this->search->search($querydata);
    1468          $this->assertCount(100, $results);
    1469  
    1470          // Search for specific text that's only in one.
    1471          $querydata->q = '42';
    1472          $results = $this->search->search($querydata);
    1473          $this->assertCount(1, $results);
    1474      }
    1475  
    1476      /**
    1477       * Carries out a raw Solr query using the Solr basic query syntax.
    1478       *
    1479       * This is used to test data contained in the index without going through Moodle processing.
    1480       *
    1481       * @param string $q Search query
    1482       * @param string[] $expected Expected titles of results, in alphabetical order
    1483       */
    1484      protected function assert_raw_solr_query_result(string $q, array $expected) {
    1485          $solr = $this->engine->get_search_client_public();
    1486          $query = new \SolrQuery($q);
    1487          $results = $solr->query($query)->getResponse()->response->docs;
    1488          if ($results) {
    1489              $titles = array_map(function($x) {
    1490                  return $x->title;
    1491              }, $results);
    1492              sort($titles);
    1493          } else {
    1494              $titles = [];
    1495          }
    1496          $this->assertEquals($expected, $titles);
    1497      }
    1498  }