[Summary view]

Differences Between: [Versions 400 and 401] [Versions 401 and 402] [Versions 401 and 403]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  namespace core_question\local\statistics;
  18  
  19  use advanced_testcase;
  20  use context;
  21  use context_module;
  22  use core_question\statistics\questions\all_calculated_for_qubaid_condition;
  23  use quiz_statistics\tests\statistics_helper;
  24  use core_question_generator;
  25  use Generator;
  26  use quiz;
  27  use quiz_attempt;
  28  use question_engine;
  29  use ReflectionMethod;
  30  
  31  /**
  32   * Tests for question statistics.
  33   *
  34   * @package   core_question
  35   * @copyright 2021 Catalyst IT Australia Pty Ltd
  36   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  37   * @covers \core_question\local\statistics\statistics_bulk_loader
  38   */
  39  class statistics_bulk_loader_test extends advanced_testcase {
  40  
  41      /** @var float Delta used when comparing statistics values out-of 1. */
  42      protected const DELTA = 0.00005;
  43  
  44      /** @var float Delta used when comparing statistics values out-of 100. */
  45      protected const PERCENT_DELTA = 0.005;
  46  
  47      /**
  48       * Test quizzes that contain a specified question.
  49       *
  50       * @covers ::get_all_places_where_questions_were_attempted
  51       */
  52      public function test_get_all_places_where_questions_were_attempted(): void {
  53          global $DB;
  54          $this->resetAfterTest();
  55          $this->setAdminUser();
  56  
  57          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'get_all_places_where_questions_were_attempted');
  58          $rcm->setAccessible(true);
  59  
  60          // Create a course.
  61          $course = $this->getDataGenerator()->create_course();
  62  
  63          // Create three quizzes.
  64          $quizgenerator = $this->getDataGenerator()->get_plugin_generator('mod_quiz');
  65          $quiz1 = $quizgenerator->create_instance([
  66              'course' => $course->id,
  67              'grade' => 100.0, 'sumgrades' => 2,
  68              'layout' => '1,2,0'
  69          ]);
  70          $quiz1context = context_module::instance($quiz1->cmid);
  71  
  72          $quiz2 = $quizgenerator->create_instance([
  73              'course' => $course->id,
  74              'grade' => 100.0, 'sumgrades' => 2,
  75              'layout' => '1,2,0'
  76          ]);
  77          $quiz2context = context_module::instance($quiz2->cmid);
  78  
  79          $quiz3 = $quizgenerator->create_instance([
  80              'course' => $course->id,
  81              'grade' => 100.0, 'sumgrades' => 2,
  82              'layout' => '1,2,0'
  83          ]);
  84          $quiz3context = context_module::instance($quiz3->cmid);
  85  
  86          // Create questions.
  87          /** @var core_question_generator $questiongenerator */
  88          $questiongenerator = $this->getDataGenerator()->get_plugin_generator('core_question');
  89          $cat = $questiongenerator->create_question_category();
  90          $question1 = $questiongenerator->create_question('shortanswer', null, ['category' => $cat->id]);
  91          $question2 = $questiongenerator->create_question('numerical', null, ['category' => $cat->id]);
  92  
  93          // Add question 1 to quiz 1 and make an attempt.
  94          quiz_add_quiz_question($question1->id, $quiz1);
  95          // Quiz 1 attempt.
  96          $this->submit_quiz($quiz1, [1 => ['answer' => 'frog']]);
  97  
  98          // Add questions 1 and 2 to quiz 2.
  99          quiz_add_quiz_question($question1->id, $quiz2);
 100          quiz_add_quiz_question($question2->id, $quiz2);
 101          $this->submit_quiz($quiz2, [1 => ['answer' => 'frog'], 2 => ['answer' => 10]]);
 102  
 103          // Checking quizzes that use question 1.
 104          $q1places = $rcm->invoke(null, [$question1->id]);
 105          $this->assertCount(2, $q1places);
 106          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz1context->id], $q1places[0]);
 107          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q1places[1]);
 108  
 109          // Checking quizzes that contain question 2.
 110          $q2places = $rcm->invoke(null, [$question2->id]);
 111          $this->assertCount(1, $q2places);
 112          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q2places[0]);
 113  
 114          // Add a random question to quiz3.
 115          quiz_add_random_questions($quiz3, 0, $cat->id, 1, false);
 116          $this->submit_quiz($quiz3, [1 => ['answer' => 'willbewrong']]);
 117  
 118          // Quiz 3 will now be in one of these arrays.
 119          $q1places = $rcm->invoke(null, [$question1->id]);
 120          $q2places = $rcm->invoke(null, [$question2->id]);
 121          if (count($q1places) == 3) {
 122              $newplace = end($q1places);
 123          } else {
 124              $newplace = end($q2places);
 125          }
 126          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz3context->id], $newplace);
 127  
 128          // Simulate the situation where the context for quiz3 is gone from the database, without
 129          // the corresponding attempt data being properly cleaned up. Ensure this does not cause errors.
 130          $DB->delete_records('context', ['id' => context_module::instance($quiz3->cmid)->id]);
 131          accesslib_clear_all_caches_for_unit_testing();
 132          // Same asserts as above, before we added quiz3.
 133          $q1places = $rcm->invoke(null, [$question1->id]);
 134          $this->assertCount(2, $q1places);
 135          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz1context->id], $q1places[0]);
 136          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q1places[1]);
 137          $q2places = $rcm->invoke(null, [$question2->id]);
 138          $this->assertCount(1, $q2places);
 139          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q2places[0]);
 140      }
 141  
 142      /**
 143       * Create 2 quizzes.
 144       *
 145       * @return array return 2 quizzes
 146       */
 147      private function prepare_quizzes(): array {
 148          // Create a course.
 149          $course = $this->getDataGenerator()->create_course();
 150  
 151          // Make 2 quizzes.
 152          $quizgenerator = $this->getDataGenerator()->get_plugin_generator('mod_quiz');
 153          $layout = '1,2,0,3,4,0';
 154          $quiz1 = $quizgenerator->create_instance([
 155              'course' => $course->id,
 156              'grade' => 100.0, 'sumgrades' => 2,
 157              'layout' => $layout
 158          ]);
 159  
 160          $quiz2 = $quizgenerator->create_instance([
 161              'course' => $course->id,
 162              'grade' => 100.0, 'sumgrades' => 2,
 163              'layout' => $layout
 164          ]);
 165  
 166          /** @var core_question_generator $questiongenerator */
 167          $questiongenerator = $this->getDataGenerator()->get_plugin_generator('core_question');
 168          $cat = $questiongenerator->create_question_category();
 169  
 170          $page = 1;
 171          $questions = [];
 172          foreach (explode(',', $layout) as $slot) {
 173              if ($slot == 0) {
 174                  $page += 1;
 175                  continue;
 176              }
 177  
 178              $question = $questiongenerator->create_question('shortanswer', null, ['category' => $cat->id]);
 179              $questions[$slot] = $question;
 180              quiz_add_quiz_question($question->id, $quiz1, $page);
 181              quiz_add_quiz_question($question->id, $quiz2, $page);
 182          }
 183  
 184          return [$quiz1, $quiz2, $questions];
 185      }
 186  
 187      /**
 188       * Submit quiz answers
 189       *
 190       * @param object $quiz
 191       * @param array $answers
 192       */
 193      private function submit_quiz(object $quiz, array $answers): void {
 194          // Create user.
 195          $user = $this->getDataGenerator()->create_user();
 196          // Create attempt.
 197          $quizobj = quiz::create($quiz->id, $user->id);
 198          $quba = question_engine::make_questions_usage_by_activity('mod_quiz', $quizobj->get_context());
 199          $quba->set_preferred_behaviour($quizobj->get_quiz()->preferredbehaviour);
 200          $timenow = time();
 201          $attempt = quiz_create_attempt($quizobj, 1, null, $timenow, false, $user->id);
 202          quiz_start_new_attempt($quizobj, $quba, $attempt, 1, $timenow);
 203          quiz_attempt_save_started($quizobj, $quba, $attempt);
 204          // Submit attempt.
 205          $attemptobj = quiz_attempt::create($attempt->id);
 206          $attemptobj->process_submitted_actions($timenow, false, $answers);
 207          $attemptobj->process_finish($timenow, false);
 208      }
 209  
 210      /**
 211       * Generate attempt answers.
 212       *
 213       * @param array $correctanswerflags array of 1 or 0
 214       * 1 : generate correct answer
 215       * 0 : generate wrong answer
 216       *
 217       * @return array
 218       */
 219      private function generate_attempt_answers(array $correctanswerflags): array {
 220          $attempt = [];
 221          for ($i = 1; $i <= 4; $i++) {
 222              if (isset($correctanswerflags) && $correctanswerflags[$i - 1] == 1) {
 223                  // Correct answer.
 224                  $attempt[$i] = ['answer' => 'frog'];
 225              } else {
 226                  $attempt[$i] = ['answer' => 'false'];
 227              }
 228          }
 229          return $attempt;
 230      }
 231  
 232      /**
 233       * Generate quizzes and submit answers.
 234       *
 235       * @param array $quiz1attempts quiz 1 attempts
 236       * @param array $quiz2attempts quiz 2 attempts
 237       *
 238       * @return array
 239       */
 240      private function prepare_and_submit_quizzes(array $quiz1attempts, array $quiz2attempts): array {
 241          list($quiz1, $quiz2, $questions) = $this->prepare_quizzes();
 242          // Submit attempts of quiz1.
 243          foreach ($quiz1attempts as $attempt) {
 244              $this->submit_quiz($quiz1, $attempt);
 245          }
 246          // Submit attempts of quiz2.
 247          foreach ($quiz2attempts as $attempt) {
 248              $this->submit_quiz($quiz2, $attempt);
 249          }
 250  
 251          // Calculate the statistics.
 252          $this->expectOutputRegex('~.*Calculations completed.*~');
 253          statistics_helper::run_pending_recalculation_tasks();
 254  
 255          return [$quiz1, $quiz2, $questions];
 256      }
 257  
 258      /**
 259       * To use private helper::extract_item_value function.
 260       *
 261       * @param all_calculated_for_qubaid_condition $statistics the batch of statistics.
 262       * @param int $questionid a question id.
 263       * @param string $item one of the field names in all_calculated_for_qubaid_condition, e.g. 'facility'.
 264       * @return float|null the required value.
 265       */
 266      private function extract_item_value(all_calculated_for_qubaid_condition $statistics,
 267                                          int $questionid, string $item): ?float {
 268          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'extract_item_value');
 269          $rcm->setAccessible(true);
 270          return $rcm->invoke(null, $statistics, $questionid, $item);
 271      }
 272  
 273      /**
 274       * To use private helper::load_statistics_for_place function (with mod_quiz component).
 275       *
 276       * @param context $context the context to load the statistics for.
 277       * @return all_calculated_for_qubaid_condition|null question statistics.
 278       */
 279      private function load_quiz_statistics_for_place(context $context): ?all_calculated_for_qubaid_condition {
 280          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'load_statistics_for_place');
 281          $rcm->setAccessible(true);
 282          return $rcm->invoke(null, 'mod_quiz', $context);
 283      }
 284  
 285      /**
 286       * Data provider for {@see test_load_question_facility()}.
 287       *
 288       * @return Generator
 289       */
 290      public function load_question_facility_provider(): Generator {
 291          yield 'Facility case 1' => [
 292              'Quiz 1 attempts' => [
 293                  $this->generate_attempt_answers([1, 0, 0, 0]),
 294              ],
 295              'Expected quiz 1 facilities' => [1.0, 0.0, 0.0, 0.0],
 296              'Quiz 2 attempts' => [
 297                  $this->generate_attempt_answers([1, 0, 0, 0]),
 298                  $this->generate_attempt_answers([1, 1, 0, 0]),
 299              ],
 300              'Expected quiz 2 facilities' => [1.0, 0.5, 0.0, 0.0],
 301              'Expected average facilities' => [1.0, 0.25, 0.0, 0.0],
 302          ];
 303          yield 'Facility case 2' => [
 304              'Quiz 1 attempts' => [
 305                  $this->generate_attempt_answers([1, 0, 0, 0]),
 306                  $this->generate_attempt_answers([1, 1, 0, 0]),
 307                  $this->generate_attempt_answers([1, 1, 1, 0]),
 308              ],
 309              'Expected quiz 1 facilities' => [1.0, 0.6667, 0.3333, 0.0],
 310              'Quiz 2 attempts' => [
 311                  $this->generate_attempt_answers([1, 0, 0, 0]),
 312                  $this->generate_attempt_answers([1, 1, 0, 0]),
 313                  $this->generate_attempt_answers([1, 1, 1, 0]),
 314                  $this->generate_attempt_answers([1, 1, 1, 1]),
 315              ],
 316              'Expected quiz 2 facilities' => [1.0, 0.75, 0.5, 0.25],
 317              'Expected average facilities' => [1.0, 0.7083, 0.4167, 0.1250],
 318          ];
 319      }
 320  
 321      /**
 322       * Test question facility
 323       *
 324       * @dataProvider load_question_facility_provider
 325       *
 326       * @param array $quiz1attempts quiz 1 attempts
 327       * @param array $expectedquiz1facilities expected quiz 1 facilities
 328       * @param array $quiz2attempts quiz 2 attempts
 329       * @param array $expectedquiz2facilities  expected quiz 2 facilities
 330       * @param array $expectedaveragefacilities expected average facilities
 331       */
 332      public function test_load_question_facility(
 333          array $quiz1attempts,
 334          array $expectedquiz1facilities,
 335          array $quiz2attempts,
 336          array $expectedquiz2facilities,
 337          array $expectedaveragefacilities)
 338      : void {
 339          $this->resetAfterTest();
 340  
 341          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 342  
 343          // Quiz 1 facilities.
 344          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 345          $quiz1facility1 = $this->extract_item_value($stats, $questions[1]->id, 'facility');
 346          $quiz1facility2 = $this->extract_item_value($stats, $questions[2]->id, 'facility');
 347          $quiz1facility3 = $this->extract_item_value($stats, $questions[3]->id, 'facility');
 348          $quiz1facility4 = $this->extract_item_value($stats, $questions[4]->id, 'facility');
 349  
 350          $this->assertEqualsWithDelta($expectedquiz1facilities[0], $quiz1facility1, self::DELTA);
 351          $this->assertEqualsWithDelta($expectedquiz1facilities[1], $quiz1facility2, self::DELTA);
 352          $this->assertEqualsWithDelta($expectedquiz1facilities[2], $quiz1facility3, self::DELTA);
 353          $this->assertEqualsWithDelta($expectedquiz1facilities[3], $quiz1facility4, self::DELTA);
 354  
 355          // Quiz 2 facilities.
 356          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 357          $quiz2facility1 = $this->extract_item_value($stats, $questions[1]->id, 'facility');
 358          $quiz2facility2 = $this->extract_item_value($stats, $questions[2]->id, 'facility');
 359          $quiz2facility3 = $this->extract_item_value($stats, $questions[3]->id, 'facility');
 360          $quiz2facility4 = $this->extract_item_value($stats, $questions[4]->id, 'facility');
 361  
 362          $this->assertEqualsWithDelta($expectedquiz2facilities[0], $quiz2facility1, self::DELTA);
 363          $this->assertEqualsWithDelta($expectedquiz2facilities[1], $quiz2facility2, self::DELTA);
 364          $this->assertEqualsWithDelta($expectedquiz2facilities[2], $quiz2facility3, self::DELTA);
 365          $this->assertEqualsWithDelta($expectedquiz2facilities[3], $quiz2facility4, self::DELTA);
 366  
 367          // Average question facilities.
 368          $stats = statistics_bulk_loader::load_aggregate_statistics(
 369              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 370              ['facility']
 371          );
 372  
 373          $this->assertEqualsWithDelta($expectedaveragefacilities[0],
 374              $stats[$questions[1]->id]['facility'], self::DELTA);
 375          $this->assertEqualsWithDelta($expectedaveragefacilities[1],
 376              $stats[$questions[2]->id]['facility'], self::DELTA);
 377          $this->assertEqualsWithDelta($expectedaveragefacilities[2],
 378              $stats[$questions[3]->id]['facility'], self::DELTA);
 379          $this->assertEqualsWithDelta($expectedaveragefacilities[3],
 380              $stats[$questions[4]->id]['facility'], self::DELTA);
 381      }
 382  
 383      /**
 384       * Data provider for {@see test_load_question_discriminative_efficiency()}.
 385       * @return Generator
 386       */
 387      public function load_question_discriminative_efficiency_provider(): Generator {
 388          yield 'Discriminative efficiency' => [
 389              'Quiz 1 attempts' => [
 390                  $this->generate_attempt_answers([1, 0, 0, 0]),
 391                  $this->generate_attempt_answers([1, 1, 0, 0]),
 392                  $this->generate_attempt_answers([1, 0, 1, 0]),
 393                  $this->generate_attempt_answers([1, 1, 1, 1]),
 394              ],
 395              'Expected quiz 1 discriminative efficiency' => [null, 33.33, 33.33, 100.00],
 396              'Quiz 2 attempts' => [
 397                  $this->generate_attempt_answers([1, 1, 1, 1]),
 398                  $this->generate_attempt_answers([0, 0, 0, 0]),
 399                  $this->generate_attempt_answers([1, 0, 0, 1]),
 400                  $this->generate_attempt_answers([0, 1, 1, 0]),
 401              ],
 402              'Expected quiz 2 discriminative efficiency' => [50.00, 50.00, 50.00, 50.00],
 403              'Expected average discriminative efficiency' => [50.00, 41.67, 41.67, 75.00],
 404          ];
 405      }
 406  
 407      /**
 408       * Test discriminative efficiency
 409       *
 410       * @dataProvider load_question_discriminative_efficiency_provider
 411       *
 412       * @param array $quiz1attempts quiz 1 attempts
 413       * @param array $expectedquiz1discriminativeefficiency expected quiz 1 discriminative efficiency
 414       * @param array $quiz2attempts quiz 2 attempts
 415       * @param array $expectedquiz2discriminativeefficiency expected quiz 2 discriminative efficiency
 416       * @param array $expectedaveragediscriminativeefficiency expected average discriminative efficiency
 417       */
 418      public function test_load_question_discriminative_efficiency(
 419          array $quiz1attempts,
 420          array $expectedquiz1discriminativeefficiency,
 421          array $quiz2attempts,
 422          array $expectedquiz2discriminativeefficiency,
 423          array $expectedaveragediscriminativeefficiency
 424      ): void {
 425          $this->resetAfterTest();
 426  
 427          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 428  
 429          // Quiz 1 discriminative efficiency.
 430          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 431          $discriminativeefficiency1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminativeefficiency');
 432          $discriminativeefficiency2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminativeefficiency');
 433          $discriminativeefficiency3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminativeefficiency');
 434          $discriminativeefficiency4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminativeefficiency');
 435  
 436          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[0],
 437                  $discriminativeefficiency1, self::PERCENT_DELTA);
 438          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[1],
 439                  $discriminativeefficiency2, self::PERCENT_DELTA);
 440          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[2],
 441                  $discriminativeefficiency3, self::PERCENT_DELTA);
 442          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[3],
 443                  $discriminativeefficiency4, self::PERCENT_DELTA);
 444  
 445          // Quiz 2 discriminative efficiency.
 446          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 447          $discriminativeefficiency1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminativeefficiency');
 448          $discriminativeefficiency2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminativeefficiency');
 449          $discriminativeefficiency3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminativeefficiency');
 450          $discriminativeefficiency4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminativeefficiency');
 451  
 452          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[0],
 453                  $discriminativeefficiency1, self::PERCENT_DELTA);
 454          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[1],
 455                  $discriminativeefficiency2, self::PERCENT_DELTA);
 456          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[2],
 457                  $discriminativeefficiency3, self::PERCENT_DELTA);
 458          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[3],
 459                  $discriminativeefficiency4, self::PERCENT_DELTA);
 460  
 461          // Average question discriminative efficiency.
 462          $stats = statistics_bulk_loader::load_aggregate_statistics(
 463              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 464              ['discriminativeefficiency']
 465          );
 466  
 467          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[0],
 468              $stats[$questions[1]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 469          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[1],
 470              $stats[$questions[2]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 471          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[2],
 472              $stats[$questions[3]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 473          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[3],
 474              $stats[$questions[4]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 475      }
 476  
 477      /**
 478       * Data provider for {@see test_load_question_discrimination_index()}.
 479       * @return Generator
 480       */
 481      public function load_question_discrimination_index_provider(): Generator {
 482          yield 'Discrimination Index' => [
 483              'Quiz 1 attempts' => [
 484                  $this->generate_attempt_answers([1, 0, 0, 0]),
 485                  $this->generate_attempt_answers([1, 1, 0, 0]),
 486                  $this->generate_attempt_answers([1, 0, 1, 0]),
 487                  $this->generate_attempt_answers([1, 1, 1, 1]),
 488              ],
 489              'Expected quiz 1 Discrimination Index' => [null, 30.15, 30.15, 81.65],
 490              'Quiz 2 attempts' => [
 491                  $this->generate_attempt_answers([1, 1, 1, 1]),
 492                  $this->generate_attempt_answers([0, 0, 0, 0]),
 493                  $this->generate_attempt_answers([1, 0, 0, 1]),
 494                  $this->generate_attempt_answers([0, 1, 1, 0]),
 495              ],
 496              'Expected quiz 2 discrimination Index' => [44.72, 44.72, 44.72, 44.72],
 497              'Expected average discrimination Index' => [44.72, 37.44, 37.44, 63.19],
 498          ];
 499      }
 500  
 501      /**
 502       * Test discrimination index
 503       *
 504       * @dataProvider load_question_discrimination_index_provider
 505       *
 506       * @param array $quiz1attempts quiz 1 attempts
 507       * @param array $expectedquiz1discriminationindex expected quiz 1 discrimination index
 508       * @param array $quiz2attempts quiz 2 attempts
 509       * @param array $expectedquiz2discriminationindex expected quiz 2 discrimination index
 510       * @param array $expectedaveragediscriminationindex expected average discrimination index
 511       */
 512      public function test_load_question_discrimination_index(
 513          array $quiz1attempts,
 514          array $expectedquiz1discriminationindex,
 515          array $quiz2attempts,
 516          array $expectedquiz2discriminationindex,
 517          array $expectedaveragediscriminationindex
 518      ): void {
 519          $this->resetAfterTest();
 520  
 521          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 522  
 523          // Quiz 1 discrimination index.
 524          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 525          $discriminationindex1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminationindex');
 526          $discriminationindex2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminationindex');
 527          $discriminationindex3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminationindex');
 528          $discriminationindex4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminationindex');
 529  
 530          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[0],
 531              $discriminationindex1, self::PERCENT_DELTA);
 532          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[1],
 533              $discriminationindex2, self::PERCENT_DELTA);
 534          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[2],
 535              $discriminationindex3, self::PERCENT_DELTA);
 536          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[3],
 537              $discriminationindex4, self::PERCENT_DELTA);
 538  
 539          // Quiz 2 discrimination index.
 540          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 541          $discriminationindex1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminationindex');
 542          $discriminationindex2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminationindex');
 543          $discriminationindex3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminationindex');
 544          $discriminationindex4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminationindex');
 545  
 546          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[0],
 547              $discriminationindex1, self::PERCENT_DELTA);
 548          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[1],
 549              $discriminationindex2, self::PERCENT_DELTA);
 550          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[2],
 551              $discriminationindex3, self::PERCENT_DELTA);
 552          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[3],
 553              $discriminationindex4, self::PERCENT_DELTA);
 554  
 555          // Average question discrimination index.
 556          $stats = statistics_bulk_loader::load_aggregate_statistics(
 557              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 558              ['discriminationindex']
 559          );
 560  
 561          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[0],
 562              $stats[$questions[1]->id]['discriminationindex'], self::PERCENT_DELTA);
 563          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[1],
 564              $stats[$questions[2]->id]['discriminationindex'], self::PERCENT_DELTA);
 565          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[2],
 566              $stats[$questions[3]->id]['discriminationindex'], self::PERCENT_DELTA);
 567          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[3],
 568              $stats[$questions[4]->id]['discriminationindex'], self::PERCENT_DELTA);
 569      }
 570  
 571      /**
 572       * Test with question statistics disabled
 573       */
 574      public function test_statistics_disabled(): void {
 575          $this->resetAfterTest();
 576  
 577          // Prepare some quizzes and attempts. Exactly what is not important to this test.
 578          $quiz1attempts = [$this->generate_attempt_answers([1, 0, 0, 0])];
 579          $quiz2attempts = [$this->generate_attempt_answers([1, 1, 1, 1])];
 580          [, , $questions] = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 581  
 582          // Prepare some useful arrays.
 583          $expectedstats = [
 584              $questions[1]->id => [],
 585              $questions[2]->id => [],
 586              $questions[3]->id => [],
 587              $questions[4]->id => [],
 588          ];
 589          $questionids = array_keys($expectedstats);
 590  
 591          // Ask to load no statistics at all.
 592          $stats = statistics_bulk_loader::load_aggregate_statistics($questionids, []);
 593  
 594          // Verify we got the right thing.
 595          $this->assertEquals($expectedstats, $stats);
 596      }
 597  }