[Summary view]

Differences Between: [Versions 400 and 403] [Versions 401 and 403] [Versions 402 and 403]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  namespace core_question\local\statistics;
  18  
  19  defined('MOODLE_INTERNAL') || die();
  20  
  21  use advanced_testcase;
  22  use context;
  23  use context_module;
  24  use core_question\statistics\questions\all_calculated_for_qubaid_condition;
  25  use quiz_statistics\tests\statistics_helper;
  26  use core_question_generator;
  27  use Generator;
  28  use mod_quiz\quiz_attempt;
  29  use mod_quiz\quiz_settings;
  30  use question_engine;
  31  use ReflectionMethod;
  32  
  33  global $CFG;
  34  require_once($CFG->dirroot . '/mod/quiz/tests/quiz_question_helper_test_trait.php');
  35  
  36  /**
  37   * Tests for question statistics.
  38   *
  39   * @package   core_question
  40   * @copyright 2021 Catalyst IT Australia Pty Ltd
  41   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  42   * @covers \core_question\local\statistics\statistics_bulk_loader
  43   */
  44  class statistics_bulk_loader_test extends advanced_testcase {
  45  
  46      use \quiz_question_helper_test_trait;
  47  
  48      /** @var float Delta used when comparing statistics values out-of 1. */
  49      protected const DELTA = 0.00005;
  50  
  51      /** @var float Delta used when comparing statistics values out-of 100. */
  52      protected const PERCENT_DELTA = 0.005;
  53  
  54      /**
  55       * Test quizzes that contain a specified question.
  56       *
  57       * @covers ::get_all_places_where_questions_were_attempted
  58       */
  59      public function test_get_all_places_where_questions_were_attempted(): void {
  60          global $DB;
  61          $this->resetAfterTest();
  62          $this->setAdminUser();
  63  
  64          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'get_all_places_where_questions_were_attempted');
  65          $rcm->setAccessible(true);
  66  
  67          // Create a course.
  68          $course = $this->getDataGenerator()->create_course();
  69  
  70          // Create three quizzes.
  71          $quizgenerator = $this->getDataGenerator()->get_plugin_generator('mod_quiz');
  72          $quiz1 = $quizgenerator->create_instance([
  73              'course' => $course->id,
  74              'grade' => 100.0, 'sumgrades' => 2,
  75              'layout' => '1,2,0'
  76          ]);
  77          $quiz1context = context_module::instance($quiz1->cmid);
  78  
  79          $quiz2 = $quizgenerator->create_instance([
  80              'course' => $course->id,
  81              'grade' => 100.0, 'sumgrades' => 2,
  82              'layout' => '1,2,0'
  83          ]);
  84          $quiz2context = context_module::instance($quiz2->cmid);
  85  
  86          $quiz3 = $quizgenerator->create_instance([
  87              'course' => $course->id,
  88              'grade' => 100.0, 'sumgrades' => 2,
  89              'layout' => '1,2,0'
  90          ]);
  91          $quiz3context = context_module::instance($quiz3->cmid);
  92  
  93          // Create questions.
  94          /** @var core_question_generator $questiongenerator */
  95          $questiongenerator = $this->getDataGenerator()->get_plugin_generator('core_question');
  96          $cat = $questiongenerator->create_question_category();
  97          $question1 = $questiongenerator->create_question('shortanswer', null, ['category' => $cat->id]);
  98          $question2 = $questiongenerator->create_question('numerical', null, ['category' => $cat->id]);
  99  
 100          // Add question 1 to quiz 1 and make an attempt.
 101          quiz_add_quiz_question($question1->id, $quiz1);
 102          // Quiz 1 attempt.
 103          $this->submit_quiz($quiz1, [1 => ['answer' => 'frog']]);
 104  
 105          // Add questions 1 and 2 to quiz 2.
 106          quiz_add_quiz_question($question1->id, $quiz2);
 107          quiz_add_quiz_question($question2->id, $quiz2);
 108          $this->submit_quiz($quiz2, [1 => ['answer' => 'frog'], 2 => ['answer' => 10]]);
 109  
 110          // Checking quizzes that use question 1.
 111          $q1places = $rcm->invoke(null, [$question1->id]);
 112          $this->assertCount(2, $q1places);
 113          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz1context->id], $q1places[0]);
 114          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q1places[1]);
 115  
 116          // Checking quizzes that contain question 2.
 117          $q2places = $rcm->invoke(null, [$question2->id]);
 118          $this->assertCount(1, $q2places);
 119          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q2places[0]);
 120  
 121          // Add a random question to quiz3.
 122          $this->add_random_questions($quiz3->id, 0, $cat->id, 1, false);
 123          $this->submit_quiz($quiz3, [1 => ['answer' => 'willbewrong']]);
 124  
 125          // Quiz 3 will now be in one of these arrays.
 126          $q1places = $rcm->invoke(null, [$question1->id]);
 127          $q2places = $rcm->invoke(null, [$question2->id]);
 128          if (count($q1places) == 3) {
 129              $newplace = end($q1places);
 130          } else {
 131              $newplace = end($q2places);
 132          }
 133          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz3context->id], $newplace);
 134  
 135          // Simulate the situation where the context for quiz3 is gone from the database, without
 136          // the corresponding attempt data being properly cleaned up. Ensure this does not cause errors.
 137          $DB->delete_records('context', ['id' => context_module::instance($quiz3->cmid)->id]);
 138          accesslib_clear_all_caches_for_unit_testing();
 139          // Same asserts as above, before we added quiz3.
 140          $q1places = $rcm->invoke(null, [$question1->id]);
 141          $this->assertCount(2, $q1places);
 142          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz1context->id], $q1places[0]);
 143          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q1places[1]);
 144          $q2places = $rcm->invoke(null, [$question2->id]);
 145          $this->assertCount(1, $q2places);
 146          $this->assertEquals((object) ['component' => 'mod_quiz', 'contextid' => $quiz2context->id], $q2places[0]);
 147      }
 148  
 149      /**
 150       * Create 2 quizzes.
 151       *
 152       * @return array return 2 quizzes
 153       */
 154      private function prepare_quizzes(): array {
 155          // Create a course.
 156          $course = $this->getDataGenerator()->create_course();
 157  
 158          // Make 2 quizzes.
 159          $quizgenerator = $this->getDataGenerator()->get_plugin_generator('mod_quiz');
 160          $layout = '1,2,0,3,4,0';
 161          $quiz1 = $quizgenerator->create_instance([
 162              'course' => $course->id,
 163              'grade' => 100.0, 'sumgrades' => 2,
 164              'layout' => $layout
 165          ]);
 166  
 167          $quiz2 = $quizgenerator->create_instance([
 168              'course' => $course->id,
 169              'grade' => 100.0, 'sumgrades' => 2,
 170              'layout' => $layout
 171          ]);
 172  
 173          /** @var core_question_generator $questiongenerator */
 174          $questiongenerator = $this->getDataGenerator()->get_plugin_generator('core_question');
 175          $cat = $questiongenerator->create_question_category();
 176  
 177          $page = 1;
 178          $questions = [];
 179          foreach (explode(',', $layout) as $slot) {
 180              if ($slot == 0) {
 181                  $page += 1;
 182                  continue;
 183              }
 184  
 185              $question = $questiongenerator->create_question('shortanswer', null, ['category' => $cat->id]);
 186              $questions[$slot] = $question;
 187              quiz_add_quiz_question($question->id, $quiz1, $page);
 188              quiz_add_quiz_question($question->id, $quiz2, $page);
 189          }
 190  
 191          return [$quiz1, $quiz2, $questions];
 192      }
 193  
 194      /**
 195       * Submit quiz answers
 196       *
 197       * @param object $quiz
 198       * @param array $answers
 199       */
 200      private function submit_quiz(object $quiz, array $answers): void {
 201          // Create user.
 202          $user = $this->getDataGenerator()->create_user();
 203          // Create attempt.
 204          $quizobj = quiz_settings::create($quiz->id, $user->id);
 205          $quba = question_engine::make_questions_usage_by_activity('mod_quiz', $quizobj->get_context());
 206          $quba->set_preferred_behaviour($quizobj->get_quiz()->preferredbehaviour);
 207          $timenow = time();
 208          $attempt = quiz_create_attempt($quizobj, 1, null, $timenow, false, $user->id);
 209          quiz_start_new_attempt($quizobj, $quba, $attempt, 1, $timenow);
 210          quiz_attempt_save_started($quizobj, $quba, $attempt);
 211          // Submit attempt.
 212          $attemptobj = quiz_attempt::create($attempt->id);
 213          $attemptobj->process_submitted_actions($timenow, false, $answers);
 214          $attemptobj->process_finish($timenow, false);
 215      }
 216  
 217      /**
 218       * Generate attempt answers.
 219       *
 220       * @param array $correctanswerflags array of 1 or 0
 221       * 1 : generate correct answer
 222       * 0 : generate wrong answer
 223       *
 224       * @return array
 225       */
 226      private function generate_attempt_answers(array $correctanswerflags): array {
 227          $attempt = [];
 228          for ($i = 1; $i <= 4; $i++) {
 229              if (isset($correctanswerflags) && $correctanswerflags[$i - 1] == 1) {
 230                  // Correct answer.
 231                  $attempt[$i] = ['answer' => 'frog'];
 232              } else {
 233                  $attempt[$i] = ['answer' => 'false'];
 234              }
 235          }
 236          return $attempt;
 237      }
 238  
 239      /**
 240       * Generate quizzes and submit answers.
 241       *
 242       * @param array $quiz1attempts quiz 1 attempts
 243       * @param array $quiz2attempts quiz 2 attempts
 244       *
 245       * @return array
 246       */
 247      private function prepare_and_submit_quizzes(array $quiz1attempts, array $quiz2attempts): array {
 248          list($quiz1, $quiz2, $questions) = $this->prepare_quizzes();
 249          // Submit attempts of quiz1.
 250          foreach ($quiz1attempts as $attempt) {
 251              $this->submit_quiz($quiz1, $attempt);
 252          }
 253          // Submit attempts of quiz2.
 254          foreach ($quiz2attempts as $attempt) {
 255              $this->submit_quiz($quiz2, $attempt);
 256          }
 257  
 258          // Calculate the statistics.
 259          $this->expectOutputRegex('~.*Calculations completed.*~');
 260          statistics_helper::run_pending_recalculation_tasks();
 261  
 262          return [$quiz1, $quiz2, $questions];
 263      }
 264  
 265      /**
 266       * To use private helper::extract_item_value function.
 267       *
 268       * @param all_calculated_for_qubaid_condition $statistics the batch of statistics.
 269       * @param int $questionid a question id.
 270       * @param string $item one of the field names in all_calculated_for_qubaid_condition, e.g. 'facility'.
 271       * @return float|null the required value.
 272       */
 273      private function extract_item_value(all_calculated_for_qubaid_condition $statistics,
 274                                          int $questionid, string $item): ?float {
 275          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'extract_item_value');
 276          $rcm->setAccessible(true);
 277          return $rcm->invoke(null, $statistics, $questionid, $item);
 278      }
 279  
 280      /**
 281       * To use private helper::load_statistics_for_place function (with mod_quiz component).
 282       *
 283       * @param context $context the context to load the statistics for.
 284       * @return all_calculated_for_qubaid_condition|null question statistics.
 285       */
 286      private function load_quiz_statistics_for_place(context $context): ?all_calculated_for_qubaid_condition {
 287          $rcm = new ReflectionMethod(statistics_bulk_loader::class, 'load_statistics_for_place');
 288          $rcm->setAccessible(true);
 289          return $rcm->invoke(null, 'mod_quiz', $context);
 290      }
 291  
 292      /**
 293       * Data provider for {@see test_load_question_facility()}.
 294       *
 295       * @return Generator
 296       */
 297      public function load_question_facility_provider(): Generator {
 298          yield 'Facility case 1' => [
 299              'Quiz 1 attempts' => [
 300                  $this->generate_attempt_answers([1, 0, 0, 0]),
 301              ],
 302              'Expected quiz 1 facilities' => [1.0, 0.0, 0.0, 0.0],
 303              'Quiz 2 attempts' => [
 304                  $this->generate_attempt_answers([1, 0, 0, 0]),
 305                  $this->generate_attempt_answers([1, 1, 0, 0]),
 306              ],
 307              'Expected quiz 2 facilities' => [1.0, 0.5, 0.0, 0.0],
 308              'Expected average facilities' => [1.0, 0.25, 0.0, 0.0],
 309          ];
 310          yield 'Facility case 2' => [
 311              'Quiz 1 attempts' => [
 312                  $this->generate_attempt_answers([1, 0, 0, 0]),
 313                  $this->generate_attempt_answers([1, 1, 0, 0]),
 314                  $this->generate_attempt_answers([1, 1, 1, 0]),
 315              ],
 316              'Expected quiz 1 facilities' => [1.0, 0.6667, 0.3333, 0.0],
 317              'Quiz 2 attempts' => [
 318                  $this->generate_attempt_answers([1, 0, 0, 0]),
 319                  $this->generate_attempt_answers([1, 1, 0, 0]),
 320                  $this->generate_attempt_answers([1, 1, 1, 0]),
 321                  $this->generate_attempt_answers([1, 1, 1, 1]),
 322              ],
 323              'Expected quiz 2 facilities' => [1.0, 0.75, 0.5, 0.25],
 324              'Expected average facilities' => [1.0, 0.7083, 0.4167, 0.1250],
 325          ];
 326      }
 327  
 328      /**
 329       * Test question facility
 330       *
 331       * @dataProvider load_question_facility_provider
 332       *
 333       * @param array $quiz1attempts quiz 1 attempts
 334       * @param array $expectedquiz1facilities expected quiz 1 facilities
 335       * @param array $quiz2attempts quiz 2 attempts
 336       * @param array $expectedquiz2facilities  expected quiz 2 facilities
 337       * @param array $expectedaveragefacilities expected average facilities
 338       */
 339      public function test_load_question_facility(
 340          array $quiz1attempts,
 341          array $expectedquiz1facilities,
 342          array $quiz2attempts,
 343          array $expectedquiz2facilities,
 344          array $expectedaveragefacilities)
 345      : void {
 346          $this->resetAfterTest();
 347  
 348          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 349  
 350          // Quiz 1 facilities.
 351          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 352          $quiz1facility1 = $this->extract_item_value($stats, $questions[1]->id, 'facility');
 353          $quiz1facility2 = $this->extract_item_value($stats, $questions[2]->id, 'facility');
 354          $quiz1facility3 = $this->extract_item_value($stats, $questions[3]->id, 'facility');
 355          $quiz1facility4 = $this->extract_item_value($stats, $questions[4]->id, 'facility');
 356  
 357          $this->assertEqualsWithDelta($expectedquiz1facilities[0], $quiz1facility1, self::DELTA);
 358          $this->assertEqualsWithDelta($expectedquiz1facilities[1], $quiz1facility2, self::DELTA);
 359          $this->assertEqualsWithDelta($expectedquiz1facilities[2], $quiz1facility3, self::DELTA);
 360          $this->assertEqualsWithDelta($expectedquiz1facilities[3], $quiz1facility4, self::DELTA);
 361  
 362          // Quiz 2 facilities.
 363          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 364          $quiz2facility1 = $this->extract_item_value($stats, $questions[1]->id, 'facility');
 365          $quiz2facility2 = $this->extract_item_value($stats, $questions[2]->id, 'facility');
 366          $quiz2facility3 = $this->extract_item_value($stats, $questions[3]->id, 'facility');
 367          $quiz2facility4 = $this->extract_item_value($stats, $questions[4]->id, 'facility');
 368  
 369          $this->assertEqualsWithDelta($expectedquiz2facilities[0], $quiz2facility1, self::DELTA);
 370          $this->assertEqualsWithDelta($expectedquiz2facilities[1], $quiz2facility2, self::DELTA);
 371          $this->assertEqualsWithDelta($expectedquiz2facilities[2], $quiz2facility3, self::DELTA);
 372          $this->assertEqualsWithDelta($expectedquiz2facilities[3], $quiz2facility4, self::DELTA);
 373  
 374          // Average question facilities.
 375          $stats = statistics_bulk_loader::load_aggregate_statistics(
 376              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 377              ['facility']
 378          );
 379  
 380          $this->assertEqualsWithDelta($expectedaveragefacilities[0],
 381              $stats[$questions[1]->id]['facility'], self::DELTA);
 382          $this->assertEqualsWithDelta($expectedaveragefacilities[1],
 383              $stats[$questions[2]->id]['facility'], self::DELTA);
 384          $this->assertEqualsWithDelta($expectedaveragefacilities[2],
 385              $stats[$questions[3]->id]['facility'], self::DELTA);
 386          $this->assertEqualsWithDelta($expectedaveragefacilities[3],
 387              $stats[$questions[4]->id]['facility'], self::DELTA);
 388      }
 389  
 390      /**
 391       * Data provider for {@see test_load_question_discriminative_efficiency()}.
 392       * @return Generator
 393       */
 394      public function load_question_discriminative_efficiency_provider(): Generator {
 395          yield 'Discriminative efficiency' => [
 396              'Quiz 1 attempts' => [
 397                  $this->generate_attempt_answers([1, 0, 0, 0]),
 398                  $this->generate_attempt_answers([1, 1, 0, 0]),
 399                  $this->generate_attempt_answers([1, 0, 1, 0]),
 400                  $this->generate_attempt_answers([1, 1, 1, 1]),
 401              ],
 402              'Expected quiz 1 discriminative efficiency' => [null, 33.33, 33.33, 100.00],
 403              'Quiz 2 attempts' => [
 404                  $this->generate_attempt_answers([1, 1, 1, 1]),
 405                  $this->generate_attempt_answers([0, 0, 0, 0]),
 406                  $this->generate_attempt_answers([1, 0, 0, 1]),
 407                  $this->generate_attempt_answers([0, 1, 1, 0]),
 408              ],
 409              'Expected quiz 2 discriminative efficiency' => [50.00, 50.00, 50.00, 50.00],
 410              'Expected average discriminative efficiency' => [50.00, 41.67, 41.67, 75.00],
 411          ];
 412      }
 413  
 414      /**
 415       * Test discriminative efficiency
 416       *
 417       * @dataProvider load_question_discriminative_efficiency_provider
 418       *
 419       * @param array $quiz1attempts quiz 1 attempts
 420       * @param array $expectedquiz1discriminativeefficiency expected quiz 1 discriminative efficiency
 421       * @param array $quiz2attempts quiz 2 attempts
 422       * @param array $expectedquiz2discriminativeefficiency expected quiz 2 discriminative efficiency
 423       * @param array $expectedaveragediscriminativeefficiency expected average discriminative efficiency
 424       */
 425      public function test_load_question_discriminative_efficiency(
 426          array $quiz1attempts,
 427          array $expectedquiz1discriminativeefficiency,
 428          array $quiz2attempts,
 429          array $expectedquiz2discriminativeefficiency,
 430          array $expectedaveragediscriminativeefficiency
 431      ): void {
 432          $this->resetAfterTest();
 433  
 434          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 435  
 436          // Quiz 1 discriminative efficiency.
 437          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 438          $discriminativeefficiency1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminativeefficiency');
 439          $discriminativeefficiency2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminativeefficiency');
 440          $discriminativeefficiency3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminativeefficiency');
 441          $discriminativeefficiency4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminativeefficiency');
 442  
 443          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[0],
 444                  $discriminativeefficiency1, self::PERCENT_DELTA);
 445          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[1],
 446                  $discriminativeefficiency2, self::PERCENT_DELTA);
 447          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[2],
 448                  $discriminativeefficiency3, self::PERCENT_DELTA);
 449          $this->assertEqualsWithDelta($expectedquiz1discriminativeefficiency[3],
 450                  $discriminativeefficiency4, self::PERCENT_DELTA);
 451  
 452          // Quiz 2 discriminative efficiency.
 453          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 454          $discriminativeefficiency1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminativeefficiency');
 455          $discriminativeefficiency2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminativeefficiency');
 456          $discriminativeefficiency3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminativeefficiency');
 457          $discriminativeefficiency4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminativeefficiency');
 458  
 459          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[0],
 460                  $discriminativeefficiency1, self::PERCENT_DELTA);
 461          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[1],
 462                  $discriminativeefficiency2, self::PERCENT_DELTA);
 463          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[2],
 464                  $discriminativeefficiency3, self::PERCENT_DELTA);
 465          $this->assertEqualsWithDelta($expectedquiz2discriminativeefficiency[3],
 466                  $discriminativeefficiency4, self::PERCENT_DELTA);
 467  
 468          // Average question discriminative efficiency.
 469          $stats = statistics_bulk_loader::load_aggregate_statistics(
 470              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 471              ['discriminativeefficiency']
 472          );
 473  
 474          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[0],
 475              $stats[$questions[1]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 476          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[1],
 477              $stats[$questions[2]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 478          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[2],
 479              $stats[$questions[3]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 480          $this->assertEqualsWithDelta($expectedaveragediscriminativeefficiency[3],
 481              $stats[$questions[4]->id]['discriminativeefficiency'], self::PERCENT_DELTA);
 482      }
 483  
 484      /**
 485       * Data provider for {@see test_load_question_discrimination_index()}.
 486       * @return Generator
 487       */
 488      public function load_question_discrimination_index_provider(): Generator {
 489          yield 'Discrimination Index' => [
 490              'Quiz 1 attempts' => [
 491                  $this->generate_attempt_answers([1, 0, 0, 0]),
 492                  $this->generate_attempt_answers([1, 1, 0, 0]),
 493                  $this->generate_attempt_answers([1, 0, 1, 0]),
 494                  $this->generate_attempt_answers([1, 1, 1, 1]),
 495              ],
 496              'Expected quiz 1 Discrimination Index' => [null, 30.15, 30.15, 81.65],
 497              'Quiz 2 attempts' => [
 498                  $this->generate_attempt_answers([1, 1, 1, 1]),
 499                  $this->generate_attempt_answers([0, 0, 0, 0]),
 500                  $this->generate_attempt_answers([1, 0, 0, 1]),
 501                  $this->generate_attempt_answers([0, 1, 1, 0]),
 502              ],
 503              'Expected quiz 2 discrimination Index' => [44.72, 44.72, 44.72, 44.72],
 504              'Expected average discrimination Index' => [44.72, 37.44, 37.44, 63.19],
 505          ];
 506      }
 507  
 508      /**
 509       * Test discrimination index
 510       *
 511       * @dataProvider load_question_discrimination_index_provider
 512       *
 513       * @param array $quiz1attempts quiz 1 attempts
 514       * @param array $expectedquiz1discriminationindex expected quiz 1 discrimination index
 515       * @param array $quiz2attempts quiz 2 attempts
 516       * @param array $expectedquiz2discriminationindex expected quiz 2 discrimination index
 517       * @param array $expectedaveragediscriminationindex expected average discrimination index
 518       */
 519      public function test_load_question_discrimination_index(
 520          array $quiz1attempts,
 521          array $expectedquiz1discriminationindex,
 522          array $quiz2attempts,
 523          array $expectedquiz2discriminationindex,
 524          array $expectedaveragediscriminationindex
 525      ): void {
 526          $this->resetAfterTest();
 527  
 528          list($quiz1, $quiz2, $questions) = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 529  
 530          // Quiz 1 discrimination index.
 531          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz1->cmid));
 532          $discriminationindex1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminationindex');
 533          $discriminationindex2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminationindex');
 534          $discriminationindex3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminationindex');
 535          $discriminationindex4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminationindex');
 536  
 537          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[0],
 538              $discriminationindex1, self::PERCENT_DELTA);
 539          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[1],
 540              $discriminationindex2, self::PERCENT_DELTA);
 541          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[2],
 542              $discriminationindex3, self::PERCENT_DELTA);
 543          $this->assertEqualsWithDelta($expectedquiz1discriminationindex[3],
 544              $discriminationindex4, self::PERCENT_DELTA);
 545  
 546          // Quiz 2 discrimination index.
 547          $stats = $this->load_quiz_statistics_for_place(context_module::instance($quiz2->cmid));
 548          $discriminationindex1 = $this->extract_item_value($stats, $questions[1]->id, 'discriminationindex');
 549          $discriminationindex2 = $this->extract_item_value($stats, $questions[2]->id, 'discriminationindex');
 550          $discriminationindex3 = $this->extract_item_value($stats, $questions[3]->id, 'discriminationindex');
 551          $discriminationindex4 = $this->extract_item_value($stats, $questions[4]->id, 'discriminationindex');
 552  
 553          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[0],
 554              $discriminationindex1, self::PERCENT_DELTA);
 555          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[1],
 556              $discriminationindex2, self::PERCENT_DELTA);
 557          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[2],
 558              $discriminationindex3, self::PERCENT_DELTA);
 559          $this->assertEqualsWithDelta($expectedquiz2discriminationindex[3],
 560              $discriminationindex4, self::PERCENT_DELTA);
 561  
 562          // Average question discrimination index.
 563          $stats = statistics_bulk_loader::load_aggregate_statistics(
 564              [$questions[1]->id, $questions[2]->id, $questions[3]->id, $questions[4]->id],
 565              ['discriminationindex']
 566          );
 567  
 568          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[0],
 569              $stats[$questions[1]->id]['discriminationindex'], self::PERCENT_DELTA);
 570          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[1],
 571              $stats[$questions[2]->id]['discriminationindex'], self::PERCENT_DELTA);
 572          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[2],
 573              $stats[$questions[3]->id]['discriminationindex'], self::PERCENT_DELTA);
 574          $this->assertEqualsWithDelta($expectedaveragediscriminationindex[3],
 575              $stats[$questions[4]->id]['discriminationindex'], self::PERCENT_DELTA);
 576      }
 577  
 578      /**
 579       * Test with question statistics disabled
 580       */
 581      public function test_statistics_disabled(): void {
 582          $this->resetAfterTest();
 583  
 584          // Prepare some quizzes and attempts. Exactly what is not important to this test.
 585          $quiz1attempts = [$this->generate_attempt_answers([1, 0, 0, 0])];
 586          $quiz2attempts = [$this->generate_attempt_answers([1, 1, 1, 1])];
 587          [, , $questions] = $this->prepare_and_submit_quizzes($quiz1attempts, $quiz2attempts);
 588  
 589          // Prepare some useful arrays.
 590          $expectedstats = [
 591              $questions[1]->id => [],
 592              $questions[2]->id => [],
 593              $questions[3]->id => [],
 594              $questions[4]->id => [],
 595          ];
 596          $questionids = array_keys($expectedstats);
 597  
 598          // Ask to load no statistics at all.
 599          $stats = statistics_bulk_loader::load_aggregate_statistics($questionids, []);
 600  
 601          // Verify we got the right thing.
 602          $this->assertEquals($expectedstats, $stats);
 603      }
 604  }