[Summary view]

Differences Between: [Versions 310 and 402] [Versions 311 and 402] [Versions 39 and 402] [Versions 400 and 402] [Versions 401 and 402]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  namespace quiz_statistics;
  18  
  19  use question_attempt;
  20  use question_bank;
  21  use question_finder;
  22  use quiz_statistics_report;
  23  
  24  defined('MOODLE_INTERNAL') || die();
  25  
  26  global $CFG;
  27  require_once($CFG->dirroot . '/mod/quiz/tests/attempt_walkthrough_from_csv_test.php');
  28  require_once($CFG->dirroot . '/mod/quiz/report/statistics/report.php');
  29  require_once($CFG->dirroot . '/mod/quiz/report/reportlib.php');
  30  
  31  /**
  32   * Quiz attempt walk through using data from csv file.
  33   *
  34   * The quiz stats below and the question stats found in qstats00.csv were calculated independently in a spreadsheet which is
  35   * available in open document or excel format here :
  36   * https://github.com/jamiepratt/moodle-quiz-tools/tree/master/statsspreadsheet
  37   *
  38   * Similarly the question variant's stats in qstats00.csv are calculated in stats_for_variant_1.xls and stats_for_variant_8.xls
  39   * The calculations in the spreadsheets are the same as for the other question stats but applied just to the attempts where the
  40   * variants appeared.
  41   *
  42   * @package    quiz_statistics
  43   * @category   test
  44   * @copyright  2013 The Open University
  45   * @author     Jamie Pratt <me@jamiep.org>
  46   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  47   */
  48  class stats_from_steps_walkthrough_test extends \mod_quiz\attempt_walkthrough_from_csv_test {
  49  
  50      /**
  51       * @var quiz_statistics_report object to do stats calculations.
  52       */
  53      protected $report;
  54  
  55      protected function get_full_path_of_csv_file(string $setname, string $test): string {
  56          // Overridden here so that __DIR__ points to the path of this file.
  57          return  __DIR__."/fixtures/{$setname}{$test}.csv";
  58      }
  59  
  60      /**
  61       * @var string[] names of the files which contain the test data.
  62       */
  63      protected $files = ['questions', 'steps', 'results', 'qstats', 'responsecounts'];
  64  
  65      /**
  66       * Create a quiz add questions to it, walk through quiz attempts and then check results.
  67       *
  68       * @param array $csvdata data read from csv file "questionsXX.csv", "stepsXX.csv" and "resultsXX.csv".
  69       * @dataProvider get_data_for_walkthrough
  70       */
  71      public function test_walkthrough_from_csv($quizsettings, $csvdata) {
  72  
  73          $this->create_quiz_simulate_attempts_and_check_results($quizsettings, $csvdata);
  74  
  75          $whichattempts = QUIZ_GRADEAVERAGE; // All attempts.
  76          $whichtries = question_attempt::ALL_TRIES;
  77          $groupstudentsjoins = new \core\dml\sql_join();
  78          list($questions, $quizstats, $questionstats, $qubaids) =
  79                      $this->check_stats_calculations_and_response_analysis($csvdata,
  80                              $whichattempts, $whichtries, $groupstudentsjoins);
  81          if ($quizsettings['testnumber'] === '00') {
  82              $this->check_variants_count_for_quiz_00($questions, $questionstats, $whichtries, $qubaids);
  83              $this->check_quiz_stats_for_quiz_00($quizstats);
  84          }
  85      }
  86  
  87      /**
  88       * Check actual question stats are the same as that found in csv file.
  89       *
  90       * @param $qstats         array data from csv file.
  91       * @param $questionstats  \core_question\statistics\questions\all_calculated_for_qubaid_condition Calculated stats.
  92       */
  93      protected function check_question_stats($qstats, $questionstats) {
  94          foreach ($qstats as $slotqstats) {
  95              foreach ($slotqstats as $statname => $slotqstat) {
  96                  if (!in_array($statname, ['slot', 'subqname'])  && $slotqstat !== '') {
  97                      $this->assert_stat_equals($slotqstat,
  98                                                $questionstats,
  99                                                $slotqstats['slot'],
 100                                                $slotqstats['subqname'],
 101                                                $slotqstats['variant'],
 102                                                $statname);
 103                  }
 104              }
 105              // Check that sub-question boolean field is correctly set.
 106              $this->assert_stat_equals(!empty($slotqstats['subqname']),
 107                                        $questionstats,
 108                                        $slotqstats['slot'],
 109                                        $slotqstats['subqname'],
 110                                        $slotqstats['variant'],
 111                                        'subquestion');
 112          }
 113      }
 114  
 115      /**
 116       * Check that the stat is as expected within a reasonable tolerance.
 117       *
 118       * @param float|string|bool $expected expected value of stat.
 119       * @param \core_question\statistics\questions\all_calculated_for_qubaid_condition $questionstats
 120       * @param int $slot
 121       * @param string $subqname if empty string then not an item stat.
 122       * @param int|string $variant if empty string then not a variantstat.
 123       * @param string $statname
 124       */
 125      protected function assert_stat_equals($expected, $questionstats, $slot, $subqname, $variant, $statname) {
 126  
 127          if ($variant === '' && $subqname === '') {
 128              $actual = $questionstats->for_slot($slot)->{$statname};
 129          } else if ($subqname !== '') {
 130              $actual = $questionstats->for_subq($this->randqids[$slot][$subqname])->{$statname};
 131          } else {
 132              $actual = $questionstats->for_slot($slot, $variant)->{$statname};
 133          }
 134          $message = "$statname for slot $slot";
 135          if ($expected === '**NULL**') {
 136              $this->assertEquals(null, $actual, $message);
 137          } else if (is_bool($expected)) {
 138              $this->assertEquals($expected, $actual, $message);
 139          } else if (is_numeric($expected)) {
 140              switch ($statname) {
 141                  case 'covariance' :
 142                  case 'discriminationindex' :
 143                  case 'discriminativeefficiency' :
 144                  case 'effectiveweight' :
 145                      $precision = 1e-5;
 146                      break;
 147                  default :
 148                      $precision = 1e-6;
 149              }
 150              $delta = abs($expected) * $precision;
 151              $this->assertEqualsWithDelta((float)$expected, $actual, $delta, $message);
 152          } else {
 153              $this->assertEquals($expected, $actual, $message);
 154          }
 155      }
 156  
 157      protected function assert_response_count_equals($question, $qubaids, $expected, $whichtries) {
 158          $responesstats = new \core_question\statistics\responses\analyser($question);
 159          $analysis = $responesstats->load_cached($qubaids, $whichtries);
 160          if (!isset($expected['subpart'])) {
 161              $subpart = 1;
 162          } else {
 163              $subpart = $expected['subpart'];
 164          }
 165          list($subpartid, $responseclassid) = $this->get_response_subpart_and_class_id($question,
 166                                                                                        $subpart,
 167                                                                                        $expected['modelresponse']);
 168  
 169          $subpartanalysis = $analysis->get_analysis_for_subpart($expected['variant'], $subpartid);
 170          $responseclassanalysis = $subpartanalysis->get_response_class($responseclassid);
 171          $actualresponsecounts = $responseclassanalysis->data_for_question_response_table('', '');
 172  
 173          foreach ($actualresponsecounts as $actualresponsecount) {
 174              if ($actualresponsecount->response == $expected['actualresponse'] || count($actualresponsecounts) == 1) {
 175                  $i = 1;
 176                  $partofanalysis = " slot {$expected['slot']}, rand q '{$expected['randq']}', variant {$expected['variant']}, ".
 177                                      "for expected model response {$expected['modelresponse']}, ".
 178                                      "actual response {$expected['actualresponse']}";
 179                  while (isset($expected['count'.$i])) {
 180                      if ($expected['count'.$i] != 0) {
 181                          $this->assertTrue(isset($actualresponsecount->trycount[$i]),
 182                              "There is no count at all for try $i on ".$partofanalysis);
 183                          $this->assertEquals($expected['count'.$i], $actualresponsecount->trycount[$i],
 184                                              "Count for try $i on ".$partofanalysis);
 185                      }
 186                      $i++;
 187                  }
 188                  if (isset($expected['totalcount'])) {
 189                      $this->assertEquals($expected['totalcount'], $actualresponsecount->totalcount,
 190                                          "Total count on ".$partofanalysis);
 191                  }
 192                  return;
 193              }
 194          }
 195          throw new \coding_exception("Expected response '{$expected['actualresponse']}' not found.");
 196      }
 197  
 198      protected function get_response_subpart_and_class_id($question, $subpart, $modelresponse) {
 199          $qtypeobj = question_bank::get_qtype($question->qtype, false);
 200          $possibleresponses = $qtypeobj->get_possible_responses($question);
 201          $possibleresponsesubpartids = array_keys($possibleresponses);
 202          if (!isset($possibleresponsesubpartids[$subpart - 1])) {
 203              throw new \coding_exception("Subpart '{$subpart}' not found.");
 204          }
 205          $subpartid = $possibleresponsesubpartids[$subpart - 1];
 206  
 207          if ($modelresponse == '[NO RESPONSE]') {
 208              return [$subpartid, null];
 209  
 210          } else if ($modelresponse == '[NO MATCH]') {
 211              return [$subpartid, 0];
 212          }
 213  
 214          $modelresponses = [];
 215          foreach ($possibleresponses[$subpartid] as $responseclassid => $subpartpossibleresponse) {
 216              $modelresponses[$responseclassid] = $subpartpossibleresponse->responseclass;
 217          }
 218          $this->assertContains($modelresponse, $modelresponses);
 219          $responseclassid = array_search($modelresponse, $modelresponses);
 220          return [$subpartid, $responseclassid];
 221      }
 222  
 223      /**
 224       * @param $responsecounts
 225       * @param $qubaids
 226       * @param $questions
 227       * @param $whichtries
 228       */
 229      protected function check_response_counts($responsecounts, $qubaids, $questions, $whichtries) {
 230          foreach ($responsecounts as $expected) {
 231              $defaultsforexpected = ['randq' => '', 'variant' => '1', 'subpart' => '1'];
 232              foreach ($defaultsforexpected as $key => $expecteddefault) {
 233                  if (!isset($expected[$key])) {
 234                      $expected[$key] = $expecteddefault;
 235                  }
 236              }
 237              if ($expected['randq'] == '') {
 238                  $question = $questions[$expected['slot']];
 239              } else {
 240                  $qid = $this->randqids[$expected['slot']][$expected['randq']];
 241                  $question = question_finder::get_instance()->load_question_data($qid);
 242              }
 243              $this->assert_response_count_equals($question, $qubaids, $expected, $whichtries);
 244          }
 245      }
 246  
 247      /**
 248       * @param $questions
 249       * @param $questionstats
 250       * @param $whichtries
 251       * @param $qubaids
 252       */
 253      protected function check_variants_count_for_quiz_00($questions, $questionstats, $whichtries, $qubaids) {
 254          $expectedvariantcounts = [2 => [1  => 6,
 255                                                    4  => 4,
 256                                                    5  => 3,
 257                                                    6  => 4,
 258                                                    7  => 2,
 259                                                    8  => 5,
 260                                                    10 => 1]];
 261  
 262          foreach ($questions as $slot => $question) {
 263              if (!question_bank::get_qtype($question->qtype, false)->can_analyse_responses()) {
 264                  continue;
 265              }
 266              $responesstats = new \core_question\statistics\responses\analyser($question);
 267              $this->assertTimeCurrent($responesstats->get_last_analysed_time($qubaids, $whichtries));
 268              $analysis = $responesstats->load_cached($qubaids, $whichtries);
 269              $variantsnos = $analysis->get_variant_nos();
 270              if (isset($expectedvariantcounts[$slot])) {
 271                  // Compare contents, ignore ordering of array, using canonicalize parameter of assertEquals.
 272                  $this->assertEqualsCanonicalizing(array_keys($expectedvariantcounts[$slot]), $variantsnos);
 273              } else {
 274                  $this->assertEquals([1], $variantsnos);
 275              }
 276              $totalspervariantno = [];
 277              foreach ($variantsnos as $variantno) {
 278  
 279                  $subpartids = $analysis->get_subpart_ids($variantno);
 280                  foreach ($subpartids as $subpartid) {
 281                      if (!isset($totalspervariantno[$subpartid])) {
 282                          $totalspervariantno[$subpartid] = [];
 283                      }
 284                      $totalspervariantno[$subpartid][$variantno] = 0;
 285  
 286                      $subpartanalysis = $analysis->get_analysis_for_subpart($variantno, $subpartid);
 287                      $classids = $subpartanalysis->get_response_class_ids();
 288                      foreach ($classids as $classid) {
 289                          $classanalysis = $subpartanalysis->get_response_class($classid);
 290                          $actualresponsecounts = $classanalysis->data_for_question_response_table('', '');
 291                          foreach ($actualresponsecounts as $actualresponsecount) {
 292                              $totalspervariantno[$subpartid][$variantno] += $actualresponsecount->totalcount;
 293                          }
 294                      }
 295                  }
 296              }
 297              // Count all counted responses for each part of question and confirm that counted responses, for most question types
 298              // are the number of attempts at the question for each question part.
 299              if ($slot != 5) {
 300                  // Slot 5 holds a multi-choice multiple question.
 301                  // Multi-choice multiple is slightly strange. Actual answer counts given for each sub part do not add up to the
 302                  // total attempt count.
 303                  // This is because each option is counted as a sub part and each option can be off or on in each attempt. Off is
 304                  // not counted in response analysis for this question type.
 305                  foreach ($totalspervariantno as $totalpervariantno) {
 306                      if (isset($expectedvariantcounts[$slot])) {
 307                          // If we know how many attempts there are at each variant we can check
 308                          // that we have counted the correct amount of responses for each variant.
 309                          $this->assertEqualsCanonicalizing($expectedvariantcounts[$slot],
 310                                              $totalpervariantno,
 311                                              "Totals responses do not add up in response analysis for slot {$slot}.");
 312                      } else {
 313                          $this->assertEquals(25,
 314                                              array_sum($totalpervariantno),
 315                                              "Totals responses do not add up in response analysis for slot {$slot}.");
 316                      }
 317                  }
 318              }
 319          }
 320  
 321          foreach ($expectedvariantcounts as $slot => $expectedvariantcount) {
 322              foreach ($expectedvariantcount as $variantno => $s) {
 323                  $this->assertEquals($s, $questionstats->for_slot($slot, $variantno)->s);
 324              }
 325          }
 326      }
 327  
 328      /**
 329       * @param $quizstats
 330       */
 331      protected function check_quiz_stats_for_quiz_00($quizstats) {
 332          $quizstatsexpected = [
 333              'median'             => 4.5,
 334              'firstattemptsavg'   => 4.617333332,
 335              'allattemptsavg'     => 4.617333332,
 336              'firstattemptscount' => 25,
 337              'allattemptscount'   => 25,
 338              'standarddeviation'  => 0.8117265554,
 339              'skewness'           => -0.092502502,
 340              'kurtosis'           => -0.7073968557,
 341              'cic'                => -87.2230935542,
 342              'errorratio'         => 136.8294900795,
 343              'standarderror'      => 1.1106813066
 344          ];
 345  
 346          foreach ($quizstatsexpected as $statname => $statvalue) {
 347              $this->assertEqualsWithDelta($statvalue, $quizstats->$statname, abs($statvalue) * 1.5e-5, $quizstats->$statname);
 348          }
 349      }
 350  
 351      /**
 352       * Check the question stats and the response counts used in the statistics report. If the appropriate files exist in fixtures/.
 353       *
 354       * @param array $csvdata Data loaded from csv files for this test.
 355       * @param string $whichattempts
 356       * @param string $whichtries
 357       * @param \core\dml\sql_join $groupstudentsjoins
 358       * @return array with contents 0 => $questions, 1 => $quizstats, 2 => $questionstats, 3 => $qubaids Might be needed for further
 359       *               testing.
 360       */
 361      protected function check_stats_calculations_and_response_analysis($csvdata, $whichattempts, $whichtries,
 362              \core\dml\sql_join $groupstudentsjoins) {
 363          $this->report = new quiz_statistics_report();
 364          $questions = $this->report->load_and_initialise_questions_for_calculations($this->quiz);
 365          list($quizstats, $questionstats) = $this->report->get_all_stats_and_analysis($this->quiz,
 366                                                                                       $whichattempts,
 367                                                                                       $whichtries,
 368                                                                                       $groupstudentsjoins,
 369                                                                                       $questions);
 370  
 371          $qubaids = quiz_statistics_qubaids_condition($this->quiz->id, $groupstudentsjoins, $whichattempts);
 372  
 373          // We will create some quiz and question stat calculator instances and some response analyser instances, just in order
 374          // to check the last analysed time then returned.
 375          $quizcalc = new calculator();
 376          // Should not be a delay of more than one second between the calculation of stats above and here.
 377          $this->assertTimeCurrent($quizcalc->get_last_calculated_time($qubaids));
 378  
 379          $qcalc = new \core_question\statistics\questions\calculator($questions);
 380          $this->assertTimeCurrent($qcalc->get_last_calculated_time($qubaids));
 381  
 382          if (isset($csvdata['responsecounts'])) {
 383              $this->check_response_counts($csvdata['responsecounts'], $qubaids, $questions, $whichtries);
 384          }
 385          if (isset($csvdata['qstats'])) {
 386              $this->check_question_stats($csvdata['qstats'], $questionstats);
 387              return [$questions, $quizstats, $questionstats, $qubaids];
 388          }
 389          return [$questions, $quizstats, $questionstats, $qubaids];
 390      }
 391  
 392  }