[Summary view]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  namespace mod_data;
  18  
  19  use coding_exception;
  20  use dml_exception;
  21  use mod_data\local\importer\csv_entries_importer;
  22  use moodle_exception;
  23  use zip_archive;
  24  
  25  /**
  26   * Unit tests for import.php.
  27   *
  28   * @package    mod_data
  29   * @category   test
  30   * @copyright  2019 Tobias Reischmann
  31   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  32   */
  33  class entries_import_test extends \advanced_testcase {
  34  
  35      /**
  36       * Set up function.
  37       */
  38      protected function setUp(): void {
  39          parent::setUp();
  40  
  41          global $CFG;
  42          require_once($CFG->dirroot . '/mod/data/lib.php');
  43          require_once($CFG->dirroot . '/lib/datalib.php');
  44          require_once($CFG->dirroot . '/lib/csvlib.class.php');
  45          require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
  46          require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
  47      }
  48  
  49      /**
  50       * Get the test data.
  51       * In this instance we are setting up database records to be used in the unit tests.
  52       *
  53       * @return array
  54       */
  55      protected function get_test_data(): array {
  56          $this->resetAfterTest(true);
  57  
  58          $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
  59          $course = $this->getDataGenerator()->create_course();
  60          $teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
  61          $this->setUser($teacher);
  62          $student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
  63  
  64          $data = $generator->create_instance(array('course' => $course->id));
  65          $cm = get_coursemodule_from_instance('data', $data->id);
  66  
  67          // Add fields.
  68          $fieldrecord = new \stdClass();
  69          $fieldrecord->name = 'ID'; // Identifier of the records for testing.
  70          $fieldrecord->type = 'number';
  71          $generator->create_field($fieldrecord, $data);
  72  
  73          $fieldrecord->name = 'Param2';
  74          $fieldrecord->type = 'text';
  75          $generator->create_field($fieldrecord, $data);
  76  
  77          $fieldrecord->name = 'filefield';
  78          $fieldrecord->type = 'file';
  79          $generator->create_field($fieldrecord, $data);
  80  
  81          $fieldrecord->name = 'picturefield';
  82          $fieldrecord->type = 'picture';
  83          $generator->create_field($fieldrecord, $data);
  84  
  85          return [
  86              'teacher' => $teacher,
  87              'student' => $student,
  88              'data' => $data,
  89              'cm' => $cm,
  90          ];
  91      }
  92  
  93      /**
  94       * Test uploading entries for a data instance without userdata.
  95       *
  96       * @throws dml_exception
  97       */
  98      public function test_import(): void {
  99          [
 100              'data' => $data,
 101              'cm' => $cm,
 102              'teacher' => $teacher,
 103          ] = $this->get_test_data();
 104  
 105          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import.csv',
 106              'test_data_import.csv');
 107          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 108  
 109          // No userdata is present in the file: Fallback is to assign the uploading user as author.
 110          $expecteduserids = array();
 111          $expecteduserids[1] = $teacher->id;
 112          $expecteduserids[2] = $teacher->id;
 113  
 114          $records = $this->get_data_records($data->id);
 115          $this->assertCount(2, $records);
 116          foreach ($records as $record) {
 117              $identifier = $record->items['ID']->content;
 118              $this->assertEquals($expecteduserids[$identifier], $record->userid);
 119          }
 120      }
 121  
 122      /**
 123       * Test uploading entries for a data instance with userdata.
 124       *
 125       * At least one entry has an identifiable user, which is assigned as author.
 126       *
 127       * @throws dml_exception
 128       */
 129      public function test_import_with_userdata(): void {
 130          [
 131              'data' => $data,
 132              'cm' => $cm,
 133              'teacher' => $teacher,
 134              'student' => $student,
 135          ] = $this->get_test_data();
 136  
 137          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
 138              'test_data_import_with_userdata.csv');
 139          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 140  
 141          $expecteduserids = array();
 142          $expecteduserids[1] = $student->id; // User student exists and is assigned as author.
 143          $expecteduserids[2] = $teacher->id; // User student2 does not exist. Fallback is the uploading user.
 144  
 145          $records = $this->get_data_records($data->id);
 146          $this->assertCount(2, $records);
 147          foreach ($records as $record) {
 148              $identifier = $record->items['ID']->content;
 149              $this->assertEquals($expecteduserids[$identifier], $record->userid);
 150          }
 151      }
 152  
 153      /**
 154       * Test uploading entries for a data instance with userdata and a defined field 'Username'.
 155       *
 156       * This should test the corner case, in which a user has defined a data fields, which has the same name
 157       * as the current lang string for username. In that case, the first Username entry is used for the field.
 158       * The second one is used to identify the author.
 159       *
 160       * @throws coding_exception
 161       * @throws dml_exception
 162       */
 163      public function test_import_with_field_username(): void {
 164          [
 165              'data' => $data,
 166              'cm' => $cm,
 167              'teacher' => $teacher,
 168              'student' => $student,
 169          ] = $this->get_test_data();
 170          $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
 171  
 172          // Add username field.
 173          $fieldrecord = new \stdClass();
 174          $fieldrecord->name = 'Username';
 175          $fieldrecord->type = 'text';
 176          $generator->create_field($fieldrecord, $data);
 177  
 178          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_field_username.csv',
 179              'test_data_import_with_field_username.csv');
 180          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 181  
 182          $expecteduserids = array();
 183          $expecteduserids[1] = $student->id; // User student exists and is assigned as author.
 184          $expecteduserids[2] = $teacher->id; // User student2 does not exist. Fallback is the uploading user.
 185          $expecteduserids[3] = $student->id; // User student exists and is assigned as author.
 186  
 187          $expectedcontent = array();
 188          $expectedcontent[1] = array(
 189              'Username' => 'otherusername1',
 190              'Param2' => 'My first entry',
 191          );
 192          $expectedcontent[2] = array(
 193              'Username' => 'otherusername2',
 194              'Param2' => 'My second entry',
 195          );
 196          $expectedcontent[3] = array(
 197              'Username' => 'otherusername3',
 198              'Param2' => 'My third entry',
 199          );
 200  
 201          $records = $this->get_data_records($data->id);
 202          $this->assertCount(3, $records);
 203          foreach ($records as $record) {
 204              $identifier = $record->items['ID']->content;
 205              $this->assertEquals($expecteduserids[$identifier], $record->userid);
 206  
 207              foreach ($expectedcontent[$identifier] as $field => $value) {
 208                  $this->assertEquals($value, $record->items[$field]->content,
 209                      "The value of field \"$field\" for the record at position \"$identifier\" " .
 210                      "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
 211              }
 212          }
 213      }
 214  
 215      /**
 216       * Test uploading entries for a data instance with a field 'Username' but only one occurrence in the csv file.
 217       *
 218       * This should test the corner case, in which a user has defined a data fields, which has the same name
 219       * as the current lang string for username. In that case, the only Username entry is used for the field.
 220       * The author should not be set.
 221       *
 222       * @throws coding_exception
 223       * @throws dml_exception
 224       */
 225      public function test_import_with_field_username_without_userdata(): void {
 226          [
 227              'data' => $data,
 228              'cm' => $cm,
 229              'teacher' => $teacher,
 230              'student' => $student,
 231          ] = $this->get_test_data();
 232          $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
 233  
 234          // Add username field.
 235          $fieldrecord = new \stdClass();
 236          $fieldrecord->name = 'Username';
 237          $fieldrecord->type = 'text';
 238          $generator->create_field($fieldrecord, $data);
 239  
 240          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
 241              'test_data_import_with_userdata.csv');
 242          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 243  
 244          // No userdata is present in the file: Fallback is to assign the uploading user as author.
 245          $expecteduserids = array();
 246          $expecteduserids[1] = $teacher->id;
 247          $expecteduserids[2] = $teacher->id;
 248  
 249          $expectedcontent = array();
 250          $expectedcontent[1] = array(
 251              'Username' => 'student',
 252              'Param2' => 'My first entry',
 253          );
 254          $expectedcontent[2] = array(
 255              'Username' => 'student2',
 256              'Param2' => 'My second entry',
 257          );
 258  
 259          $records = $this->get_data_records($data->id);
 260          $this->assertCount(2, $records);
 261          foreach ($records as $record) {
 262              $identifier = $record->items['ID']->content;
 263              $this->assertEquals($expecteduserids[$identifier], $record->userid);
 264  
 265              foreach ($expectedcontent[$identifier] as $field => $value) {
 266                  $this->assertEquals($value, $record->items[$field]->content,
 267                      "The value of field \"$field\" for the record at position \"$identifier\" " .
 268                      "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
 269              }
 270          }
 271      }
 272  
 273      /**
 274       * Tests the import including files from a zip archive.
 275       *
 276       * @covers \mod_data\local\importer\entries_importer
 277       * @covers \mod_data\local\importer\csv_entries_importer
 278       * @return void
 279       */
 280      public function test_import_with_files(): void {
 281          [
 282              'data' => $data,
 283              'cm' => $cm,
 284          ] = $this->get_test_data();
 285  
 286          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files.zip',
 287              'test_data_import_with_files.zip');
 288          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 289  
 290          $records = $this->get_data_records($data->id);
 291          $ziparchive = new zip_archive();
 292          $ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files.zip');
 293  
 294          $importedcontent = array_values($records)[0]->items;
 295          $this->assertEquals(17, $importedcontent['ID']->content);
 296          $this->assertEquals('samplefile.png', $importedcontent['filefield']->content);
 297          $this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
 298  
 299          // We now check if content of imported file from zip content is identical to the content of the file
 300          // stored in the mod_data record in the field 'filefield'.
 301          $fileindex = array_values(array_map(fn($file) => $file->index,
 302              array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplefile.png')))[0];
 303          $filestream = $ziparchive->get_stream($fileindex);
 304          $filefield = data_get_field_from_name('filefield', $data);
 305          $filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
 306          $this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
 307              $filefieldfilecontent);
 308          fclose($filestream);
 309  
 310          // We now check if content of imported picture from zip content is identical to the content of the picture file
 311          // stored in the mod_data record in the field 'picturefield'.
 312          $fileindex = array_values(array_map(fn($file) => $file->index,
 313              array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplepicture.png')))[0];
 314          $filestream = $ziparchive->get_stream($fileindex);
 315          $filefield = data_get_field_from_name('picturefield', $data);
 316          $filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
 317          $this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
 318              $filefieldfilecontent);
 319          fclose($filestream);
 320          $this->assertCount(1, $importer->get_added_records_messages());
 321          $ziparchive->close();
 322      }
 323  
 324      /**
 325       * Tests the import including files from a zip archive.
 326       *
 327       * @covers \mod_data\local\importer\entries_importer
 328       * @covers \mod_data\local\importer\csv_entries_importer
 329       * @return void
 330       */
 331      public function test_import_with_files_missing_file(): void {
 332          [
 333              'data' => $data,
 334              'cm' => $cm,
 335          ] = $this->get_test_data();
 336  
 337          $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip',
 338              'test_data_import_with_files_missing_file.zip');
 339          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 340  
 341          $records = $this->get_data_records($data->id);
 342          $ziparchive = new zip_archive();
 343          $ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip');
 344  
 345          $importedcontent = array_values($records)[0]->items;
 346          $this->assertEquals(17, $importedcontent['ID']->content);
 347          $this->assertFalse(isset($importedcontent['filefield']));
 348          $this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
 349          $this->assertCount(1, $importer->get_added_records_messages());
 350          $ziparchive->close();
 351      }
 352  
 353      /**
 354       * Returns the records of the data instance.
 355       *
 356       * Each records has an item entry, which contains all fields associated with this item.
 357       * Each fields has the parameters name, type and content.
 358       *
 359       * @param int $dataid Id of the data instance.
 360       * @return array The records of the data instance.
 361       * @throws dml_exception
 362       */
 363      private function get_data_records(int $dataid): array {
 364          global $DB;
 365  
 366          $records = $DB->get_records('data_records', ['dataid' => $dataid]);
 367          foreach ($records as $record) {
 368              $sql = 'SELECT f.name, f.type, con.content FROM
 369                  {data_content} con JOIN {data_fields} f ON con.fieldid = f.id
 370                  WHERE con.recordid = :recordid';
 371              $items = $DB->get_records_sql($sql, array('recordid' => $record->id));
 372              $record->items = $items;
 373          }
 374          return $records;
 375      }
 376  
 377      /**
 378       * Tests if the amount of imported records is counted properly.
 379       *
 380       * @covers \mod_data\local\importer\csv_entries_importer::import_csv
 381       * @covers \mod_data\local\importer\csv_entries_importer::get_added_records_messages
 382       * @dataProvider get_added_record_messages_provider
 383       * @param string $datafilecontent the content of the datafile to test as string
 384       * @param int $expectedcount the expected count of messages depending on the datafile content
 385       */
 386      public function test_get_added_record_messages(string $datafilecontent, int $expectedcount): void {
 387          [
 388              'data' => $data,
 389              'cm' => $cm,
 390          ] = $this->get_test_data();
 391  
 392          // First we need to create the zip file from the provided data.
 393          $tmpdir = make_request_directory();
 394          $datafile = $tmpdir . '/entries_import_test_datafile_tmp_' . time() . '.csv';
 395          file_put_contents($datafile, $datafilecontent);
 396  
 397          $importer = new csv_entries_importer($datafile, 'testdatafile.csv');
 398          $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 399          $this->assertEquals($expectedcount, count($importer->get_added_records_messages()));
 400      }
 401  
 402      /**
 403       * Data provider method for self::test_get_added_record_messages.
 404       *
 405       * @return array data for testing
 406       */
 407      public function get_added_record_messages_provider(): array {
 408          return [
 409              'only header' => [
 410                  'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL,
 411                  'expectedcount' => 0 // One line is being assumed to be the header.
 412              ],
 413              'one record' => [
 414                  'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
 415                      . '5,"some short text",testfilename.pdf,testpicture.png',
 416                  'expectedcount' => 1
 417              ],
 418              'two records' => [
 419                  'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
 420                      . '5,"some short text",testfilename.pdf,testpicture.png' . PHP_EOL
 421                      . '3,"other text",testfilename2.pdf,testpicture2.png',
 422                  'expectedcount' => 2
 423              ],
 424          ];
 425      }
 426  }