[Summary view]

Differences Between: [Versions 310 and 401] [Versions 311 and 401] [Versions 39 and 401] [Versions 400 and 401]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * Core file system class definition.
  19   *
  20   * @package   core_files
  21   * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
  22   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  defined('MOODLE_INTERNAL') || die();
  26  
  27  /**
  28   * File system class used for low level access to real files in filedir.
  29   *
  30   * @package   core_files
  31   * @category  files
  32   * @copyright 2017 Andrew Nicols <andrew@nicols.co.uk>
  33   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  34   */
  35  class file_system_filedir extends file_system {
  36  
  37      /**
  38       * @var string The path to the local copy of the filedir.
  39       */
  40      protected $filedir = null;
  41  
  42      /**
  43       * @var string The path to the trashdir.
  44       */
  45      protected $trashdir = null;
  46  
  47      /**
  48       * @var string Default directory permissions for new dirs.
  49       */
  50      protected $dirpermissions = null;
  51  
  52      /**
  53       * @var string Default file permissions for new files.
  54       */
  55      protected $filepermissions = null;
  56  
  57  
  58      /**
  59       * Perform any custom setup for this type of file_system.
  60       */
  61      public function __construct() {
  62          global $CFG;
  63  
  64          if (isset($CFG->filedir)) {
  65              $this->filedir = $CFG->filedir;
  66          } else {
  67              $this->filedir = $CFG->dataroot.'/filedir';
  68          }
  69  
  70          if (isset($CFG->trashdir)) {
  71              $this->trashdir = $CFG->trashdir;
  72          } else {
  73              $this->trashdir = $CFG->dataroot.'/trashdir';
  74          }
  75  
  76          $this->dirpermissions = $CFG->directorypermissions;
  77          $this->filepermissions = $CFG->filepermissions;
  78  
  79          // Make sure the file pool directory exists.
  80          if (!is_dir($this->filedir)) {
  81              if (!mkdir($this->filedir, $this->dirpermissions, true)) {
  82                  // Permission trouble.
  83                  throw new file_exception('storedfilecannotcreatefiledirs');
  84              }
  85  
  86              // Place warning file in file pool root.
  87              if (!file_exists($this->filedir.'/warning.txt')) {
  88                  file_put_contents($this->filedir.'/warning.txt',
  89                          'This directory contains the content of uploaded files and is controlled by Moodle code. ' .
  90                          'Do not manually move, change or rename any of the files and subdirectories here.');
  91                  chmod($this->filedir . '/warning.txt', $this->filepermissions);
  92              }
  93          }
  94  
  95          // Make sure the trashdir directory exists too.
  96          if (!is_dir($this->trashdir)) {
  97              if (!mkdir($this->trashdir, $this->dirpermissions, true)) {
  98                  // Permission trouble.
  99                  throw new file_exception('storedfilecannotcreatefiledirs');
 100              }
 101          }
 102      }
 103  
 104      /**
 105       * Get the full path for the specified hash, including the path to the filedir.
 106       *
 107       * @param string $contenthash The content hash
 108       * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
 109       * @return string The full path to the content file
 110       */
 111      protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false) {
 112          return $this->get_fulldir_from_hash($contenthash) . '/' .$contenthash;
 113      }
 114  
 115      /**
 116       * Get a remote filepath for the specified stored file.
 117       *
 118       * @param stored_file $file The file to fetch the path for
 119       * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found.
 120       * @return string The full path to the content file
 121       */
 122      public function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) {
 123          $filepath = $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound);
 124  
 125          // Try content recovery.
 126          if ($fetchifnotfound && !is_readable($filepath)) {
 127              $this->recover_file($file);
 128          }
 129  
 130          return $filepath;
 131      }
 132  
 133      /**
 134       * Get a remote filepath for the specified stored file.
 135       *
 136       * @param stored_file $file The file to serve.
 137       * @return string full path to pool file with file content
 138       */
 139      public function get_remote_path_from_storedfile(stored_file $file) {
 140          return $this->get_local_path_from_storedfile($file, false);
 141      }
 142  
 143      /**
 144       * Get the full path for the specified hash, including the path to the filedir.
 145       *
 146       * @param string $contenthash The content hash
 147       * @return string The full path to the content file
 148       */
 149      protected function get_remote_path_from_hash($contenthash) {
 150          return $this->get_local_path_from_hash($contenthash, false);
 151      }
 152  
 153      /**
 154       * Get the full directory to the stored file, including the path to the
 155       * filedir, and the directory which the file is actually in.
 156       *
 157       * Note: This function does not ensure that the file is present on disk.
 158       *
 159       * @param stored_file $file The file to fetch details for.
 160       * @return string The full path to the content directory
 161       */
 162      protected function get_fulldir_from_storedfile(stored_file $file) {
 163          return $this->get_fulldir_from_hash($file->get_contenthash());
 164      }
 165  
 166      /**
 167       * Get the full directory to the stored file, including the path to the
 168       * filedir, and the directory which the file is actually in.
 169       *
 170       * @param string $contenthash The content hash
 171       * @return string The full path to the content directory
 172       */
 173      protected function get_fulldir_from_hash($contenthash) {
 174          return $this->filedir . '/' . $this->get_contentdir_from_hash($contenthash);
 175      }
 176  
 177      /**
 178       * Get the content directory for the specified content hash.
 179       * This is the directory that the file will be in, but without the
 180       * fulldir.
 181       *
 182       * @param string $contenthash The content hash
 183       * @return string The directory within filedir
 184       */
 185      protected function get_contentdir_from_hash($contenthash) {
 186          $l1 = $contenthash[0] . $contenthash[1];
 187          $l2 = $contenthash[2] . $contenthash[3];
 188          return "$l1/$l2";
 189      }
 190  
 191      /**
 192       * Get the content path for the specified content hash within filedir.
 193       *
 194       * This does not include the filedir, and is often used by file systems
 195       * as the object key for storage and retrieval.
 196       *
 197       * @param string $contenthash The content hash
 198       * @return string The filepath within filedir
 199       */
 200      protected function get_contentpath_from_hash($contenthash) {
 201          return $this->get_contentdir_from_hash($contenthash) . '/' . $contenthash;
 202      }
 203  
 204      /**
 205       * Get the full directory for the specified hash in the trash, including the path to the
 206       * trashdir, and the directory which the file is actually in.
 207       *
 208       * @param string $contenthash The content hash
 209       * @return string The full path to the trash directory
 210       */
 211      protected function get_trash_fulldir_from_hash($contenthash) {
 212          return $this->trashdir . '/' . $this->get_contentdir_from_hash($contenthash);
 213      }
 214  
 215      /**
 216       * Get the full path for the specified hash in the trash, including the path to the trashdir.
 217       *
 218       * @param string $contenthash The content hash
 219       * @return string The full path to the trash file
 220       */
 221      protected function get_trash_fullpath_from_hash($contenthash) {
 222          return $this->trashdir . '/' . $this->get_contentpath_from_hash($contenthash);
 223      }
 224  
 225      /**
 226       * Copy content of file to given pathname.
 227       *
 228       * @param stored_file $file The file to be copied
 229       * @param string $target real path to the new file
 230       * @return bool success
 231       */
 232      public function copy_content_from_storedfile(stored_file $file, $target) {
 233          $source = $this->get_local_path_from_storedfile($file, true);
 234          return copy($source, $target);
 235      }
 236  
 237      /**
 238       * Tries to recover missing content of file from trash.
 239       *
 240       * @param stored_file $file stored_file instance
 241       * @return bool success
 242       */
 243      protected function recover_file(stored_file $file) {
 244          $contentfile = $this->get_local_path_from_storedfile($file, false);
 245  
 246          if (file_exists($contentfile)) {
 247              // The file already exists on the file system. No need to recover.
 248              return true;
 249          }
 250  
 251          $contenthash = $file->get_contenthash();
 252          $contentdir = $this->get_fulldir_from_storedfile($file);
 253          $trashfile = $this->get_trash_fullpath_from_hash($contenthash);
 254          $alttrashfile = "{$this->trashdir}/{$contenthash}";
 255  
 256          if (!is_readable($trashfile)) {
 257              // The trash file was not found. Check the alternative trash file too just in case.
 258              if (!is_readable($alttrashfile)) {
 259                  return false;
 260              }
 261              // The alternative trash file in trash root exists.
 262              $trashfile = $alttrashfile;
 263          }
 264  
 265          if (filesize($trashfile) != $file->get_filesize() or file_storage::hash_from_path($trashfile) != $contenthash) {
 266              // The files are different. Leave this one in trash - something seems to be wrong with it.
 267              return false;
 268          }
 269  
 270          if (!is_dir($contentdir)) {
 271              if (!mkdir($contentdir, $this->dirpermissions, true)) {
 272                  // Unable to create the target directory.
 273                  return false;
 274              }
 275          }
 276  
 277          // Perform a rename - these are generally atomic which gives us big
 278          // performance wins, especially for large files.
 279          return rename($trashfile, $contentfile);
 280      }
 281  
 282      /**
 283       * Marks pool file as candidate for deleting.
 284       *
 285       * @param string $contenthash
 286       */
 287      public function remove_file($contenthash) {
 288          if (!self::is_file_removable($contenthash)) {
 289              // Don't remove the file - it's still in use.
 290              return;
 291          }
 292  
 293          if (!$this->is_file_readable_remotely_by_hash($contenthash)) {
 294              // The file wasn't found in the first place. Just ignore it.
 295              return;
 296          }
 297  
 298          $trashpath  = $this->get_trash_fulldir_from_hash($contenthash);
 299          $trashfile  = $this->get_trash_fullpath_from_hash($contenthash);
 300          $contentfile = $this->get_local_path_from_hash($contenthash, true);
 301  
 302          if (!is_dir($trashpath)) {
 303              mkdir($trashpath, $this->dirpermissions, true);
 304          }
 305  
 306          if (file_exists($trashfile)) {
 307              // A copy of this file is already in the trash.
 308              // Remove the old version.
 309              unlink($contentfile);
 310              return;
 311          }
 312  
 313          // Move the contentfile to the trash, and fix permissions as required.
 314          rename($contentfile, $trashfile);
 315  
 316          // Fix permissions, only if needed.
 317          $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4));
 318          if ((int)$this->filepermissions !== $currentperms) {
 319              chmod($trashfile, $this->filepermissions);
 320          }
 321      }
 322  
 323      /**
 324       * Cleanup the trash directory.
 325       */
 326      public function cron() {
 327          $this->empty_trash();
 328      }
 329  
 330      protected function empty_trash() {
 331          fulldelete($this->trashdir);
 332          set_config('fileslastcleanup', time());
 333      }
 334  
 335      /**
 336       * Add the supplied file to the file system.
 337       *
 338       * Note: If overriding this function, it is advisable to store the file
 339       * in the path returned by get_local_path_from_hash as there may be
 340       * subsequent uses of the file in the same request.
 341       *
 342       * @param string $pathname Path to file currently on disk
 343       * @param string $contenthash SHA1 hash of content if known (performance only)
 344       * @return array (contenthash, filesize, newfile)
 345       */
 346      public function add_file_from_path($pathname, $contenthash = null) {
 347  
 348          list($contenthash, $filesize) = $this->validate_hash_and_file_size($contenthash, $pathname);
 349  
 350          $hashpath = $this->get_fulldir_from_hash($contenthash);
 351          $hashfile = $this->get_local_path_from_hash($contenthash, false);
 352  
 353          $newfile = true;
 354  
 355          $hashsize = self::check_file_exists_and_get_size($hashfile);
 356          if ($hashsize !== null) {
 357              if ($hashsize === $filesize) {
 358                  return array($contenthash, $filesize, false);
 359              }
 360              if (file_storage::hash_from_path($hashfile) === $contenthash) {
 361                  // Jackpot! We have a hash collision.
 362                  mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
 363                  copy($pathname, "$this->filedir/jackpot/{$contenthash}_1");
 364                  copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2");
 365                  throw new file_pool_content_exception($contenthash);
 366              }
 367              debugging("Replacing invalid content file $contenthash");
 368              unlink($hashfile);
 369              $newfile = false;
 370          }
 371  
 372          if (!is_dir($hashpath)) {
 373              if (!mkdir($hashpath, $this->dirpermissions, true)) {
 374                  // Permission trouble.
 375                  throw new file_exception('storedfilecannotcreatefiledirs');
 376              }
 377          }
 378  
 379          // Let's try to prevent some race conditions.
 380  
 381          $prev = ignore_user_abort(true);
 382          if (file_exists($hashfile.'.tmp')) {
 383              @unlink($hashfile.'.tmp');
 384          }
 385          if (!copy($pathname, $hashfile.'.tmp')) {
 386              // Borked permissions or out of disk space.
 387              @unlink($hashfile.'.tmp');
 388              ignore_user_abort($prev);
 389              throw new file_exception('storedfilecannotcreatefile');
 390          }
 391          if (file_storage::hash_from_path($hashfile.'.tmp') !== $contenthash) {
 392              // Highly unlikely edge case, but this can happen on an NFS volume with no space remaining.
 393              @unlink($hashfile.'.tmp');
 394              ignore_user_abort($prev);
 395              throw new file_exception('storedfilecannotcreatefile');
 396          }
 397          if (!rename($hashfile.'.tmp', $hashfile)) {
 398              // Something very strange went wrong.
 399              @unlink($hashfile . '.tmp');
 400              // Note, we don't try to clean up $hashfile. Almost certainly, if it exists
 401              // (e.g. written by another process?) it will be right, so don't wipe it.
 402              ignore_user_abort($prev);
 403              throw new file_exception('storedfilecannotcreatefile');
 404          }
 405          chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
 406          if (file_exists($hashfile.'.tmp')) {
 407              // Just in case anything fails in a weird way.
 408              @unlink($hashfile.'.tmp');
 409          }
 410          ignore_user_abort($prev);
 411  
 412          return array($contenthash, $filesize, $newfile);
 413      }
 414  
 415      /**
 416       * Checks if the file exists and gets its size. This function avoids a specific issue with
 417       * networked file systems if they incorrectly report the file exists, but then decide it doesn't
 418       * as soon as you try to get the file size.
 419       *
 420       * @param string $hashfile File to check
 421       * @return int|null Null if the file does not exist, or the result of filesize(), or -1 if error
 422       */
 423      protected static function check_file_exists_and_get_size(string $hashfile): ?int {
 424          if (!file_exists($hashfile)) {
 425              // The file does not exist, return null.
 426              return null;
 427          }
 428  
 429          // In some networked file systems, it's possible that file_exists will return true when
 430          // the file doesn't exist (due to caching), but filesize will then return false because
 431          // it doesn't exist.
 432          $hashsize = @filesize($hashfile);
 433          if ($hashsize !== false) {
 434              // We successfully got a file size. Return it.
 435              return $hashsize;
 436          }
 437  
 438          // If we can't get the filesize, let's check existence again to see if we really
 439          // for sure think it exists.
 440          clearstatcache();
 441          if (!file_exists($hashfile)) {
 442              // The file doesn't exist any more, so return null.
 443              return null;
 444          }
 445  
 446          // It still thinks the file exists, but filesize failed, so we had better return an invalid
 447          // value for filesize.
 448          return -1;
 449      }
 450  
 451      /**
 452       * Add a file with the supplied content to the file system.
 453       *
 454       * Note: If overriding this function, it is advisable to store the file
 455       * in the path returned by get_local_path_from_hash as there may be
 456       * subsequent uses of the file in the same request.
 457       *
 458       * @param string $content file content - binary string
 459       * @return array (contenthash, filesize, newfile)
 460       */
 461      public function add_file_from_string($content) {
 462          global $CFG;
 463  
 464          $contenthash = file_storage::hash_from_string($content);
 465          // Binary length.
 466          $filesize = strlen($content ?? '');
 467  
 468          $hashpath = $this->get_fulldir_from_hash($contenthash);
 469          $hashfile = $this->get_local_path_from_hash($contenthash, false);
 470  
 471          $newfile = true;
 472  
 473          $hashsize = self::check_file_exists_and_get_size($hashfile);
 474          if ($hashsize !== null) {
 475              if ($hashsize === $filesize) {
 476                  return array($contenthash, $filesize, false);
 477              }
 478              if (file_storage::hash_from_path($hashfile) === $contenthash) {
 479                  // Jackpot! We have a hash collision.
 480                  mkdir("$this->filedir/jackpot/", $this->dirpermissions, true);
 481                  copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1");
 482                  file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content);
 483                  throw new file_pool_content_exception($contenthash);
 484              }
 485              debugging("Replacing invalid content file $contenthash");
 486              unlink($hashfile);
 487              $newfile = false;
 488          }
 489  
 490          if (!is_dir($hashpath)) {
 491              if (!mkdir($hashpath, $this->dirpermissions, true)) {
 492                  // Permission trouble.
 493                  throw new file_exception('storedfilecannotcreatefiledirs');
 494              }
 495          }
 496  
 497          // Hopefully this works around most potential race conditions.
 498  
 499          $prev = ignore_user_abort(true);
 500  
 501          if (!empty($CFG->preventfilelocking)) {
 502              $newsize = file_put_contents($hashfile.'.tmp', $content);
 503          } else {
 504              $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX);
 505          }
 506  
 507          if ($newsize === false) {
 508              // Borked permissions most likely.
 509              ignore_user_abort($prev);
 510              throw new file_exception('storedfilecannotcreatefile');
 511          }
 512          if (filesize($hashfile.'.tmp') !== $filesize) {
 513              // Out of disk space?
 514              unlink($hashfile.'.tmp');
 515              ignore_user_abort($prev);
 516              throw new file_exception('storedfilecannotcreatefile');
 517          }
 518          if (!rename($hashfile.'.tmp', $hashfile)) {
 519              // Something very strange went wrong.
 520              @unlink($hashfile . '.tmp');
 521              // Note, we don't try to clean up $hashfile. Almost certainly, if it exists
 522              // (e.g. written by another process?) it will be right, so don't wipe it.
 523              ignore_user_abort($prev);
 524              throw new file_exception('storedfilecannotcreatefile');
 525          }
 526          chmod($hashfile, $this->filepermissions); // Fix permissions if needed.
 527          if (file_exists($hashfile.'.tmp')) {
 528              // Just in case anything fails in a weird way.
 529              @unlink($hashfile.'.tmp');
 530          }
 531          ignore_user_abort($prev);
 532  
 533          return array($contenthash, $filesize, $newfile);
 534      }
 535  
 536  }