[Summary view]

Differences Between: [Versions 310 and 401] [Versions 311 and 401] [Versions 39 and 401] [Versions 400 and 401]

   1  <?php
   2  
   3  /*
   4   * This file is part of Mustache.php.
   5   *
   6   * (c) 2010-2017 Justin Hileman
   7   *
   8   * For the full copyright and license information, please view the LICENSE
   9   * file that was distributed with this source code.
  10   */
  11  
  12  /**
  13   * Mustache Tokenizer class.
  14   *
  15   * This class is responsible for turning raw template source into a set of Mustache tokens.
  16   */
  17  class Mustache_Tokenizer
  18  {
  19      // Finite state machine states
  20      const IN_TEXT     = 0;
  21      const IN_TAG_TYPE = 1;
  22      const IN_TAG      = 2;
  23  
  24      // Token types
  25      const T_SECTION      = '#';
  26      const T_INVERTED     = '^';
  27      const T_END_SECTION  = '/';
  28      const T_COMMENT      = '!';
  29      const T_PARTIAL      = '>';
  30      const T_PARENT       = '<';
  31      const T_DELIM_CHANGE = '=';
  32      const T_ESCAPED      = '_v';
  33      const T_UNESCAPED    = '{';
  34      const T_UNESCAPED_2  = '&';
  35      const T_TEXT         = '_t';
  36      const T_PRAGMA       = '%';
  37      const T_BLOCK_VAR    = '$';
  38      const T_BLOCK_ARG    = '$arg';
  39  
  40      // Valid token types
  41      private static $tagTypes = array(
  42          self::T_SECTION      => true,
  43          self::T_INVERTED     => true,
  44          self::T_END_SECTION  => true,
  45          self::T_COMMENT      => true,
  46          self::T_PARTIAL      => true,
  47          self::T_PARENT       => true,
  48          self::T_DELIM_CHANGE => true,
  49          self::T_ESCAPED      => true,
  50          self::T_UNESCAPED    => true,
  51          self::T_UNESCAPED_2  => true,
  52          self::T_PRAGMA       => true,
  53          self::T_BLOCK_VAR    => true,
  54      );
  55  
  56      // Token properties
  57      const TYPE    = 'type';
  58      const NAME    = 'name';
  59      const OTAG    = 'otag';
  60      const CTAG    = 'ctag';
  61      const LINE    = 'line';
  62      const INDEX   = 'index';
  63      const END     = 'end';
  64      const INDENT  = 'indent';
  65      const NODES   = 'nodes';
  66      const VALUE   = 'value';
  67      const FILTERS = 'filters';
  68  
  69      private $state;
  70      private $tagType;
  71      private $buffer;
  72      private $tokens;
  73      private $seenTag;
  74      private $line;
  75  
  76      private $otag;
  77      private $otagChar;
  78      private $otagLen;
  79  
  80      private $ctag;
  81      private $ctagChar;
  82      private $ctagLen;
  83  
  84      /**
  85       * Scan and tokenize template source.
  86       *
  87       * @throws Mustache_Exception_SyntaxException when mismatched section tags are encountered
  88       * @throws Mustache_Exception_InvalidArgumentException when $delimiters string is invalid
  89       *
  90       * @param string $text       Mustache template source to tokenize
  91       * @param string $delimiters Optionally, pass initial opening and closing delimiters (default: empty string)
  92       *
  93       * @return array Set of Mustache tokens
  94       */
  95      public function scan($text, $delimiters = '')
  96      {
  97          // Setting mbstring.func_overload makes things *really* slow.
  98          // Let's do everyone a favor and scan this string as ASCII instead.
  99          //
 100          // The INI directive was removed in PHP 8.0 so we don't need to check there (and can drop it
 101          // when we remove support for older versions of PHP).
 102          //
 103          // @codeCoverageIgnoreStart
 104          $encoding = null;
 105          if (version_compare(PHP_VERSION, '8.0.0', '<')) {
 106              if (function_exists('mb_internal_encoding') && ini_get('mbstring.func_overload') & 2) {
 107                  $encoding = mb_internal_encoding();
 108                  mb_internal_encoding('ASCII');
 109              }
 110          }
 111          // @codeCoverageIgnoreEnd
 112  
 113          $this->reset();
 114  
 115          if (is_string($delimiters) && $delimiters = trim($delimiters)) {
 116              $this->setDelimiters($delimiters);
 117          }
 118  
 119          $len = strlen($text);
 120          for ($i = 0; $i < $len; $i++) {
 121              switch ($this->state) {
 122                  case self::IN_TEXT:
 123                      $char = $text[$i];
 124                      // Test whether it's time to change tags.
 125                      if ($char === $this->otagChar && substr($text, $i, $this->otagLen) === $this->otag) {
 126                          $i--;
 127                          $this->flushBuffer();
 128                          $this->state = self::IN_TAG_TYPE;
 129                      } else {
 130                          $this->buffer .= $char;
 131                          if ($char === "\n") {
 132                              $this->flushBuffer();
 133                              $this->line++;
 134                          }
 135                      }
 136                      break;
 137  
 138                  case self::IN_TAG_TYPE:
 139                      $i += $this->otagLen - 1;
 140                      $char = $text[$i + 1];
 141                      if (isset(self::$tagTypes[$char])) {
 142                          $tag = $char;
 143                          $this->tagType = $tag;
 144                      } else {
 145                          $tag = null;
 146                          $this->tagType = self::T_ESCAPED;
 147                      }
 148  
 149                      if ($this->tagType === self::T_DELIM_CHANGE) {
 150                          $i = $this->changeDelimiters($text, $i);
 151                          $this->state = self::IN_TEXT;
 152                      } elseif ($this->tagType === self::T_PRAGMA) {
 153                          $i = $this->addPragma($text, $i);
 154                          $this->state = self::IN_TEXT;
 155                      } else {
 156                          if ($tag !== null) {
 157                              $i++;
 158                          }
 159                          $this->state = self::IN_TAG;
 160                      }
 161                      $this->seenTag = $i;
 162                      break;
 163  
 164                  default:
 165                      $char = $text[$i];
 166                      // Test whether it's time to change tags.
 167                      if ($char === $this->ctagChar && substr($text, $i, $this->ctagLen) === $this->ctag) {
 168                          $token = array(
 169                              self::TYPE  => $this->tagType,
 170                              self::NAME  => trim($this->buffer),
 171                              self::OTAG  => $this->otag,
 172                              self::CTAG  => $this->ctag,
 173                              self::LINE  => $this->line,
 174                              self::INDEX => ($this->tagType === self::T_END_SECTION) ? $this->seenTag - $this->otagLen : $i + $this->ctagLen,
 175                          );
 176  
 177                          if ($this->tagType === self::T_UNESCAPED) {
 178                              // Clean up `{{{ tripleStache }}}` style tokens.
 179                              if ($this->ctag === '}}') {
 180                                  if (($i + 2 < $len) && $text[$i + 2] === '}') {
 181                                      $i++;
 182                                  } else {
 183                                      $msg = sprintf(
 184                                          'Mismatched tag delimiters: %s on line %d',
 185                                          $token[self::NAME],
 186                                          $token[self::LINE]
 187                                      );
 188  
 189                                      throw new Mustache_Exception_SyntaxException($msg, $token);
 190                                  }
 191                              } else {
 192                                  $lastName = $token[self::NAME];
 193                                  if (substr($lastName, -1) === '}') {
 194                                      $token[self::NAME] = trim(substr($lastName, 0, -1));
 195                                  } else {
 196                                      $msg = sprintf(
 197                                          'Mismatched tag delimiters: %s on line %d',
 198                                          $token[self::NAME],
 199                                          $token[self::LINE]
 200                                      );
 201  
 202                                      throw new Mustache_Exception_SyntaxException($msg, $token);
 203                                  }
 204                              }
 205                          }
 206  
 207                          $this->buffer = '';
 208                          $i += $this->ctagLen - 1;
 209                          $this->state = self::IN_TEXT;
 210                          $this->tokens[] = $token;
 211                      } else {
 212                          $this->buffer .= $char;
 213                      }
 214                      break;
 215              }
 216          }
 217  
 218          if ($this->state !== self::IN_TEXT) {
 219              $this->throwUnclosedTagException();
 220          }
 221  
 222          $this->flushBuffer();
 223  
 224          // Restore the user's encoding...
 225          // @codeCoverageIgnoreStart
 226          if ($encoding) {
 227              mb_internal_encoding($encoding);
 228          }
 229          // @codeCoverageIgnoreEnd
 230  
 231          return $this->tokens;
 232      }
 233  
 234      /**
 235       * Helper function to reset tokenizer internal state.
 236       */
 237      private function reset()
 238      {
 239          $this->state    = self::IN_TEXT;
 240          $this->tagType  = null;
 241          $this->buffer   = '';
 242          $this->tokens   = array();
 243          $this->seenTag  = false;
 244          $this->line     = 0;
 245  
 246          $this->otag     = '{{';
 247          $this->otagChar = '{';
 248          $this->otagLen  = 2;
 249  
 250          $this->ctag     = '}}';
 251          $this->ctagChar = '}';
 252          $this->ctagLen  = 2;
 253      }
 254  
 255      /**
 256       * Flush the current buffer to a token.
 257       */
 258      private function flushBuffer()
 259      {
 260          if (strlen($this->buffer) > 0) {
 261              $this->tokens[] = array(
 262                  self::TYPE  => self::T_TEXT,
 263                  self::LINE  => $this->line,
 264                  self::VALUE => $this->buffer,
 265              );
 266              $this->buffer   = '';
 267          }
 268      }
 269  
 270      /**
 271       * Change the current Mustache delimiters. Set new `otag` and `ctag` values.
 272       *
 273       * @throws Mustache_Exception_SyntaxException when delimiter string is invalid
 274       *
 275       * @param string $text  Mustache template source
 276       * @param int    $index Current tokenizer index
 277       *
 278       * @return int New index value
 279       */
 280      private function changeDelimiters($text, $index)
 281      {
 282          $startIndex = strpos($text, '=', $index) + 1;
 283          $close      = '=' . $this->ctag;
 284          $closeIndex = strpos($text, $close, $index);
 285  
 286          if ($closeIndex === false) {
 287              $this->throwUnclosedTagException();
 288          }
 289  
 290          $token = array(
 291              self::TYPE => self::T_DELIM_CHANGE,
 292              self::LINE => $this->line,
 293          );
 294  
 295          try {
 296              $this->setDelimiters(trim(substr($text, $startIndex, $closeIndex - $startIndex)));
 297          } catch (Mustache_Exception_InvalidArgumentException $e) {
 298              throw new Mustache_Exception_SyntaxException($e->getMessage(), $token);
 299          }
 300  
 301          $this->tokens[] = $token;
 302  
 303          return $closeIndex + strlen($close) - 1;
 304      }
 305  
 306      /**
 307       * Set the current Mustache `otag` and `ctag` delimiters.
 308       *
 309       * @throws Mustache_Exception_InvalidArgumentException when delimiter string is invalid
 310       *
 311       * @param string $delimiters
 312       */
 313      private function setDelimiters($delimiters)
 314      {
 315          if (!preg_match('/^\s*(\S+)\s+(\S+)\s*$/', $delimiters, $matches)) {
 316              throw new Mustache_Exception_InvalidArgumentException(sprintf('Invalid delimiters: %s', $delimiters));
 317          }
 318  
 319          list($_, $otag, $ctag) = $matches;
 320  
 321          $this->otag     = $otag;
 322          $this->otagChar = $otag[0];
 323          $this->otagLen  = strlen($otag);
 324  
 325          $this->ctag     = $ctag;
 326          $this->ctagChar = $ctag[0];
 327          $this->ctagLen  = strlen($ctag);
 328      }
 329  
 330      /**
 331       * Add pragma token.
 332       *
 333       * Pragmas are hoisted to the front of the template, so all pragma tokens
 334       * will appear at the front of the token list.
 335       *
 336       * @param string $text
 337       * @param int    $index
 338       *
 339       * @return int New index value
 340       */
 341      private function addPragma($text, $index)
 342      {
 343          $end    = strpos($text, $this->ctag, $index);
 344          if ($end === false) {
 345              $this->throwUnclosedTagException();
 346          }
 347  
 348          $pragma = trim(substr($text, $index + 2, $end - $index - 2));
 349  
 350          // Pragmas are hoisted to the front of the template.
 351          array_unshift($this->tokens, array(
 352              self::TYPE => self::T_PRAGMA,
 353              self::NAME => $pragma,
 354              self::LINE => 0,
 355          ));
 356  
 357          return $end + $this->ctagLen - 1;
 358      }
 359  
 360      private function throwUnclosedTagException()
 361      {
 362          $name = trim($this->buffer);
 363          if ($name !== '') {
 364              $msg = sprintf('Unclosed tag: %s on line %d', $name, $this->line);
 365          } else {
 366              $msg = sprintf('Unclosed tag on line %d', $this->line);
 367          }
 368  
 369          throw new Mustache_Exception_SyntaxException($msg, array(
 370              self::TYPE  => $this->tagType,
 371              self::NAME  => $name,
 372              self::OTAG  => $this->otag,
 373              self::CTAG  => $this->ctag,
 374              self::LINE  => $this->line,
 375              self::INDEX => $this->seenTag - $this->otagLen,
 376          ));
 377      }
 378  }