[Summary view]

Differences Between: [Versions 400 and 401] [Versions 400 and 402] [Versions 400 and 403]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * Wrapper to run previously set-up behat tests in parallel.
  19   *
  20   * @package    tool_behat
  21   * @copyright  2014 NetSpot Pty Ltd
  22   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  if (isset($_SERVER['REMOTE_ADDR'])) {
  26      die(); // No access from web!
  27  }
  28  
  29  define('CLI_SCRIPT', true);
  30  define('ABORT_AFTER_CONFIG', true);
  31  define('CACHE_DISABLE_ALL', true);
  32  define('NO_OUTPUT_BUFFERING', true);
  33  
  34  require_once(__DIR__ .'/../../../../config.php');
  35  require_once (__DIR__.'/../../../../lib/clilib.php');
  36  require_once (__DIR__.'/../../../../lib/behat/lib.php');
  37  require_once (__DIR__.'/../../../../lib/behat/classes/behat_command.php');
  38  require_once (__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');
  39  
  40  error_reporting(E_ALL | E_STRICT);
  41  ini_set('display_errors', '1');
  42  ini_set('log_errors', '1');
  43  
  44  list($options, $unrecognised) = cli_get_params(
  45      array(
  46          'stop-on-failure' => 0,
  47          'verbose'  => false,
  48          'replace'  => '',
  49          'help'     => false,
  50          'tags'     => '',
  51          'profile'  => '',
  52          'feature'  => '',
  53          'suite'    => '',
  54          'fromrun'  => 1,
  55          'torun'    => 0,
  56          'single-run' => false,
  57          'rerun' => 0,
  58          'auto-rerun' => 0,
  59      ),
  60      array(
  61          'h' => 'help',
  62          't' => 'tags',
  63          'p' => 'profile',
  64          's' => 'single-run',
  65      )
  66  );
  67  
  68  // Checking run.php CLI script usage.
  69  $help = "
  70  Behat utilities to run behat tests in parallel
  71  
  72  Usage:
  73    php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace=\"{run}\"] [--fromrun=value --torun=value] [--help]
  74  
  75  Options:
  76  --BEHAT_OPTION     Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html
  77  --feature          Only execute specified feature file (Absolute path of feature file).
  78  --suite            Specified theme scenarios will be executed.
  79  --replace          Replace args string with run process number, useful for output.
  80  --fromrun          Execute run starting from (Used for parallel runs on different vms)
  81  --torun            Execute run till (Used for parallel runs on different vms)
  82  --rerun            Re-run scenarios that failed during last execution.
  83  --auto-rerun       Automatically re-run scenarios that failed during last execution.
  84  
  85  -h, --help         Print out this help
  86  
  87  Example from Moodle root directory:
  88  \$ php admin/tool/behat/cli/run.php --tags=\"@javascript\"
  89  
  90  More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
  91  ";
  92  
  93  if (!empty($options['help'])) {
  94      echo $help;
  95      exit(0);
  96  }
  97  
  98  $parallelrun = behat_config_manager::get_behat_run_config_value('parallel');
  99  
 100  // Check if the options provided are valid to run behat.
 101  if ($parallelrun === false) {
 102      // Parallel run should not have fromrun or torun options greater than 1.
 103      if (($options['fromrun'] > 1) || ($options['torun'] > 1)) {
 104          echo "Test site is not initialized  for parallel run." . PHP_EOL;
 105          exit(1);
 106      }
 107  } else {
 108      // Ensure fromrun is within limits of initialized test site.
 109      if (!empty($options['fromrun']) && ($options['fromrun'] > $parallelrun)) {
 110          echo "From run (" . $options['fromrun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
 111          exit(1);
 112      }
 113  
 114      // Default torun is maximum parallel runs and should be less than equal to parallelruns.
 115      if (empty($options['torun'])) {
 116          $options['torun'] = $parallelrun;
 117      } else {
 118          if ($options['torun'] > $parallelrun) {
 119              echo "To run (" . $options['torun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL;
 120              exit(1);
 121          }
 122      }
 123  }
 124  
 125  // Capture signals and ensure we clean symlinks.
 126  if (extension_loaded('pcntl')) {
 127      $disabled = explode(',', ini_get('disable_functions'));
 128      if (!in_array('pcntl_signal', $disabled)) {
 129          pcntl_signal(SIGTERM, "signal_handler");
 130          pcntl_signal(SIGINT, "signal_handler");
 131      }
 132  }
 133  
 134  $time = microtime(true);
 135  array_walk($unrecognised, function (&$v) {
 136      if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1=\"\$2\"", $v)) {
 137          $v = $x;
 138      } else if (!preg_match("#^-#", $v)) {
 139          $v = escapeshellarg($v);
 140      }
 141  });
 142  $extraopts = $unrecognised;
 143  
 144  if ($options['profile']) {
 145      $profile = $options['profile'];
 146  
 147      // If profile passed is not set, then exit (note we skip if the 'replace' option is found within the 'profile' value).
 148      if (!isset($CFG->behat_config[$profile]) && !isset($CFG->behat_profiles[$profile]) &&
 149              !($options['replace'] && (strpos($profile, (string) $options['replace']) !== false))) {
 150  
 151          echo "Invalid profile passed: " . $profile . PHP_EOL;
 152          exit(1);
 153      }
 154  
 155      $extraopts['profile'] = '--profile="' . $profile . '"';
 156      // By default, profile tags will be used.
 157      if (!empty($CFG->behat_config[$profile]['filters']['tags'])) {
 158          $tags = $CFG->behat_config[$profile]['filters']['tags'];
 159      }
 160  }
 161  
 162  // Command line tags have precedence (std behat behavior).
 163  if ($options['tags']) {
 164      $tags = $options['tags'];
 165      $extraopts['tags'] = '--tags="' . $tags . '"';
 166  }
 167  
 168  // Add suite option if specified.
 169  if ($options['suite']) {
 170      $extraopts['suite'] = '--suite="' . $options['suite'] . '"';
 171  }
 172  
 173  // Feature should be added to last, for behat command.
 174  if ($options['feature']) {
 175      $extraopts['feature'] = $options['feature'];
 176      // Only run 1 process as process.
 177      // Feature file is picked from absolute path provided, so no need to check for behat.yml.
 178      $options['torun'] = $options['fromrun'];
 179  }
 180  
 181  // Set of options to pass to behat.
 182  $extraoptstr = implode(' ', $extraopts);
 183  
 184  // If rerun is passed then ensure we just run the failed processes.
 185  $lastfailedstatus = 0;
 186  $lasttorun = $options['torun'];
 187  $lastfromrun = $options['fromrun'];
 188  if ($options['rerun']) {
 189      // Get last combined failed status.
 190      $lastfailedstatus = behat_config_manager::get_behat_run_config_value('lastcombinedfailedstatus');
 191      $lasttorun = behat_config_manager::get_behat_run_config_value('lasttorun');
 192      $lastfromrun = behat_config_manager::get_behat_run_config_value('lastfromrun');
 193  
 194      if ($lastfailedstatus !== false) {
 195          $extraoptstr .= ' --rerun';
 196      }
 197  
 198      // If torun is less than last torun, then just set this to min last to run and similar for fromrun.
 199      if ($options['torun'] < $lasttorun) {
 200          $options['torun'];
 201      }
 202      if ($options['fromrun'] > $lastfromrun) {
 203          $options['fromrun'];
 204      }
 205      unset($options['rerun']);
 206  }
 207  
 208  $cmds = array();
 209  $exitcodes = array();
 210  $status = 0;
 211  $verbose = empty($options['verbose']) ? false : true;
 212  
 213  // Execute behat run commands.
 214  if (empty($parallelrun)) {
 215      $cwd = getcwd();
 216      chdir(__DIR__);
 217      $runtestscommand = behat_command::get_behat_command(false, false, true);
 218      $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
 219      $runtestscommand .= ' ' . $extraoptstr;
 220      $cmds['singlerun'] = $runtestscommand;
 221  
 222      echo "Running single behat site:" . PHP_EOL;
 223      passthru("php $runtestscommand", $status);
 224      $exitcodes['singlerun'] = $status;
 225      chdir($cwd);
 226  } else {
 227  
 228      echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
 229  
 230      for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
 231          $lastfailed = 1 & $lastfailedstatus >> ($i - 1);
 232  
 233          // Bypass if not failed in last run.
 234          if ($lastfailedstatus && !$lastfailed && ($i <= $lasttorun) && ($i >= $lastfromrun)) {
 235              continue;
 236          }
 237  
 238          $CFG->behatrunprocess = $i;
 239  
 240          // Options parameters to be added to each run.
 241          $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr;
 242  
 243          $behatcommand = behat_command::get_behat_command(false, false, true);
 244          $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
 245  
 246          // Command to execute behat run.
 247          $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
 248          echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
 249      }
 250  
 251      if (empty($cmds)) {
 252          echo "No commands to execute " . PHP_EOL;
 253          exit(1);
 254      }
 255  
 256      // Create site symlink if necessary.
 257      if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
 258          echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
 259          exit(1);
 260      }
 261  
 262      // Save torun and from run, so it can be used to detect if it was executed in last run.
 263      behat_config_manager::set_behat_run_config_value('lasttorun', $options['torun']);
 264      behat_config_manager::set_behat_run_config_value('lastfromrun', $options['fromrun']);
 265  
 266      // Keep no delay by default, between each parallel, let user decide.
 267      if (!defined('BEHAT_PARALLEL_START_DELAY')) {
 268          define('BEHAT_PARALLEL_START_DELAY', 0);
 269      }
 270  
 271      // Execute all commands, relative to moodle root directory.
 272      $processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../", BEHAT_PARALLEL_START_DELAY);
 273      $stoponfail = empty($options['stop-on-failure']) ? false : true;
 274  
 275      // Print header.
 276      print_process_start_info($processes);
 277  
 278      // Print combined run o/p from processes.
 279      $exitcodes = print_combined_run_output($processes, $stoponfail);
 280      // Time to finish run.
 281      $time = round(microtime(true) - $time, 1);
 282      echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
 283      ksort($exitcodes);
 284  
 285      // Print exit info from each run.
 286      // Status bits contains pass/fail status of parallel runs.
 287      foreach ($exitcodes as $name => $exitcode) {
 288          if ($exitcode) {
 289              $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
 290              $status |= (1 << ($runno - 1));
 291          }
 292      }
 293  
 294      // Print each process information.
 295      print_each_process_info($processes, $verbose, $status);
 296  }
 297  
 298  // Save final exit code containing which run failed.
 299  behat_config_manager::set_behat_run_config_value('lastcombinedfailedstatus', $status);
 300  
 301  // Show exit code from each process, if any process failed and how to rerun failed process.
 302  if ($verbose || $status) {
 303      // Check if status of last run is failure and rerun is suggested.
 304      if (!empty($options['auto-rerun']) && $status) {
 305          // Rerun for the number of tries passed.
 306          for ($i = 0; $i < $options['auto-rerun']; $i++) {
 307  
 308              // Run individual commands, to avoid parallel failures.
 309              foreach ($exitcodes as $behatrunname => $exitcode) {
 310                  // If not failed in last run, then skip.
 311                  if ($exitcode == 0) {
 312                      continue;
 313                  }
 314  
 315                  // This was a failure.
 316                  echo "*** Re-running behat run: $behatrunname ***" . PHP_EOL;
 317                  if ($verbose) {
 318                      echo "Executing: " . $cmds[$behatrunname] . " --rerun" . PHP_EOL;
 319                  }
 320  
 321                  passthru("php $cmds[$behatrunname] --rerun", $rerunstatus);
 322  
 323                  // Update exit code.
 324                  $exitcodes[$behatrunname] = $rerunstatus;
 325              }
 326          }
 327  
 328          // Update status after auto-rerun finished.
 329          $status = 0;
 330          foreach ($exitcodes as $name => $exitcode) {
 331              if ($exitcode) {
 332                  if (!empty($parallelrun)) {
 333                      $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
 334                  } else {
 335                      $runno = 1;
 336                  }
 337                  $status |= (1 << ($runno - 1));
 338              }
 339          }
 340      }
 341  
 342      // Show final o/p with re-run commands.
 343      if ($status) {
 344          if (!empty($parallelrun)) {
 345              // Echo exit codes.
 346              echo "Exit codes for each behat run: " . PHP_EOL;
 347              foreach ($exitcodes as $run => $exitcode) {
 348                  echo $run . ": " . $exitcode . PHP_EOL;
 349              }
 350              unset($extraopts['fromrun']);
 351              unset($extraopts['torun']);
 352              if (!empty($options['replace'])) {
 353                  $extraopts['replace'] = '--replace="' . $options['replace'] . '"';
 354              }
 355          }
 356  
 357          echo "To re-run failed processes, you can use following command:" . PHP_EOL;
 358          $extraopts['rerun'] = '--rerun';
 359          $extraoptstr = implode(' ', $extraopts);
 360          echo behat_command::get_behat_command(true, true, true) . " " . $extraoptstr . PHP_EOL;
 361      }
 362      echo PHP_EOL;
 363  }
 364  
 365  // Remove site symlink if necessary.
 366  behat_config_manager::drop_parallel_site_links();
 367  
 368  exit($status);
 369  
 370  /**
 371   * Signal handler for terminal exit.
 372   *
 373   * @param int $signal signal number.
 374   */
 375  function signal_handler($signal) {
 376      switch ($signal) {
 377          case SIGTERM:
 378          case SIGKILL:
 379          case SIGINT:
 380              // Remove site symlink if necessary.
 381              behat_config_manager::drop_parallel_site_links();
 382              exit(1);
 383      }
 384  }
 385  
 386  /**
 387   * Prints header from the first process.
 388   *
 389   * @param array $processes list of processes to loop though.
 390   */
 391  function print_process_start_info($processes) {
 392      $printed = false;
 393      // Keep looping though processes, till we get first process o/p.
 394      while (!$printed) {
 395          usleep(10000);
 396          foreach ($processes as $name => $process) {
 397              // Exit if any process has stopped.
 398              if (!$process->isRunning()) {
 399                  $printed = true;
 400                  break;
 401              }
 402  
 403              $op = explode(PHP_EOL, $process->getOutput());
 404              if (count($op) >= 3) {
 405                  foreach ($op as $line) {
 406                      if (trim($line) && (strpos($line, '.') !== 0)) {
 407                          echo $line . PHP_EOL;
 408                      }
 409                  }
 410                  $printed = true;
 411              }
 412          }
 413      }
 414  }
 415  
 416  /**
 417   * Loop though all processes and print combined o/p
 418   *
 419   * @param array $processes list of processes to loop though.
 420   * @param bool $stoponfail Stop all processes and exit if failed.
 421   * @return array list of exit codes from all processes.
 422   */
 423  function print_combined_run_output($processes, $stoponfail = false) {
 424      $exitcodes = array();
 425      $maxdotsonline = 70;
 426      $remainingprintlen = $maxdotsonline;
 427      $progresscount = 0;
 428      while (count($exitcodes) != count($processes)) {
 429          usleep(10000);
 430          foreach ($processes as $name => $process) {
 431              if ($process->isRunning()) {
 432                  $op = $process->getIncrementalOutput();
 433                  if (trim($op)) {
 434                      $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
 435                      // Exit process if anything fails.
 436                      if ($stoponfail && (strpos($update, 'F') !== false)) {
 437                          $process->stop(0);
 438                      }
 439  
 440                      $strlentoprint = strlen($update);
 441  
 442                      // If not enough dots printed on line then just print.
 443                      if ($strlentoprint < $remainingprintlen) {
 444                          echo $update;
 445                          $remainingprintlen = $remainingprintlen - $strlentoprint;
 446                      } else if ($strlentoprint == $remainingprintlen) {
 447                          $progresscount += $maxdotsonline;
 448                          echo $update ." " . $progresscount . PHP_EOL;
 449                          $remainingprintlen = $maxdotsonline;
 450                      } else {
 451                          while ($part = substr($update, 0, $remainingprintlen) > 0) {
 452                              $progresscount += $maxdotsonline;
 453                              echo $part . " " . $progresscount . PHP_EOL;
 454                              $update = substr($update, $remainingprintlen);
 455                              $remainingprintlen = $maxdotsonline;
 456                          }
 457                      }
 458                  }
 459              } else {
 460                  $exitcodes[$name] = $process->getExitCode();
 461                  if ($stoponfail && ($exitcodes[$name] != 0)) {
 462                      foreach ($processes as $l => $p) {
 463                          $exitcodes[$l] = -1;
 464                          $process->stop(0);
 465                      }
 466                  }
 467              }
 468          }
 469      }
 470  
 471      echo PHP_EOL;
 472      return $exitcodes;
 473  }
 474  
 475  /**
 476   * Loop though all processes and print combined o/p
 477   *
 478   * @param array $processes list of processes to loop though.
 479   * @param bool $verbose Show verbose output for each process.
 480   */
 481  function print_each_process_info($processes, $verbose = false, $status = 0) {
 482      foreach ($processes as $name => $process) {
 483          echo "**************** [" . $name . "] ****************" . PHP_EOL;
 484          if ($verbose) {
 485              echo $process->getOutput();
 486              echo $process->getErrorOutput();
 487  
 488          } else if ($status) {
 489              // Only show failed o/p.
 490              $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name);
 491              if ((1 << ($runno - 1)) & $status) {
 492                  echo $process->getOutput();
 493                  echo $process->getErrorOutput();
 494              } else {
 495                  echo get_status_lines_from_run_op($process);
 496              }
 497  
 498          } else {
 499              echo get_status_lines_from_run_op($process);
 500          }
 501          echo PHP_EOL;
 502      }
 503  }
 504  
 505  /**
 506   * Extract status information from behat o/p and return.
 507   * @param Symfony\Component\Process\Process $process
 508   * @return string
 509   */
 510  function get_status_lines_from_run_op(Symfony\Component\Process\Process $process) {
 511      $statusstr = '';
 512      $op = explode(PHP_EOL, $process->getOutput());
 513      foreach ($op as $line) {
 514          // Don't print progress .
 515          if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
 516              (strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
 517              (strpos($line, 'Browser specific fixes ') !== 0)) {
 518              $statusstr .= $line . PHP_EOL;
 519          }
 520      }
 521  
 522      return $statusstr;
 523  }
 524