Search moodle.org's
Developer Documentation

See Release Notes
Long Term Support Release

  • Bug fixes for general core bugs in 4.1.x will end 13 November 2023 (12 months).
  • Bug fixes for security issues in 4.1.x will end 10 November 2025 (36 months).
  • PHP version: minimum PHP 7.4.0 Note: minimum PHP version has increased since Moodle 4.0. PHP 8.0.x is supported too.
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.

/**
 * Wrapper to run previously set-up behat tests in parallel.
 *
 * @package    tool_behat
 * @copyright  2014 NetSpot Pty Ltd
 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
 */

if (isset($_SERVER['REMOTE_ADDR'])) {
    die(); // No access from web!
}

define('CLI_SCRIPT', true);
define('ABORT_AFTER_CONFIG', true);
define('CACHE_DISABLE_ALL', true);
define('NO_OUTPUT_BUFFERING', true);

require_once(__DIR__ .'/../../../../config.php');
require_once(__DIR__.'/../../../../lib/clilib.php');
require_once(__DIR__.'/../../../../lib/behat/lib.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_command.php');
require_once(__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');

error_reporting(E_ALL | E_STRICT);
ini_set('display_errors', '1');
ini_set('log_errors', '1');

list($options, $unrecognised) = cli_get_params(
    array(
        'stop-on-failure' => 0,
        'verbose'  => false,
        'replace'  => '',
        'help'     => false,
        'tags'     => '',
        'profile'  => '',
        'feature'  => '',
        'suite'    => '',
        'fromrun'  => 1,
        'torun'    => 0,
        'single-run' => false,
        'rerun' => 0,
        'auto-rerun' => 0,
    ),
    array(
        'h' => 'help',
        't' => 'tags',
        'p' => 'profile',
        's' => 'single-run',
    )
);

// Checking run.php CLI script usage.
$help = "
Behat utilities to run behat tests in parallel

Usage:
  php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace=\"{run}\"] [--fromrun=value --torun=value] [--help]

Options:
--BEHAT_OPTION     Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html
--feature          Only execute specified feature file (Absolute path of feature file).
--suite            Specified theme scenarios will be executed.
--replace          Replace args string with run process number, useful for output.
--fromrun          Execute run starting from (Used for parallel runs on different vms)
--torun            Execute run till (Used for parallel runs on different vms)
--rerun            Re-run scenarios that failed during last execution.
--auto-rerun       Automatically re-run scenarios that failed during last execution.

-h, --help         Print out this help

Example from Moodle root directory:
\$ php admin/tool/behat/cli/run.php --tags=\"@javascript\"

< More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
> More info in https://moodledev.io/general/development/tools/behat/running
"; if (!empty($options['help'])) { echo $help; exit(0); } $parallelrun = behat_config_manager::get_behat_run_config_value('parallel'); // Check if the options provided are valid to run behat. if ($parallelrun === false) { // Parallel run should not have fromrun or torun options greater than 1. if (($options['fromrun'] > 1) || ($options['torun'] > 1)) { echo "Test site is not initialized for parallel run." . PHP_EOL; exit(1); } } else { // Ensure fromrun is within limits of initialized test site. if (!empty($options['fromrun']) && ($options['fromrun'] > $parallelrun)) { echo "From run (" . $options['fromrun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL; exit(1); } // Default torun is maximum parallel runs and should be less than equal to parallelruns. if (empty($options['torun'])) { $options['torun'] = $parallelrun; } else { if ($options['torun'] > $parallelrun) { echo "To run (" . $options['torun'] . ") is more than site with parallel runs (" . $parallelrun . ")" . PHP_EOL; exit(1); } } } // Capture signals and ensure we clean symlinks. if (extension_loaded('pcntl')) { $disabled = explode(',', ini_get('disable_functions')); if (!in_array('pcntl_signal', $disabled)) { pcntl_signal(SIGTERM, "signal_handler"); pcntl_signal(SIGINT, "signal_handler"); } } $time = microtime(true); array_walk($unrecognised, function (&$v) { if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1=\"\$2\"", $v)) { $v = $x; } else if (!preg_match("#^-#", $v)) { $v = escapeshellarg($v); } }); $extraopts = $unrecognised; if ($options['profile']) { $profile = $options['profile']; // If profile passed is not set, then exit (note we skip if the 'replace' option is found within the 'profile' value). if (!isset($CFG->behat_config[$profile]) && !isset($CFG->behat_profiles[$profile]) && !($options['replace'] && (strpos($profile, (string) $options['replace']) !== false))) { echo "Invalid profile passed: " . $profile . PHP_EOL; exit(1); } $extraopts['profile'] = '--profile="' . $profile . '"'; // By default, profile tags will be used. if (!empty($CFG->behat_config[$profile]['filters']['tags'])) { $tags = $CFG->behat_config[$profile]['filters']['tags']; } } // Command line tags have precedence (std behat behavior). if ($options['tags']) { $tags = $options['tags']; $extraopts['tags'] = '--tags="' . $tags . '"'; } // Add suite option if specified. if ($options['suite']) { $extraopts['suite'] = '--suite="' . $options['suite'] . '"'; } // Feature should be added to last, for behat command. if ($options['feature']) { $extraopts['feature'] = $options['feature']; // Only run 1 process as process. // Feature file is picked from absolute path provided, so no need to check for behat.yml. $options['torun'] = $options['fromrun']; } // Set of options to pass to behat. $extraoptstr = implode(' ', $extraopts); // If rerun is passed then ensure we just run the failed processes. $lastfailedstatus = 0; $lasttorun = $options['torun']; $lastfromrun = $options['fromrun']; if ($options['rerun']) { // Get last combined failed status. $lastfailedstatus = behat_config_manager::get_behat_run_config_value('lastcombinedfailedstatus'); $lasttorun = behat_config_manager::get_behat_run_config_value('lasttorun'); $lastfromrun = behat_config_manager::get_behat_run_config_value('lastfromrun'); if ($lastfailedstatus !== false) { $extraoptstr .= ' --rerun'; } // If torun is less than last torun, then just set this to min last to run and similar for fromrun. if ($options['torun'] < $lasttorun) { $options['torun']; } if ($options['fromrun'] > $lastfromrun) { $options['fromrun']; } unset($options['rerun']); } $cmds = array(); $exitcodes = array(); $status = 0; $verbose = empty($options['verbose']) ? false : true; // Execute behat run commands. if (empty($parallelrun)) { $cwd = getcwd(); chdir(__DIR__); $runtestscommand = behat_command::get_behat_command(false, false, true); $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath(); $runtestscommand .= ' ' . $extraoptstr; $cmds['singlerun'] = $runtestscommand; echo "Running single behat site:" . PHP_EOL; passthru("php $runtestscommand", $status); $exitcodes['singlerun'] = $status; chdir($cwd); } else { echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL; for ($i = $options['fromrun']; $i <= $options['torun']; $i++) { $lastfailed = 1 & $lastfailedstatus >> ($i - 1); // Bypass if not failed in last run. if ($lastfailedstatus && !$lastfailed && ($i <= $lasttorun) && ($i >= $lastfromrun)) { continue; } $CFG->behatrunprocess = $i; // Options parameters to be added to each run. $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraoptstr) : $extraoptstr; $behatcommand = behat_command::get_behat_command(false, false, true); $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i); // Command to execute behat run. $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts; echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL; } if (empty($cmds)) { echo "No commands to execute " . PHP_EOL; exit(1); } // Create site symlink if necessary. if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) { echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL; exit(1); } // Save torun and from run, so it can be used to detect if it was executed in last run. behat_config_manager::set_behat_run_config_value('lasttorun', $options['torun']); behat_config_manager::set_behat_run_config_value('lastfromrun', $options['fromrun']); // Keep no delay by default, between each parallel, let user decide. if (!defined('BEHAT_PARALLEL_START_DELAY')) { define('BEHAT_PARALLEL_START_DELAY', 0); } // Execute all commands, relative to moodle root directory. $processes = cli_execute_parallel($cmds, __DIR__ . "/../../../../", BEHAT_PARALLEL_START_DELAY); $stoponfail = empty($options['stop-on-failure']) ? false : true; // Print header. print_process_start_info($processes); // Print combined run o/p from processes. $exitcodes = print_combined_run_output($processes, $stoponfail); // Time to finish run.
< $time = round(microtime(true) - $time, 1);
> $time = round(microtime(true) - $time, 0);
echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL; ksort($exitcodes); // Print exit info from each run. // Status bits contains pass/fail status of parallel runs. foreach ($exitcodes as $name => $exitcode) { if ($exitcode) { $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name); $status |= (1 << ($runno - 1)); } } // Print each process information. print_each_process_info($processes, $verbose, $status); } // Save final exit code containing which run failed. behat_config_manager::set_behat_run_config_value('lastcombinedfailedstatus', $status); // Show exit code from each process, if any process failed and how to rerun failed process. if ($verbose || $status) { // Check if status of last run is failure and rerun is suggested. if (!empty($options['auto-rerun']) && $status) { // Rerun for the number of tries passed. for ($i = 0; $i < $options['auto-rerun']; $i++) { // Run individual commands, to avoid parallel failures. foreach ($exitcodes as $behatrunname => $exitcode) { // If not failed in last run, then skip. if ($exitcode == 0) { continue; } // This was a failure. echo "*** Re-running behat run: $behatrunname ***" . PHP_EOL; if ($verbose) { echo "Executing: " . $cmds[$behatrunname] . " --rerun" . PHP_EOL; } passthru("php $cmds[$behatrunname] --rerun", $rerunstatus); // Update exit code. $exitcodes[$behatrunname] = $rerunstatus; } } // Update status after auto-rerun finished. $status = 0; foreach ($exitcodes as $name => $exitcode) { if ($exitcode) { if (!empty($parallelrun)) { $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name); } else { $runno = 1; } $status |= (1 << ($runno - 1)); } } } // Show final o/p with re-run commands. if ($status) { if (!empty($parallelrun)) { // Echo exit codes. echo "Exit codes for each behat run: " . PHP_EOL; foreach ($exitcodes as $run => $exitcode) { echo $run . ": " . $exitcode . PHP_EOL; } unset($extraopts['fromrun']); unset($extraopts['torun']); if (!empty($options['replace'])) { $extraopts['replace'] = '--replace="' . $options['replace'] . '"'; } } echo "To re-run failed processes, you can use following command:" . PHP_EOL; $extraopts['rerun'] = '--rerun'; $extraoptstr = implode(' ', $extraopts); echo behat_command::get_behat_command(true, true, true) . " " . $extraoptstr . PHP_EOL; } echo PHP_EOL; } // Remove site symlink if necessary. behat_config_manager::drop_parallel_site_links(); exit($status); /** * Signal handler for terminal exit. * * @param int $signal signal number. */ function signal_handler($signal) { switch ($signal) { case SIGTERM: case SIGKILL: case SIGINT: // Remove site symlink if necessary. behat_config_manager::drop_parallel_site_links(); exit(1); } } /** * Prints header from the first process. * * @param array $processes list of processes to loop though. */ function print_process_start_info($processes) { $printed = false; // Keep looping though processes, till we get first process o/p. while (!$printed) { usleep(10000); foreach ($processes as $name => $process) { // Exit if any process has stopped. if (!$process->isRunning()) { $printed = true; break; } $op = explode(PHP_EOL, $process->getOutput()); if (count($op) >= 3) { foreach ($op as $line) { if (trim($line) && (strpos($line, '.') !== 0)) { echo $line . PHP_EOL; } } $printed = true; } } } } /** * Loop though all processes and print combined o/p * * @param array $processes list of processes to loop though. * @param bool $stoponfail Stop all processes and exit if failed. * @return array list of exit codes from all processes. */ function print_combined_run_output($processes, $stoponfail = false) { $exitcodes = array(); $maxdotsonline = 70; $remainingprintlen = $maxdotsonline; $progresscount = 0; while (count($exitcodes) != count($processes)) { usleep(10000); foreach ($processes as $name => $process) { if ($process->isRunning()) { $op = $process->getIncrementalOutput(); if (trim($op)) { $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op); // Exit process if anything fails. if ($stoponfail && (strpos($update, 'F') !== false)) { $process->stop(0); }
< $strlentoprint = strlen($update);
> $strlentoprint = strlen($update ?? '');
// If not enough dots printed on line then just print. if ($strlentoprint < $remainingprintlen) { echo $update; $remainingprintlen = $remainingprintlen - $strlentoprint; } else if ($strlentoprint == $remainingprintlen) { $progresscount += $maxdotsonline; echo $update ." " . $progresscount . PHP_EOL; $remainingprintlen = $maxdotsonline; } else { while ($part = substr($update, 0, $remainingprintlen) > 0) { $progresscount += $maxdotsonline; echo $part . " " . $progresscount . PHP_EOL; $update = substr($update, $remainingprintlen); $remainingprintlen = $maxdotsonline; } } } } else { $exitcodes[$name] = $process->getExitCode(); if ($stoponfail && ($exitcodes[$name] != 0)) { foreach ($processes as $l => $p) { $exitcodes[$l] = -1; $process->stop(0); } } } } } echo PHP_EOL; return $exitcodes; } /** * Loop though all processes and print combined o/p * * @param array $processes list of processes to loop though. * @param bool $verbose Show verbose output for each process. */ function print_each_process_info($processes, $verbose = false, $status = 0) { foreach ($processes as $name => $process) { echo "**************** [" . $name . "] ****************" . PHP_EOL; if ($verbose) { echo $process->getOutput(); echo $process->getErrorOutput(); } else if ($status) { // Only show failed o/p. $runno = str_replace(BEHAT_PARALLEL_SITE_NAME, '', $name); if ((1 << ($runno - 1)) & $status) { echo $process->getOutput(); echo $process->getErrorOutput(); } else { echo get_status_lines_from_run_op($process); } } else { echo get_status_lines_from_run_op($process); } echo PHP_EOL; } } /** * Extract status information from behat o/p and return. * @param Symfony\Component\Process\Process $process * @return string */ function get_status_lines_from_run_op(Symfony\Component\Process\Process $process) { $statusstr = ''; $op = explode(PHP_EOL, $process->getOutput()); foreach ($op as $line) { // Don't print progress . if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) && (strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) && (strpos($line, 'Browser specific fixes ') !== 0)) { $statusstr .= $line . PHP_EOL; } } return $statusstr; }