Arquivos
hhvm/hphp/test/run
T
Herman Venter 736f997ba8 Use PHP test framework for testing hphpd in sandbox server mode
Debugger tests that run hphp in non local mode (i.e. with the "-h some-server:some-port" option specified) currently use a clunky specialized C++ test harness that invokes PHP scripts that talk to the debugger client via the obsolete client API. This revision moves those tests to the new PHP only framework. The debugger client is now controlled by piping in commands, just like the other debugger tests. The main difference is that the debugger client is connected to a server instance and a separate PHP driver is used to load the server, load the client, and then load pages from the server so that the client can hit breakpoints. It also checks that the client can respond to ctrl-c by actually sending a SIGINT signal to the client process.
2013-07-22 11:34:06 -07:00

722 linhas
19 KiB
PHP
Arquivo Executável

#!/usr/bin/env php
<?php
/**
* Run the test suites in various configurations.
*/
function usage() {
global $argv;
return "usage: $argv[0] [-m jit|interp] [-r] <test/directories>";
}
function help() {
global $argv;
$ztestexample = 'test/zend/good/*/*z*.php'; // sep. for syntax highlighting
$help = <<<EOT
This is the hhvm test-suite runner. For more detailed documentation,
see hphp/test/README.md.
The test argument may be a path to a php test file, a directory name, or
one of a few pre-defined suite names that this script knows about.
If you work with hhvm a lot, you might consider a bash alias:
alias ht="path/to/fbcode/hphp/test/run"
Examples:
# Quick tests in JIT mode:
% $argv[0] test/quick
# Slow tests in interp mode:
% $argv[0] -m interp test/slow
# Slow closure tests in JIT mode:
% $argv[0] test/slow/closure
# Slow closure tests in JIT mode with RepoAuthoritative:
% $argv[0] -r test/slow/closure
# Slow array tests, in RepoAuthoritative:
% $argv[0] -r test/slow/array
# Zend tests with a "z" in their name:
% $argv[0] $ztestexample
# Quick tests in JIT mode with some extra runtime options:
% $argv[0] test/quick -a '-vEval.JitMaxTranslations=120 -vEval.HHIRJumpOpts=0'
EOT;
return usage().$help;
}
function error($message) {
print "$message\n";
exit(1);
}
function hphp_home() {
return realpath(__DIR__.'/../..');
}
function idx($array, $key, $default = null) {
return isset($array[$key]) ? $array[$key] : $default;
}
function idx_file($array, $key, $default = null) {
$file = is_file(idx($array, $key)) ? realpath($array[$key]) : $default;
if (!is_file($file)) {
error("$file doesn't exist");
}
return rel_path($file);
}
function bin_root() {
$dir = hphp_home() . '/' . idx($_ENV, 'FBMAKE_BIN_ROOT', '_bin');
return is_dir($dir) ?
$dir : # fbmake
hphp_home() # github
;
}
function verify_hhbc() {
return idx($_ENV, 'VERIFY_HHBC', bin_root().'/verify.hhbc');
}
function read_file($file) {
return file_exists($file) ?
preg_replace('/\s+/', ' ', (file_get_contents($file))) : "";
}
// http://stackoverflow.com/questions/2637945/
function rel_path($to) {
$from = explode('/', getcwd().'/');
$to = explode('/', $to);
$relPath = $to;
foreach($from as $depth => $dir) {
// find first non-matching dir
if($dir === $to[$depth]) {
// ignore this directory
array_shift($relPath);
} else {
// get number of remaining dirs to $from
$remaining = count($from) - $depth;
if($remaining > 1) {
// add traversals up to first matching dir
$padLength = (count($relPath) + $remaining - 1) * -1;
$relPath = array_pad($relPath, $padLength, '..');
break;
} else {
$relPath[0] = './' . $relPath[0];
}
}
}
return implode('/', $relPath);
}
function get_options($argv) {
$parameters = array(
'repo' => 'r',
'mode:' => 'm:',
'server' => '',
'help' => 'h',
'verbose' => 'v',
'fbmake' => '',
'threads:' => '',
'args:' => 'a:',
);
$options = array();
$files = array();
for ($i = 1; $i < count($argv); $i++) {
$arg = $argv[$i];
$found = false;
if ($arg && $arg[0] == '-') {
foreach ($parameters as $long => $short) {
if ($arg == '-'.str_replace(':', '', $short) ||
$arg == '--'.str_replace(':', '', $long)) {
if (substr($long, -1, 1) == ':') {
$value = $argv[++$i];
} else {
$value = true;
}
$options[str_replace(':', '', $long)] = $value;
$found = true;
break;
}
}
}
if (!$found && $arg) {
$files[] = $arg;
}
}
return array($options, $files);
}
/*
* We support some 'special' file names, that just know where the test
* suites are, to avoid typing 'hphp/test/foo'.
*/
function map_convenience_filename($file) {
$mappage = array(
'quick' => 'hphp/test/quick',
'slow' => 'hphp/test/slow',
'debugger' => 'hphp/test/server/debugger/tests',
'zend' => 'hphp/test/zend/good',
'zend_bad' => 'hphp/test/zend/bad',
'facebook' => 'hphp/facebook/test',
);
if (!isset($mappage[$file])) {
return $file;
}
return hphp_home().'/'.$mappage[$file];
}
function find_tests($files) {
if (!$files) {
$files = array('quick');
}
if ($files == array('all')) {
$files = array('quick', 'slow', 'zend');
}
foreach ($files as &$file) {
$file = map_convenience_filename($file);
if (!@stat($file)) {
error("Not valid file or directory: '$file'");
}
$file = preg_replace(',//+,', '/', realpath($file));
$file = preg_replace(',^'.getcwd().'/,', '', $file);
}
$files = implode(' ', $files);
$tests = explode("\n", shell_exec(
"find $files -name '*.php' -o -name '*.hhas'"
));
if (!$tests) {
error(usage());
}
asort($tests);
return array_filter($tests);
}
function find_test_ext($test, $ext) {
if (is_file("{$test}.{$ext}")) {
return "{$test}.{$ext}";
}
return find_file_for_dir(dirname($test), "config.{$ext}");
}
function find_file($test, $name) {
return find_file_for_dir(dirname($test), $name);
}
function find_file_for_dir($dir, $name) {
while (($dir !== '.') && is_dir($dir)) {
$file = "$dir/$name";
if (is_file($file)) {
return $file;
}
$dir = dirname($dir);
}
$file = __DIR__.'/'.$name;
if (file_exists($file)) {
return $file;
}
return null;
}
function find_debug_config($test, $name) {
$debug_config = find_file_for_dir(dirname($test), $name);
if ($debug_config !== null) {
return "-m debug --debug-config ".$debug_config;
}
return "";
}
function mode_cmd($options) {
$repo_args = "-vRepo.Local.Mode=-- -vRepo.Central.Path=".verify_hhbc();
$jit_args = "$repo_args -vEval.Jit=true";
$mode = idx($options, 'mode');
switch ($mode) {
case '':
case 'jit':
return "$jit_args";
case 'interp':
return "$repo_args -vEval.Jit=0";
default:
error("-m must be one of jit | interp. Got: '$mode'");
}
}
function extra_args($options) {
return idx($options, 'args', '');
}
// Return the command and the env to run it in.
function hhvm_cmd($options, $test, $test_run = null) {
if ($test_run === null) {
$test_run = $test;
}
$cmd = implode(" ", array(
idx_file($_ENV, 'HHVM_BIN', bin_root().'/hphp/hhvm/hhvm'),
'--config',
find_test_ext($test, 'hdf'),
find_debug_config($test, 'hphpd.hdf'),
mode_cmd($options),
'-vEval.EnableArgsInBacktraces=true',
read_file(find_test_ext($test, 'opts')),
extra_args($options),
'--file',
escapeshellarg($test_run)
));
$env = $_ENV;
$in = find_test_ext($test, 'in');
if ($in !== null) {
$cmd .= ' < ' . escapeshellarg($in);
// If we're piping the input into the command then setup a simple
// dumb terminal so hhvm doesn't try to control it and pollute the
// output with control characters, which could change depending on
// a wide variety of terminal settings.
$env["TERM"] = "dumb";
}
return array($cmd, $env);
}
function hphp_cmd($options, $test) {
return implode(" ", array(
idx_file($_ENV, 'HPHP_BIN', bin_root().'/hphp/hhvm/hphp'),
'--config',
find_file($test, 'hphp_config.hdf'),
read_file("$test.hphp_opts"),
"-thhbc -l0 -k1 -o $test.repo $test",
));
}
class Status {
private static $results = array();
private static $mode = 0;
const MODE_NORMAL = 0;
const MODE_VERBOSE = 1;
const MODE_FBMAKE = 2;
public static function setMode($mode) {
self::$mode = $mode;
}
public static function pass($test) {
array_push(self::$results, array('name' => $test, 'status' => 'passed'));
switch (self::$mode) {
case self::MODE_NORMAL:
if (self::hasColor()) {
print "\033[1;32m.\033[0m";
} else {
print '.';
}
break;
case self::MODE_VERBOSE:
if (self::hasColor()) {
print "$test \033[1;32mpassed\033[0m\n";
} else {
print "$test passed";
}
break;
case self::MODE_FBMAKE:
self::sayFBMake($test, 'passed');
break;
}
}
public static function skip($test) {
switch (self::$mode) {
case self::MODE_NORMAL:
if (self::hasColor()) {
print "\033[1;33ms\033[0m";
} else {
print 's';
}
break;
case self::MODE_VERBOSE:
if (self::hasColor()) {
print "$test \033[1;33mskipped\033[0m\n";
} else {
print "$test skipped";
}
break;
case self::MODE_FBMAKE:
// Say nothing...
break;
}
}
public static function fail($test) {
array_push(self::$results, array(
'name' => $test,
'status' => 'failed',
'details' => @file_get_contents("$test.diff")
));
switch (self::$mode) {
case self::MODE_NORMAL:
$diff = @file_get_contents($test.'.diff');
if (self::hasColor()) {
print "\n\033[0;31m$test\033[0m\n$diff";
} else {
print "\nFAILED: $test\n$diff";
}
break;
case self::MODE_VERBOSE:
if (self::hasColor()) {
print "$test \033[0;31mFAILED\033[0m\n";
} else {
print "$test FAILED\n";
}
break;
case self::MODE_FBMAKE:
self::sayFBMake($test, 'failed');
break;
}
}
private static function sayFBMake($test, $status) {
$start = array('op' => 'start', 'test' => $test);
$end = array('op' => 'test_done', 'test' => $test, 'status' => $status);
if ($status == 'failed') {
$end['details'] = @file_get_contents("$test.diff");
}
self::say($start, $end);
}
public static function getResults() {
return self::$results;
}
/** Output is in the format expected by JsonTestRunner. */
public static function say(/* ... */) {
$data = array_map(function($row) {
return json_encode($row, JSON_UNESCAPED_SLASHES) . "\n";
}, func_get_args());
fwrite(STDERR, implode("", $data));
}
private static function hasColor() {
return posix_isatty(STDOUT);
}
}
function run($options, $tests, $bad_test_file) {
if (isset($options['verbose'])) {
Status::setMode(Status::MODE_VERBOSE);
}
if (isset($options['fbmake'])) {
Status::setMode(Status::MODE_FBMAKE);
}
foreach ($tests as $test) {
$status = run_test($options, $test);
if ($status === 'skip') {
Status::skip($test);
} else if ($status) {
Status::pass($test);
} else {
Status::fail($test);
}
}
file_put_contents($bad_test_file, json_encode(Status::getResults()));
foreach (Status::getResults() as $result) {
if ($result['status'] == 'failed') {
return 1;
}
}
return 0;
}
function skip_test($options, $test) {
$skipif_test = find_test_ext($test, 'skipif');
if (!$skipif_test) {
return false;
}
list($hhvm, $_) = hhvm_cmd($options, $test, $skipif_test);
$out = shell_exec($hhvm . ' 2> /dev/null');
$out = trim($out);
return (bool)strlen($out);
}
function run_test($options, $test) {
list($hhvm, $hhvm_env) = hhvm_cmd($options, $test);
$hhvm = __DIR__."/../tools/timeout.sh -t 300 $hhvm";
$output = "";
if (skip_test($options, $test)) {
return 'skip';
}
if (isset($options['repo'])) {
if (strpos($test, '.hhas') !== false ||
strpos($hhvm, '-m debug') != false ||
file_exists($test.'.norepo')) {
return 'skip';
} else {
unlink("$test.repo/hhvm.hhbc");
$hphp = hphp_cmd($options, $test);
$output .= shell_exec("$hphp 2>&1");
$hhvm .= " -vRepo.Authoritative=true ".
"-vRepo.Central.Path=$test.repo/hhvm.hhbc";
}
}
$descriptorspec = array(
0 => array("pipe", "r"),
1 => array("pipe", "w"),
2 => array("pipe", "w"),
);
$process = proc_open("$hhvm 2>&1", $descriptorspec, $pipes, null, $hhvm_env);
if (!is_resource($process)) {
file_put_contents("$test.diff", "Couldn't invoke $hhvm");
return false;
}
fclose($pipes[0]);
$output .= stream_get_contents($pipes[1]);
file_put_contents("$test.out", $output);
fclose($pipes[1]);
// hhvm redirects errors to stdout, so anything on stderr is really bad
$stderr = stream_get_contents($pipes[2]);
if ($stderr) {
file_put_contents(
"$test.diff",
"Test failed because the process wrote on stderr:\n$stderr"
);
return false;
}
fclose($pipes[2]);
proc_close($process);
// Needed for testing non-hhvm binaries that don't actually run the code
// e.g. util/parser/test/parse_tester.cpp
if ($output == "FORCE PASS") {
return true;
}
if (file_exists("$test.expect")) {
$diff_cmds = "--text -u";
exec("diff --text -u $test.expect $test.out > $test.diff 2>&1",
$_, $status);
// unix 0 == success
return !$status;
} else if (file_exists("$test.expectf")) {
$wanted_re = file_get_contents("$test.expectf");
// do preg_quote, but miss out any %r delimited sections
$temp = "";
$r = "%r";
$startOffset = 0;
$length = strlen($wanted_re);
while($startOffset < $length) {
$start = strpos($wanted_re, $r, $startOffset);
if ($start !== false) {
// we have found a start tag
$end = strpos($wanted_re, $r, $start+2);
if ($end === false) {
// unbalanced tag, ignore it.
$end = $start = $length;
}
} else {
// no more %r sections
$start = $end = $length;
}
// quote a non re portion of the string
$temp = $temp.preg_quote(substr($wanted_re, $startOffset,
($start - $startOffset)), '/');
// add the re unquoted.
if ($end > $start) {
$temp = $temp.'('.substr($wanted_re, $start+2, ($end - $start-2)).')';
}
$startOffset = $end + 2;
}
$wanted_re = $temp;
$wanted_re = str_replace(
array('%binary_string_optional%'),
'string',
$wanted_re
);
$wanted_re = str_replace(
array('%unicode_string_optional%'),
'string',
$wanted_re
);
$wanted_re = str_replace(
array('%unicode\|string%', '%string\|unicode%'),
'string',
$wanted_re
);
$wanted_re = str_replace(
array('%u\|b%', '%b\|u%'),
'',
$wanted_re
);
// Stick to basics
$wanted_re = str_replace('%e', '\\' . DIRECTORY_SEPARATOR, $wanted_re);
$wanted_re = str_replace('%s', '[^\r\n]+', $wanted_re);
$wanted_re = str_replace('%S', '[^\r\n]*', $wanted_re);
$wanted_re = str_replace('%a', '.+', $wanted_re);
$wanted_re = str_replace('%A', '.*', $wanted_re);
$wanted_re = str_replace('%w', '\s*', $wanted_re);
$wanted_re = str_replace('%i', '[+-]?\d+', $wanted_re);
$wanted_re = str_replace('%d', '\d+', $wanted_re);
$wanted_re = str_replace('%x', '[0-9a-fA-F]+', $wanted_re);
$wanted_re = str_replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?',
$wanted_re);
$wanted_re = str_replace('%c', '.', $wanted_re);
// %f allows two points "-.0.0" but that is the best *simple* expression
# a poor man's aide for debugging
shell_exec("diff --text -u $test.expectf $test.out > $test.diff 2>&1");
// Normalize newlines
$wanted_re = preg_replace("/(\r\n?|\n)/", "\n", $wanted_re);
$output = preg_replace("/(\r\n?|\n)/", "\n", $output);
return preg_match("/^$wanted_re\$/s", $output);
} else if (file_exists("$test.expectregex")) {
$wanted_re = file_get_contents("$test.expectregex");
# a poor man's aide for debugging
shell_exec("diff --text -u $test.expectregex $test.out > $test.diff 2>&1");
return preg_match("/^$wanted_re\$/s", $output);
}
}
function num_cpus() {
$data = file('/proc/stat');
$cores = 0;
foreach($data as $line) {
if (preg_match('/^cpu[0-9]/', $line)) {
$cores++;
}
}
return $cores;
}
function main($argv) {
ini_set('pcre.backtrack_limit', PHP_INT_MAX);
list($options, $files) = get_options($argv);
if (isset($options['help'])) {
error(help());
}
$tests = find_tests($files);
$threads = min(count($tests), idx($options, 'threads', num_cpus() + 1));
if (!isset($options['fbmake'])) {
print "Running ".count($tests)." tests in $threads threads\n";
}
# Try to construct the buckets so the test results are ready in approximately
# alphabetical order
$test_buckets = array();
$i = 0;
foreach ($tests as $test) {
$test_buckets[$i][] = $test;
$i = ($i + 1) % $threads;
}
# Spawn off worker threads
$children = array();
# A poor man's shared memory
$bad_test_files = array();
for ($i = 0; $i < $threads; $i++) {
$bad_test_file = tempnam('/tmp', 'test-run-');
$bad_test_files[] = $bad_test_file;
$pid = pcntl_fork();
if ($pid == -1) {
error('could not fork');
} else if ($pid) {
$children[] = $pid;
} else {
exit(run($options, $test_buckets[$i], $bad_test_file));
}
}
# Wait for the kids
$return_value = 0;
foreach ($children as $child) {
pcntl_waitpid($child, $status);
$return_value |= pcntl_wexitstatus($status);
}
$results = array();
foreach ($bad_test_files as $bad_test_file) {
$json = json_decode(file_get_contents($bad_test_file), true);
if (!is_array($json)) {
error(
"A test thread didn't send json to the controller. WTF. ".
"If your code isn't horribly broken, please complain loudly"
);
}
$results = array_merge($results, $json);
}
if (isset($options['fbmake'])) {
Status::say(array('op' => 'all_done', 'results' => $results));
} else {
if (!$return_value) {
print "\nAll tests passed.\n\n".<<<SHIP
| | |
)_) )_) )_)
)___))___))___)\
)____)____)_____)\\
_____|____|____|____\\\__
---------\ SHIP IT /---------
^^^^^ ^^^^^^^^^^^^^^^^^^^^^
^^^^ ^^^^ ^^^ ^^
^^^^ ^^^
SHIP
."\n";
} else {
$failed = array();
foreach ($results as $result) {
if ($result['status'] == 'failed') {
$failed[] = $result['name'];
}
}
asort($failed);
$header_start = "\n\033[0;33m";
$header_end = "\033[0m\n";
print "\n".count($failed)." tests failed\n";
print $header_start."See the diffs:".$header_end.
implode("\n", array_map(
function($test) { return 'cat '.$test.'.diff'; },
$failed))."\n";
print $header_start."Run these by hand:".$header_end;
foreach ($failed as $test) {
list($command, $_) = hhvm_cmd($options, $test);
if (isset($options['repo'])) {
$command .= " -vRepo.Authoritative=true ";
$command = str_replace(verify_hhbc(), "$test.repo/hhvm.hhbc",
$command);
$command = hphp_cmd($options, $test)."\n".$command."\n";
}
print "$command\n";
}
print $header_start."Re-run just the failing tests:".$header_end.
"$argv[0] ".implode(' ', $failed)."\n";
}
}
return $return_value;
}
exit(main($argv));