diff --git a/hphp/test/README.md b/hphp/test/README.md new file mode 100644 index 000000000..cd78ae7cc --- /dev/null +++ b/hphp/test/README.md @@ -0,0 +1,51 @@ +# Suites + +Tests are grouped into "suites". They are just directories. Suites can have +subdirectories if you want to group them even more. Running a suite will run +all sub-suites. + +* vm - The most common. Put your test here by default. +* zend/good - Passing tests from Zend's suite. +* zend/bad - Failing tests from Zend. Fix these and move them to zend/good. +* vm-perf - Some performance tests that aren't commonly run. + +# File Layout + +The format is the same as Zend's `.phpt` but instead of sections it is +separate files with the section name converted to an extension. This allows +you to easily run the .php file without first running the test suite. + +These are the allowed extensions: + +* .php - The source of the test. +* .expect - The exact string expected output. +* .expectf - The exact string expected output with formating characters. +* .expectregex - A regex that matches the output. +* .out - When you run the test, the output will be stored here. +* .diff - The diff for .expect tests. +* .hhas - HipHop Assembly. + +You must have one `.php`; one and only one of `.expect`, `.expectf`, and +`.expectregex`; and the rest are optional. + +Name your test in a descriptive manner and when in doubt break your test into +many files. You can use comments too so future engineers know if it is a real +breakage or they need to change the expected output. + +## Format Characters + +These can appear in `.expectf` files. + +| Char | Description | Regex +|------|--------------------------------------------|------- +| %e | Path separator | \/ +| %s | Any characters except newlines | [^\r\n]+ +| %S | Optionally any characters except newlines | [^\r\n]* +| %a | Any characters | .+ +| %A | Optionally any characters | .* +| %w | Optional whitespace | \s* +| %i | Integer with optional sign | [+-]?\d+ +| %d | Digits | \d+ +| %x | Hex | [0-9a-fA-F]+ +| %f | Float | [+-]?\.?\d+\.?\d|(?:[Ee][+-]?\d+)? +| %c | Character | . diff --git a/hphp/test/verify b/hphp/test/verify new file mode 100755 index 000000000..a23a98ce4 --- /dev/null +++ b/hphp/test/verify @@ -0,0 +1,761 @@ +#!/usr/bin/perl -w +################################################################################ +# +# Test harness. +# +################################################################################ + +use threads; +use threads::shared; +use Thread::Semaphore; + +# Shut off buffering. +select(STDOUT); +$| = 1; + +# +# Parse command-line arguments. +# +use Getopt::Long; +Getopt::Long::config("bundling"); # Allow -hv rather than forcing -h -v. + +# Set option defaults for optional arguments. +$opt_help = 0; +$opt_verbose = 0; +$opt_quiet = 0; +$opt_srcdir = "."; +$opt_objdir = "."; +$opt_command = '%1$s/%3$s'; +$opt_ustats = 0; +$opt_zero = 0; +$opt_server = 0; +$opt_port = 8080; +$opt_home = "."; +$opt_threads = `cat /proc/cpuinfo | grep '^processor' | wc -l`; +if ($opt_threads > 20) { + $opt_threads = 20; +} +$opt_reduce = ""; +$opt_coverage = ""; +$opt_hhvm = "hhvm"; +$opt_cpu_time_limit = 0; +$opt_real_time_limit = 0; +$opt_no_exp = 0; + +$opt_retval = +&GetOptions("h|help" => \$opt_help, + "v|verbose" => \$opt_verbose, + "q|quiet" => \$opt_quiet, + "s|srcdir=s" => \$opt_srcdir, + "o|objdir=s" => \$opt_objdir, + "u|ustats" => \$opt_ustats, + "c|command=s" => \$opt_command, + "z|zero" => \$opt_zero, + "server" => \$opt_server, + "port:i" => \$opt_port, + "home:s" => \$opt_home, + "threads=i" => \$opt_threads, + "r|reduce=s" => \$opt_reduce, + "coverage=s" => \$opt_coverage, + "no-exp" => \$opt_no_exp, + "hhvm=s" => \$opt_hhvm, + "cpu-time-limit=i" => \$opt_cpu_time_limit, + "real-time-limit=i" => \$opt_real_time_limit, + ); + +# Munge directory paths if necessary. +if (defined($opt_srcdir) && $opt_srcdir eq "") +{ + $opt_srcdir = "."; +} +if (defined($opt_objdir) && $opt_objdir eq "") +{ + $opt_objdir = "."; +} + +if ($opt_help) +{ + &usage(); + exit(0); +} + +if ($opt_retval == 0) +{ + &usage(); + exit 1; +} + +if ($opt_verbose && $opt_quiet) +{ + print STDERR "-v and -q are incompatible\n"; + &usage(); + exit 1; +} + +if ($#ARGV + 1 == 0) +{ + print STDERR "No tests specified\n"; + &usage(); + exit 1; +} + +if ($opt_verbose) +{ + print STDERR "Option values: h:$opt_help, v:$opt_verbose, " + . "s:\"$opt_srcdir\", o:\"$opt_objdir\" " + . "q:$opt_quiet, u:$opt_ustats, z:$opt_zero\n"; + printf STDERR "Tests (%d total): @ARGV\n", $#ARGV + 1; +} + +if ($opt_server) +{ + $command = sprintf ($opt_command, $opt_home, $opt_port); + $server_pid = fork(); + if ($server_pid == 0) + { + `sudo $command 2>&1 > /dev/null`; + } + else + { + # wait for server to warm up + `sleep 5`; + } +} + +if (!$opt_threads) { + $opt_threads = 1; +} + +# +# Create and print header. +# +@TSTATS = +( + "--------------------------------------------------------------------------\n", + "Test c_user c_system c_total chng\n", + " passed/FAILED h_user h_system h_total %% chng\n" + ); + +if (!$opt_quiet) +{ + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } +} + +# +# Run sequence test(s). +# +$total_utime = 0.0; # Total user time. +$total_stime = 0.0; # Total system time. +$total_hutime = 0.0; # Total historical user time. +$total_hstime = 0.0; # Total historical system time. +$total_ntime = 0.0; # Total time for tests that have historical data. + +@test_buckets = (); +%tests_okay = (); +%tests_num_failed_subtests = (); +%tests_num_subtests = (); +%tests_utime = (); +%tests_stime = (); +# Extra debugging info generated by reduce or coverage. +# We store them in tests_extra_dbg instead of directly printing them out is +# that the output might be interleaved when the tests are run concurrently. +%tests_extra_dbg = (); +share(%tests_okay); +share(%tests_num_failed_subtests); +share(%tests_num_subtests); +share(%tests_utime); +share(%tests_stime); +share(%tests_extra_dbg); +# Try to construct the buckets so the test results are ready in approximately +# alphabetical order +for ($i = 0; $i < $opt_threads; $i++) +{ + push @test_buckets, []; +} +$i = 0; +foreach $test (@ARGV) +{ + push @{$test_buckets[$i]}, $test; + $i = ($i + 1) % $opt_threads; +} + +# Spawn off worker threads +for ($i = 0; $i < $opt_threads; $i++) +{ + threads->create(\&run_tests, $test_buckets[$i]); +} + +foreach $test (@ARGV) +{ + # Strip out any whitespace in $test. + $test =~ s/^\s*(.*)\s*$/$1/; + + $okay = 1; + + my (@TSTATS); + my ($t_str); + @TSTATS = ("--------------------------------------------------------------------------\n"); + + $t_str = sprintf ("%s%s", $test, ' ' x (40 - length($test))); + @TSTATS = (@TSTATS, $t_str); + if (!$opt_quiet) + { + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } + } + + ($okay, $num_failed_subtests, $num_subtests, $utime, $stime) = wait_for_test($test); + # num_failed_subtests and num_subtests will be zero in the diff mode. + ($hutime, $hstime) = &print_stats($test, $okay, $num_failed_subtests, $num_subtests, $utime, $stime); + { + lock(%tests_okay); + print $tests_extra_dbg{$test}; + } + + # Print coverage on success, or run reduce on failure. + + $total_hutime += $hutime; + $total_hstime += $hstime; + + if ($okay) + { + $total_utime += $utime; + $total_stime += $stime; + } + else + { + @FAILED_TESTS = (@FAILED_TESTS, $test); + } + + # If there were historical data, add the run time to the total time to + # compare against the historical run time. + if (0 < ($hutime + $hstime)) + { + $total_ntime += $utime + $stime; + } +} + +# Print summary stats. +$tt_str = sprintf ("%d / %d passed (%5.2f%%%%)", + ($#ARGV + 1) - ($#FAILED_TESTS + 1), + $#ARGV + 1, + (($#ARGV + 1) - ($#FAILED_TESTS + 1)) + / ($#ARGV + 1) * 100); + +$t_str = sprintf ("Totals %7.2f %7.2f %7.2f" + . " %7.2f\n" + . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n", + $total_utime, $total_stime, $total_utime + $total_stime, + ($total_ntime - ($total_hutime + $total_hstime)), + $tt_str . ' ' x (40 - length($tt_str)), + $total_hutime, $total_hstime, $total_hutime + $total_hstime, + ($total_hutime + $total_hstime == 0.0) ? 0.0 : + (($total_ntime + - ($total_hutime + $total_hstime)) + / ($total_hutime + $total_hstime) * 100)); + +$thread_warning = ""; +if ($opt_threads > 1) { + $thread_warning = "WARNING: Multiple threads were used. CPU times " + . "are wildly inaccurate\n" +} + +@TSTATS = ("--------------------------------------------------------------------------\n", + $thread_warning, + $t_str, + "--------------------------------------------------------------------------\n" + ); +if (!$opt_quiet) +{ + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } + if (@FAILED_TESTS) { + printf STDOUT "failed tests:\n"; + } + foreach $fail (@FAILED_TESTS) + { + printf STDOUT "%50s\n", $fail; + } +} + + +if ($opt_server) +{ + kill 9, $server_pid; + `ps aux | grep hphpi | grep -v perl | cut -c10-15 | sudo xargs kill -9 \\ + 2>&1 >/dev/null` +} + +if ($#FAILED_TESTS >= 0) +{ + # One or more tests failed, so return an error. + exit 1; +} +# End of main execution. + +sub run_tests +{ + foreach my $test (@{$_[0]}) + { + my ($okay, $num_failed_subtests, $num_subtests, $utime, $stime, $extra_dbg) = run_test($test); + { + lock(%tests_okay); + $tests_okay{$test} = $okay; + $tests_num_failed_subtests{$test} = $num_failed_subtests; + $tests_num_subtests{$test} = $num_subtests; + $tests_utime{$test} = $utime; + $tests_stime{$test} = $stime; + $tests_extra_dbg{$test} = $extra_dbg; + cond_signal(%tests_okay); + } + } +} + +sub wait_for_test +{ + my ($test) = @_; + while (1) + { + lock(%tests_okay); + if (exists $tests_okay{$test}) { + return ($tests_okay{$test}, + $tests_num_failed_subtests{$test}, + $tests_num_subtests{$test}, + $tests_utime{$test}, + $tests_stime{$test}, + $tests_extra_dbg{$test}); + } + cond_wait(%tests_okay); + } +} + +sub run_test +{ + my ($test) = @_; + my ($okay) = 1; + my ($num_failed_subtests) = 0; + my ($num_subtests) = 0; + my ($tutime, $tstime); + my ($extra_dbg) = ""; + my ($utime, $stime, $cutime, $cstime); + + if ($opt_server) + { + $command = "GET http://localhost:" . $opt_port . "/" . $test; + } + else + { + $command = sprintf ($opt_command, $opt_srcdir, $opt_objdir, $test); + if ($opt_cpu_time_limit) + { + $command = "(ulimit -t " . $opt_cpu_time_limit . "; " + . "timeout " . $opt_real_time_limit . " " + . $command . ")"; + } + } + ($utime, $stime, $cutime, $cstime) = times; + my $default_filter = "$opt_srcdir/test/vm/default.filter"; + if (-e "$opt_srcdir/$test.filter") { + `$command 2>&1 | $default_filter | $opt_srcdir/$test.filter > $opt_objdir/$test.out`; + } else { + `$command 2>&1 | $default_filter > $opt_objdir/$test.out`; + } + ($utime, $stime, $tutime, $tstime) = times; + + # Subtract the before time from the after time. + $tutime -= $cutime; + $tstime -= $cstime; + + if ($opt_zero) + { + if ($?) + { + $okay = 0; + if ($opt_verbose) + { + print STDERR + "\"$opt_objdir/$test > $opt_objdir/$test.out 2>&1\" returned $?\n"; + } + } + } + + # Compare the results with the expected, and update "okay". + if (! -e "$opt_objdir/$test.out") { + $okay = 0; + if ($opt_verbose) + { + print STDERR + "Nonexistent output file \"$opt_objdir/$test.out\"\n"; + } + } + elsif (!$opt_no_exp && -e "$opt_srcdir/$test.exp") + { + # Diff mode. + $diff_args = "--text -u"; + if ($opt_server) + { + `diff $diff_args -I HipHop $opt_srcdir/$test.exp $opt_objdir/$test.out \\ + > $opt_objdir/$test.diff 2>&1`; + } + else + { + `diff $diff_args $opt_srcdir/$test.exp $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`; + } + if ($?) + { + # diff returns non-zero if there is a difference. + $okay = 0; + } + } + elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expect") + { + `diff $diff_args $opt_srcdir/$test.expect $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`; + if ($?) + { + # diff returns non-zero if there is a difference. + $okay = 0; + } + } + elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expectf") + { + open (FILE, "<$opt_objdir/$test.expectf"); + @lines = ; + close(FILE); + $wanted_re = join('', @lines); + + open (FILE, "<$opt_objdir/$test.out"); + @lines = ; + close(FILE); + $content = join('', @lines); + + # from run-tests.php + $temp = ""; + $r = "%r"; + $startOffset = 0; + $length = length($wanted_re); + while ($startOffset < $length) + { + $start = index($wanted_re, $r, $startOffset); + if ($start != -1) + { + $end = index($wanted_re, $r, $start+2); + if ($end == -1) + { + # unbalanced tag, ignore it. + $end = $start = $length; + } + } + else + { + $start = $end = $length; + } + + $temp = $temp . quotemeta(substr($wanted_re, $startOffset, $start - $startOffset)); + if ($end > $start) + { + $temp = $temp . '(' . substr($wanted_re, $start.2, $end - $start-2) . ')'; + } + + $startOffset = $end + 2; + } + $wanted_re = $temp; + + ## perl escapes % but we want to use %s as our replacement + $wanted_re =~ s/\\%/%/g; + + $wanted_re =~ s/%binary_string_optional%/string/g; + $wanted_re =~ s/%unicode_string_optional%/string/g; + $wanted_re =~ s/%unicode\\\|string%/string/g; + $wanted_re =~ s/%string\\\|unicode%/string/g; + $wanted_re =~ s/%u\\\|b%//g; + $wanted_re =~ s/%b\\\|u%//g; + + # Stick to basics; + $wanted_re =~ s/%e/\\\//g; + $wanted_re =~ s/%s/[^\r\n]+/g; + $wanted_re =~ s/%S/[^\r\n]*/g; + $wanted_re =~ s/%a/.+/g; + $wanted_re =~ s/%A/.*/g; + $wanted_re =~ s/%w/\\s*/g; + $wanted_re =~ s/%i/[+-]?\\d+/g; + $wanted_re =~ s/%d/\\d+/g; + $wanted_re =~ s/%x/[0-9a-fA-F]+/g; + $wanted_re =~ s/%f/[+-]?\\.?\\d+\\.?\\d*(?:[Ee][+-]?\\d+)?/g; + $wanted_re =~ s/%c/./g; + + unless ($content =~ /^$wanted_re$/) + { + $okay = 0; + } + } + elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expectregex") + { + open (FILE, "<$opt_objdir/$test.expectregex"); + @lines = ; + close(FILE); + $regex = join('', @lines); + + open (FILE, "<$opt_objdir/$test.out"); + @lines = ; + close(FILE); + $content = join('', @lines); + + unless ($content =~ /$regex/) + { + $okay = 0; + } + } + else + { + # Sequence mode. + if (open (STEST_OUT, "<$opt_objdir/$test.out")) + { + $num_subtests = 0; + $num_failed_subtests = 0; + + while (defined($line = )) + { + if ($line =~ /1\.\.(\d+)/) + { + $num_subtests = $1; + last; + } + } + if ($num_subtests == 0) + { + $okay = 0; + if ($opt_verbose) + { + print STDERR "Malformed or missing 1..n line\n"; + } + } + else + { + for ($subtest = 1; $subtest <= $num_subtests; $subtest++) + { + while (defined($line = )) + { + if ($line =~ /^not\s+ok\s+(\d+)?/) + { + $not = 1; + $test_num = $1; + last; + } + elsif ($line =~ /^ok\s+(\d+)?/) + { + $not = 0; + $test_num = $1; + last; + } + } + if (defined($line)) + { + if (defined($test_num) && ($test_num != $subtest)) + { + # There was no output printed for one or more tests. + for (; $subtest < $test_num; $subtest++) + { + $num_failed_subtests++; + } + } + if ($not) + { + $num_failed_subtests++; + } + } + else + { + for (; $subtest <= $num_subtests; $subtest++) + { + $num_failed_subtests++; + } + } + } + + if (0 < $num_failed_subtests) + { + $okay = 0; + } + } + } + else + { + if (!$opt_quiet) + { + print STDERR "Cannot open output file \"$opt_objdir/$test.out\"\n"; + } + exit 1; + } + } + + if ($okay == 0) + { + # Run reduce to pinpoint the tracelet that causes the error. + # For now, reduce has to be run sequentially, because it copies the input + # file to the same temporary place. + if ($opt_reduce ne "") + { + $cmd = $opt_reduce . " '" . $opt_hhvm . "' " . $test; + if ($opt_cpu_time_limit) + { + $cmd = join(" ", + $cmd, + $opt_cpu_time_limit, + $opt_real_time_limit); + } + $extra_dbg = `$cmd 2> /dev/null`; + } + } + else + { + # Print the coverage. + # Need run the command again in the tracing mode. + if ($opt_coverage ne "") + { + $command = sprintf($opt_command, $opt_srcdir, $opt_objdir, $test); + # Need calculate # of spills, reloads, and pushes/pops + # around natives which are logged in level 3. + $command = join("", "TRACE=tx64:3 ", + "HPHP_TRACE_FILE=", $test, ".log ", + $command); + `$command 2> /dev/null`; + $extra_dbg = `$opt_coverage < $test.log 2> /dev/null`; + } + } + + return ($okay, $num_failed_subtests, $num_subtests, $tutime, + $tstime, $extra_dbg); +} + +sub print_stats +{ + my ($test, $okay, $failed_subtests, $subtests, $utime, $stime) = @_; + my ($hutime, $hstime); +# my (TEST_PERF); + my (@TSTATS); + my ($t_str, $pass_str); + + $pass_str = $okay ? "passed" : "*** FAILED ***"; + if ((0 != $subtests) && (!$okay)) + { + $pass_str = $pass_str . " ($failed_subtests/$subtests failed)"; + } + $pass_str = $pass_str . ' ' x (39 - length($pass_str)); + + if (-r "$test.perf") + { + if (!open (TEST_PERF, "<$opt_objdir/$test.perf")) + { + print STDERR "Unable to open \"$opt_objdir/$test.perf\"\n"; + exit 1; + } + $_ = ; + + ($hutime, $hstime) = split; + close TEST_PERF; + + $t_str = sprintf (" %7.2f %7.2f %7.2f %7.2f\n" + . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n", + $utime, $stime, $utime + $stime, + ($utime + $stime) - ($hutime + $hstime), + $pass_str, + $hutime, $hstime, $hutime + $hstime, + (($hutime + $hstime) == 0.0) ? 0.0 : + ((($utime + $stime) - ($hutime + $hstime)) + / ($hutime + $hstime) * 100)); + } + else + { + $hutime = 0.0; + $hstime = 0.0; + + $t_str = sprintf (" %7.2f %7.2f %7.2f \n" + . " %s\n", + $utime, $stime, $utime + $stime, + $pass_str); + } + @TSTATS = ($t_str); + if (!$opt_quiet) + { + foreach $line (@TSTATS) + { + printf STDOUT "$line"; + } + } + + if ($okay && $opt_ustats) + { + if (!open (TEST_PERF, ">$opt_objdir/$test.perf")) + { + if (!$opt_quiet) + { + print STDERR "Unable to update \"$opt_objdir/$test.perf\"\n"; + } + } + else + { + print TEST_PERF "$utime $stime\n"; + close TEST_PERF; + } + } + + return ($hutime, $hstime); +} + +sub usage +{ + print <] + + + Option | Description + --------------+------------------------------------------------------------- + -h --help | Print usage and exit. + -v --verbose | Verbose (incompatible with quiet). + -q --quiet | Quiet (incompatible with verbose). + -s --srcdir | Path to source tree (default is "."). + -o --objdir | Path to object tree (default is "."). + -c --command | Command template, where the following substitutions are + | performed: + | %1\$s : srcdir (see --srcdir) + | %2\$s : objdir (see --objdir) + | %3\$s : + | (default is '%1\$s/%3\$s'). + -r --reduce | The path to the reduce tool. + | Do not run reduce if it is empty. + | (default is empty) + --coverage | The path to the coverage tool. + | Do not run coverage if it is empty. + | (default is empty) + -t | Set the time limit for each test case + | (default is unlimited). + -u --ustats | Update historical statistics (stored in ".perf". + -z --zero | Consider non-zero exit code to be an error. + --no-exp | Don't diff with .exp files + --------------+------------------------------------------------------------- + + If .exp exists and --no-exp is not specified, 's output is + diff'ed with .exp. Any difference is considered failure. + + If .exp does not exist or --no-exp is specified, output to stdout of + the following form is expected: + + 1.. + {not }ok[ 1] + {not }ok[ 2] + ... + {not }ok[ n] + + 1 <= < 2^31 + + Lines which do not match the patterns shown above are ignored. +EOF +} +# vim:filetype=perl: diff --git a/hphp/tools/import_zend_test.py b/hphp/tools/import_zend_test.py index 2f925ecb3..fbd1649fa 100644 --- a/hphp/tools/import_zend_test.py +++ b/hphp/tools/import_zend_test.py @@ -6,10 +6,12 @@ then copies the good ones to test/zend/good and the bad ones to test/zend/bad. """ import argparse +import glob +import json import os import re -import subprocess import shutil +import subprocess import sys bad_tests = ( @@ -74,12 +76,6 @@ parser.add_argument( type=str, help="zend path to import tests from." ) -parser.add_argument( - "-n", - "--dont_run", - action='store_true', - help="don't run run_verify.sh. Just parse the .diff files." -) parser.add_argument( "-o", "--only", @@ -128,11 +124,15 @@ def walk(filename, source): for i in sections.keys(): sections[i] = '\n'.join(sections[i]) + unsupported_sections = ('INI', 'POST_RAW') + for name in unsupported_sections: + if sections.has_key(name): + print "Unsupported test with section --%s--: " % name, filename + return + if not sections.has_key('FILE'): print "Malformed test, no --FILE--: ", filename return - - test = sections['FILE'] dest_filename = os.path.basename(filename).replace('.phpt', '.php') source_dir = source.lower().replace('/tests', '').replace('/', '-') @@ -151,77 +151,40 @@ def walk(filename, source): exp = exp.replace('\n\nWarning:', '\nWarning:') exp = exp.replace('\n\nNotice:', '\nNotice:') + match_rest_of_line = '%a' + if key == 'EXPECTREGEX': + match_rest_of_line = '.+' + + exp = re.sub(r'Fatal\\? error\\?:.*', 'HipHop Fatal error: '+match_rest_of_line, exp) + exp = re.sub(r'Warning\\?:.*', 'HipHop Warning: '+match_rest_of_line, exp) + exp = re.sub(r'Notice\\?:.*', 'HipHop Notice: '+match_rest_of_line, exp) + for error in errors: exp = re.sub(error[0], error[1], exp) sections[key] = exp - def kill_error_messages(exp): - exp = re.sub(r'Fatal\\? error\\?:.*', 'HipHop Fatal error: .+', exp) - exp = re.sub(r'Warning\\?:.*', 'HipHop Warning: .+', exp) - exp = re.sub(r'Notice\\?:.*', 'HipHop Notice: .+', exp) - return exp + cur_dir = os.path.dirname(__file__) + dest_subdir = os.path.join(cur_dir, '../test/zend/all', source_dir) + mkdir_p(dest_subdir) + full_dest_filename = os.path.join(dest_subdir, dest_filename) if sections.has_key('EXPECT'): exp = sections['EXPECT'] - exp = kill_error_messages(exp) + # we use %a for error messages so always write expectf + file(full_dest_filename+'.expectf', 'w').write(exp) elif sections.has_key('EXPECTREGEX'): exp = sections['EXPECTREGEX'] - exp = kill_error_messages(exp) + file(full_dest_filename+'.expectregex', 'w').write(exp) elif sections.has_key('EXPECTF'): - wanted_re = sections['EXPECTF'] - - # from run-tests.php - temp = ""; - r = "%r"; - startOffset = 0; - length = len(wanted_re); - while startOffset < length: - start = wanted_re.find(r, startOffset) - if start != -1: - end = wanted_re.find(r, start+2); - if end == -1: - # unbalanced tag, ignore it. - end = start = length; - else: - start = end = length; - - temp = temp + re.escape(wanted_re[startOffset:start - startOffset]) - if (end > start): - temp = temp + '(' + wanted_re[start+2:end - start-2] + ')' - - startOffset = end + 2 - - wanted_re = temp - - ## different from php, since python escapes % - wanted_re = wanted_re.replace('\\%', '%') - wanted_re = kill_error_messages(wanted_re) - - wanted_re = wanted_re.replace('%binary_string_optional%', 'string') - wanted_re = wanted_re.replace('%unicode_string_optional%', 'string') - wanted_re = wanted_re.replace('%unicode\|string%', 'string') - wanted_re = wanted_re.replace('%string\|unicode%', 'string') - wanted_re = wanted_re.replace('%u\|b%', '') - wanted_re = wanted_re.replace('%b\|u%', '') - - # Stick to basics - wanted_re = wanted_re.replace('%e', '\\/') - wanted_re = wanted_re.replace('%s', '[^\r\n]+') - wanted_re = wanted_re.replace('%S', '[^\r\n]*') - wanted_re = wanted_re.replace('%a', '.+') - wanted_re = wanted_re.replace('%A', '.*') - wanted_re = wanted_re.replace('%w', '\s*') - wanted_re = wanted_re.replace('%i', '[+-]?\d+') - wanted_re = wanted_re.replace('%d', '\d+') - wanted_re = wanted_re.replace('%x', '[0-9a-fA-F]+') - wanted_re = wanted_re.replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?') - wanted_re = wanted_re.replace('%c', '.') - exp = wanted_re + exp = sections['EXPECTF'] + file(full_dest_filename+'.expectf', 'w').write(exp) else: print "Malformed test, no --EXPECT-- or --EXPECTF-- or --EXPECTREGEX--: ", filename return + test = sections['FILE'] + if sections.has_key('POST'): test = test.replace( '", "unlink('test.php');\n?>") if 'bug44805.php' in full_dest_filename: test = test.replace("1)) {\n\tunlink($file2", "2)) {\n\tunlink($file2") file(full_dest_filename, 'w').write(test) - file(full_dest_filename+'.exp', 'w').write(exp) if args.zend_path: test_dirs = (('Zend/tests'), ('tests'), ('sapi'), ('ext')) @@ -344,98 +295,55 @@ if not os.path.isdir('test/zend/all'): print "Running all tests from test/zend/bad" shutil.copytree('test/zend/bad', 'test/zend/all') -if not args.dont_run: - env = os.environ - env.update({'VQ':'interp', 'TEST_PATH':'zend/all'}) - stdout = open(os.devnull, 'wb') - if args.verbose: - stdout = None - proc = subprocess.Popen( - ['tools/run_verify.sh'], - env=env, - stdout=stdout, - stderr=subprocess.STDOUT +print "Running all tests from zend/all" + +stdout = subprocess.Popen( + [ + 'tools/verify_to_json.php', + 'run_verify.sh', + 'interp', + 'zend/all', + '_bin', + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT +).communicate()[0] + +# fbmake, you are crazy +results = json.loads('['+stdout.strip().replace("\n", ",\n")+']')[-1]['results'] + +if args.verbose: + print results + +for test in results: + filename = test['name'] + good_file = filename.replace('all', 'good', 1) + bad_file = filename.replace('all', 'bad', 1) + mkdir_p(os.path.dirname(good_file)) + mkdir_p(os.path.dirname(bad_file)) + + if test['status'] == 'passed': + dest_file = good_file + delete_file = bad_file + subpath = 'good' + else: + dest_file = bad_file + delete_file = good_file + subpath = 'bad' + + exps = glob.glob(filename+'.expect*') + if not exps: + # this file is probably generated while running tests :( + continue + + source_file_exp = exps[0] + _, dest_ext = os.path.splitext(source_file_exp) + os.rename(filename, dest_file) + file(dest_file+dest_ext, 'w').write( + file(source_file_exp).read().replace('/all', '/' + subpath) ) - proc.wait() - -for root, dirs, files in os.walk('test/zend/all'): - for filename in files: - if not filename.endswith('.php'): - continue - filename = os.path.join(root, filename) - - def all_exist(filename): - extensions = ('', '.out', '.exp') - for ext in extensions: - if not os.path.exists(filename+ext): - # something crazy is going on - return False - return True - if not all_exist(filename): - continue - - good_file = filename.replace('all', 'good', 1) - bad_file = filename.replace('all', 'bad', 1) - mkdir_p(os.path.dirname(good_file)) - mkdir_p(os.path.dirname(bad_file)) - - def isOkDiff(original_name): - global args - for test in bad_tests: - if test in original_name: - return False - - # no diff file or is empty - if (not os.path.exists(original_name + '.diff') or \ - os.stat(original_name + '.diff')[6] == 0): - if args.verbose: - print '\n', original_name, '\nNo .diff, passed' - return True - - # PHP is very inconsistent with whitespace in tests - diff = file(original_name + '.diff').read() - diff = re.sub(r'-(.*)\n\\ No newline at end of file\n\+\1', '', diff) - if not re.search(r'\n-', diff): - if args.verbose: - print '\n', original_name, '\nOnly whitespace .diff, passed' - return True - - # I hack a bit and store the regex in the .exp file, use that - wanted_re = file(original_name + '.exp').read().strip() - output = file(original_name + '.out').read().strip() - - import sre_constants - try: - match = re.match(wanted_re, output) - except (OverflowError, AssertionError, sre_constants.error) as e: - if args.verbose: - print '\n', original_name, '\nException', '\n', e - return False - - if args.verbose: - print '\n', original_name, '\n', repr(wanted_re), '\n', repr(output), '\n', match is not None - return match and match.group() == output - - - if isOkDiff(filename): - dest_file = good_file - source_file_exp = filename+'.out' - delete_file = bad_file - subpath = 'good' - else: - dest_file = bad_file - source_file_exp = filename+'.exp' - delete_file = good_file - subpath = 'bad' - - os.rename(filename, dest_file) - file(dest_file+'.exp', 'w').write( - file(source_file_exp).read().replace('/all', '/' + subpath) - ) - if os.path.exists(delete_file): - os.unlink(delete_file) - if os.path.exists(delete_file+'.exp'): - os.unlink(delete_file+'.exp') + for f in glob.glob(delete_file+"*"): + os.unlink(f) if not args.dirty: shutil.rmtree('test/zend/all') diff --git a/hphp/tools/run_verify.sh b/hphp/tools/run_verify.sh index 9310d5f49..57f2edba0 100755 --- a/hphp/tools/run_verify.sh +++ b/hphp/tools/run_verify.sh @@ -39,7 +39,7 @@ QTESTS_SKIP='autoload5.php condinfinite.php condinfinite2.php define_b.php include_backtrace2.php backup_cycle_collector.php redeclared_class1.php redeclared_class2.php' -VERIFY_SCRIPT=./test/verify.sh +VERIFY_SCRIPT=./test/verify ###################################################################### diff --git a/hphp/tools/run_verify_parse.sh b/hphp/tools/run_verify_parse.sh index 89eee9b08..52d03f68b 100755 --- a/hphp/tools/run_verify_parse.sh +++ b/hphp/tools/run_verify_parse.sh @@ -9,7 +9,7 @@ # : ${FBMAKE_BIN_ROOT=$HPHP_HOME/_bin} -VERIFY_SCRIPT=./test/verify.sh +VERIFY_SCRIPT=./test/verify PARSE_TEST=$FBMAKE_BIN_ROOT/hphp/util/parser/test/parse_tester # some tests are expected not to parse diff --git a/hphp/tools/verify_to_json.php b/hphp/tools/verify_to_json.php new file mode 100755 index 000000000..0f27f7db8 --- /dev/null +++ b/hphp/tools/verify_to_json.php @@ -0,0 +1,28 @@ +#!/bin/env php +