Support zend's test formats
Zend tests have 3 possible sections for test output. EXPECT, EXPECTF, and EXPECTREGEX. I think we should do the exact same thing. The .filter thing is difficult to write tests that have random numbers involved and the PHP community should already be familiar with the EXPECTF format strings. Migration plan: * Kill all .filter files * Rename all .exp to .expect * Stop supporting .exp I'm basically planning on suporting all of zend's sections as file extensions. Basically, if they have a section ##FOO## we will have a file ##test.php.foo## (where we need them). I changed the import script to use our json encoding instead of deciphering what happened from the ##.diff## files (since not everything will have a ##.diff## file now) We have 50 ##.filter## tests. I'll go convert them to expectf and see if they are easier to read.
Esse commit está contido em:
ptarjan
@@ -0,0 +1,51 @@
|
||||
# Suites
|
||||
|
||||
Tests are grouped into "suites". They are just directories. Suites can have
|
||||
subdirectories if you want to group them even more. Running a suite will run
|
||||
all sub-suites.
|
||||
|
||||
* vm - The most common. Put your test here by default.
|
||||
* zend/good - Passing tests from Zend's suite.
|
||||
* zend/bad - Failing tests from Zend. Fix these and move them to zend/good.
|
||||
* vm-perf - Some performance tests that aren't commonly run.
|
||||
|
||||
# File Layout
|
||||
|
||||
The format is the same as Zend's `.phpt` but instead of sections it is
|
||||
separate files with the section name converted to an extension. This allows
|
||||
you to easily run the .php file without first running the test suite.
|
||||
|
||||
These are the allowed extensions:
|
||||
|
||||
* .php - The source of the test.
|
||||
* .expect - The exact string expected output.
|
||||
* .expectf - The exact string expected output with formating characters.
|
||||
* .expectregex - A regex that matches the output.
|
||||
* .out - When you run the test, the output will be stored here.
|
||||
* .diff - The diff for .expect tests.
|
||||
* .hhas - HipHop Assembly.
|
||||
|
||||
You must have one `.php`; one and only one of `.expect`, `.expectf`, and
|
||||
`.expectregex`; and the rest are optional.
|
||||
|
||||
Name your test in a descriptive manner and when in doubt break your test into
|
||||
many files. You can use comments too so future engineers know if it is a real
|
||||
breakage or they need to change the expected output.
|
||||
|
||||
## Format Characters
|
||||
|
||||
These can appear in `.expectf` files.
|
||||
|
||||
| Char | Description | Regex
|
||||
|------|--------------------------------------------|-------
|
||||
| %e | Path separator | \/
|
||||
| %s | Any characters except newlines | [^\r\n]+
|
||||
| %S | Optionally any characters except newlines | [^\r\n]*
|
||||
| %a | Any characters | .+
|
||||
| %A | Optionally any characters | .*
|
||||
| %w | Optional whitespace | \s*
|
||||
| %i | Integer with optional sign | [+-]?\d+
|
||||
| %d | Digits | \d+
|
||||
| %x | Hex | [0-9a-fA-F]+
|
||||
| %f | Float | [+-]?\.?\d+\.?\d|(?:[Ee][+-]?\d+)?
|
||||
| %c | Character | .
|
||||
Arquivo executável
+761
@@ -0,0 +1,761 @@
|
||||
#!/usr/bin/perl -w
|
||||
################################################################################
|
||||
#
|
||||
# Test harness.
|
||||
#
|
||||
################################################################################
|
||||
|
||||
use threads;
|
||||
use threads::shared;
|
||||
use Thread::Semaphore;
|
||||
|
||||
# Shut off buffering.
|
||||
select(STDOUT);
|
||||
$| = 1;
|
||||
|
||||
#
|
||||
# Parse command-line arguments.
|
||||
#
|
||||
use Getopt::Long;
|
||||
Getopt::Long::config("bundling"); # Allow -hv rather than forcing -h -v.
|
||||
|
||||
# Set option defaults for optional arguments.
|
||||
$opt_help = 0;
|
||||
$opt_verbose = 0;
|
||||
$opt_quiet = 0;
|
||||
$opt_srcdir = ".";
|
||||
$opt_objdir = ".";
|
||||
$opt_command = '%1$s/%3$s';
|
||||
$opt_ustats = 0;
|
||||
$opt_zero = 0;
|
||||
$opt_server = 0;
|
||||
$opt_port = 8080;
|
||||
$opt_home = ".";
|
||||
$opt_threads = `cat /proc/cpuinfo | grep '^processor' | wc -l`;
|
||||
if ($opt_threads > 20) {
|
||||
$opt_threads = 20;
|
||||
}
|
||||
$opt_reduce = "";
|
||||
$opt_coverage = "";
|
||||
$opt_hhvm = "hhvm";
|
||||
$opt_cpu_time_limit = 0;
|
||||
$opt_real_time_limit = 0;
|
||||
$opt_no_exp = 0;
|
||||
|
||||
$opt_retval =
|
||||
&GetOptions("h|help" => \$opt_help,
|
||||
"v|verbose" => \$opt_verbose,
|
||||
"q|quiet" => \$opt_quiet,
|
||||
"s|srcdir=s" => \$opt_srcdir,
|
||||
"o|objdir=s" => \$opt_objdir,
|
||||
"u|ustats" => \$opt_ustats,
|
||||
"c|command=s" => \$opt_command,
|
||||
"z|zero" => \$opt_zero,
|
||||
"server" => \$opt_server,
|
||||
"port:i" => \$opt_port,
|
||||
"home:s" => \$opt_home,
|
||||
"threads=i" => \$opt_threads,
|
||||
"r|reduce=s" => \$opt_reduce,
|
||||
"coverage=s" => \$opt_coverage,
|
||||
"no-exp" => \$opt_no_exp,
|
||||
"hhvm=s" => \$opt_hhvm,
|
||||
"cpu-time-limit=i" => \$opt_cpu_time_limit,
|
||||
"real-time-limit=i" => \$opt_real_time_limit,
|
||||
);
|
||||
|
||||
# Munge directory paths if necessary.
|
||||
if (defined($opt_srcdir) && $opt_srcdir eq "")
|
||||
{
|
||||
$opt_srcdir = ".";
|
||||
}
|
||||
if (defined($opt_objdir) && $opt_objdir eq "")
|
||||
{
|
||||
$opt_objdir = ".";
|
||||
}
|
||||
|
||||
if ($opt_help)
|
||||
{
|
||||
&usage();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if ($opt_retval == 0)
|
||||
{
|
||||
&usage();
|
||||
exit 1;
|
||||
}
|
||||
|
||||
if ($opt_verbose && $opt_quiet)
|
||||
{
|
||||
print STDERR "-v and -q are incompatible\n";
|
||||
&usage();
|
||||
exit 1;
|
||||
}
|
||||
|
||||
if ($#ARGV + 1 == 0)
|
||||
{
|
||||
print STDERR "No tests specified\n";
|
||||
&usage();
|
||||
exit 1;
|
||||
}
|
||||
|
||||
if ($opt_verbose)
|
||||
{
|
||||
print STDERR "Option values: h:$opt_help, v:$opt_verbose, "
|
||||
. "s:\"$opt_srcdir\", o:\"$opt_objdir\" "
|
||||
. "q:$opt_quiet, u:$opt_ustats, z:$opt_zero\n";
|
||||
printf STDERR "Tests (%d total): @ARGV\n", $#ARGV + 1;
|
||||
}
|
||||
|
||||
if ($opt_server)
|
||||
{
|
||||
$command = sprintf ($opt_command, $opt_home, $opt_port);
|
||||
$server_pid = fork();
|
||||
if ($server_pid == 0)
|
||||
{
|
||||
`sudo $command 2>&1 > /dev/null`;
|
||||
}
|
||||
else
|
||||
{
|
||||
# wait for server to warm up
|
||||
`sleep 5`;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$opt_threads) {
|
||||
$opt_threads = 1;
|
||||
}
|
||||
|
||||
#
|
||||
# Create and print header.
|
||||
#
|
||||
@TSTATS =
|
||||
(
|
||||
"--------------------------------------------------------------------------\n",
|
||||
"Test c_user c_system c_total chng\n",
|
||||
" passed/FAILED h_user h_system h_total %% chng\n"
|
||||
);
|
||||
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
foreach $line (@TSTATS)
|
||||
{
|
||||
printf STDOUT "$line";
|
||||
}
|
||||
}
|
||||
|
||||
#
|
||||
# Run sequence test(s).
|
||||
#
|
||||
$total_utime = 0.0; # Total user time.
|
||||
$total_stime = 0.0; # Total system time.
|
||||
$total_hutime = 0.0; # Total historical user time.
|
||||
$total_hstime = 0.0; # Total historical system time.
|
||||
$total_ntime = 0.0; # Total time for tests that have historical data.
|
||||
|
||||
@test_buckets = ();
|
||||
%tests_okay = ();
|
||||
%tests_num_failed_subtests = ();
|
||||
%tests_num_subtests = ();
|
||||
%tests_utime = ();
|
||||
%tests_stime = ();
|
||||
# Extra debugging info generated by reduce or coverage.
|
||||
# We store them in tests_extra_dbg instead of directly printing them out is
|
||||
# that the output might be interleaved when the tests are run concurrently.
|
||||
%tests_extra_dbg = ();
|
||||
share(%tests_okay);
|
||||
share(%tests_num_failed_subtests);
|
||||
share(%tests_num_subtests);
|
||||
share(%tests_utime);
|
||||
share(%tests_stime);
|
||||
share(%tests_extra_dbg);
|
||||
# Try to construct the buckets so the test results are ready in approximately
|
||||
# alphabetical order
|
||||
for ($i = 0; $i < $opt_threads; $i++)
|
||||
{
|
||||
push @test_buckets, [];
|
||||
}
|
||||
$i = 0;
|
||||
foreach $test (@ARGV)
|
||||
{
|
||||
push @{$test_buckets[$i]}, $test;
|
||||
$i = ($i + 1) % $opt_threads;
|
||||
}
|
||||
|
||||
# Spawn off worker threads
|
||||
for ($i = 0; $i < $opt_threads; $i++)
|
||||
{
|
||||
threads->create(\&run_tests, $test_buckets[$i]);
|
||||
}
|
||||
|
||||
foreach $test (@ARGV)
|
||||
{
|
||||
# Strip out any whitespace in $test.
|
||||
$test =~ s/^\s*(.*)\s*$/$1/;
|
||||
|
||||
$okay = 1;
|
||||
|
||||
my (@TSTATS);
|
||||
my ($t_str);
|
||||
@TSTATS = ("--------------------------------------------------------------------------\n");
|
||||
|
||||
$t_str = sprintf ("%s%s", $test, ' ' x (40 - length($test)));
|
||||
@TSTATS = (@TSTATS, $t_str);
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
foreach $line (@TSTATS)
|
||||
{
|
||||
printf STDOUT "$line";
|
||||
}
|
||||
}
|
||||
|
||||
($okay, $num_failed_subtests, $num_subtests, $utime, $stime) = wait_for_test($test);
|
||||
# num_failed_subtests and num_subtests will be zero in the diff mode.
|
||||
($hutime, $hstime) = &print_stats($test, $okay, $num_failed_subtests, $num_subtests, $utime, $stime);
|
||||
{
|
||||
lock(%tests_okay);
|
||||
print $tests_extra_dbg{$test};
|
||||
}
|
||||
|
||||
# Print coverage on success, or run reduce on failure.
|
||||
|
||||
$total_hutime += $hutime;
|
||||
$total_hstime += $hstime;
|
||||
|
||||
if ($okay)
|
||||
{
|
||||
$total_utime += $utime;
|
||||
$total_stime += $stime;
|
||||
}
|
||||
else
|
||||
{
|
||||
@FAILED_TESTS = (@FAILED_TESTS, $test);
|
||||
}
|
||||
|
||||
# If there were historical data, add the run time to the total time to
|
||||
# compare against the historical run time.
|
||||
if (0 < ($hutime + $hstime))
|
||||
{
|
||||
$total_ntime += $utime + $stime;
|
||||
}
|
||||
}
|
||||
|
||||
# Print summary stats.
|
||||
$tt_str = sprintf ("%d / %d passed (%5.2f%%%%)",
|
||||
($#ARGV + 1) - ($#FAILED_TESTS + 1),
|
||||
$#ARGV + 1,
|
||||
(($#ARGV + 1) - ($#FAILED_TESTS + 1))
|
||||
/ ($#ARGV + 1) * 100);
|
||||
|
||||
$t_str = sprintf ("Totals %7.2f %7.2f %7.2f"
|
||||
. " %7.2f\n"
|
||||
. " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
|
||||
$total_utime, $total_stime, $total_utime + $total_stime,
|
||||
($total_ntime - ($total_hutime + $total_hstime)),
|
||||
$tt_str . ' ' x (40 - length($tt_str)),
|
||||
$total_hutime, $total_hstime, $total_hutime + $total_hstime,
|
||||
($total_hutime + $total_hstime == 0.0) ? 0.0 :
|
||||
(($total_ntime
|
||||
- ($total_hutime + $total_hstime))
|
||||
/ ($total_hutime + $total_hstime) * 100));
|
||||
|
||||
$thread_warning = "";
|
||||
if ($opt_threads > 1) {
|
||||
$thread_warning = "WARNING: Multiple threads were used. CPU times "
|
||||
. "are wildly inaccurate\n"
|
||||
}
|
||||
|
||||
@TSTATS = ("--------------------------------------------------------------------------\n",
|
||||
$thread_warning,
|
||||
$t_str,
|
||||
"--------------------------------------------------------------------------\n"
|
||||
);
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
foreach $line (@TSTATS)
|
||||
{
|
||||
printf STDOUT "$line";
|
||||
}
|
||||
if (@FAILED_TESTS) {
|
||||
printf STDOUT "failed tests:\n";
|
||||
}
|
||||
foreach $fail (@FAILED_TESTS)
|
||||
{
|
||||
printf STDOUT "%50s\n", $fail;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if ($opt_server)
|
||||
{
|
||||
kill 9, $server_pid;
|
||||
`ps aux | grep hphpi | grep -v perl | cut -c10-15 | sudo xargs kill -9 \\
|
||||
2>&1 >/dev/null`
|
||||
}
|
||||
|
||||
if ($#FAILED_TESTS >= 0)
|
||||
{
|
||||
# One or more tests failed, so return an error.
|
||||
exit 1;
|
||||
}
|
||||
# End of main execution.
|
||||
|
||||
sub run_tests
|
||||
{
|
||||
foreach my $test (@{$_[0]})
|
||||
{
|
||||
my ($okay, $num_failed_subtests, $num_subtests, $utime, $stime, $extra_dbg) = run_test($test);
|
||||
{
|
||||
lock(%tests_okay);
|
||||
$tests_okay{$test} = $okay;
|
||||
$tests_num_failed_subtests{$test} = $num_failed_subtests;
|
||||
$tests_num_subtests{$test} = $num_subtests;
|
||||
$tests_utime{$test} = $utime;
|
||||
$tests_stime{$test} = $stime;
|
||||
$tests_extra_dbg{$test} = $extra_dbg;
|
||||
cond_signal(%tests_okay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub wait_for_test
|
||||
{
|
||||
my ($test) = @_;
|
||||
while (1)
|
||||
{
|
||||
lock(%tests_okay);
|
||||
if (exists $tests_okay{$test}) {
|
||||
return ($tests_okay{$test},
|
||||
$tests_num_failed_subtests{$test},
|
||||
$tests_num_subtests{$test},
|
||||
$tests_utime{$test},
|
||||
$tests_stime{$test},
|
||||
$tests_extra_dbg{$test});
|
||||
}
|
||||
cond_wait(%tests_okay);
|
||||
}
|
||||
}
|
||||
|
||||
sub run_test
|
||||
{
|
||||
my ($test) = @_;
|
||||
my ($okay) = 1;
|
||||
my ($num_failed_subtests) = 0;
|
||||
my ($num_subtests) = 0;
|
||||
my ($tutime, $tstime);
|
||||
my ($extra_dbg) = "";
|
||||
my ($utime, $stime, $cutime, $cstime);
|
||||
|
||||
if ($opt_server)
|
||||
{
|
||||
$command = "GET http://localhost:" . $opt_port . "/" . $test;
|
||||
}
|
||||
else
|
||||
{
|
||||
$command = sprintf ($opt_command, $opt_srcdir, $opt_objdir, $test);
|
||||
if ($opt_cpu_time_limit)
|
||||
{
|
||||
$command = "(ulimit -t " . $opt_cpu_time_limit . "; "
|
||||
. "timeout " . $opt_real_time_limit . " "
|
||||
. $command . ")";
|
||||
}
|
||||
}
|
||||
($utime, $stime, $cutime, $cstime) = times;
|
||||
my $default_filter = "$opt_srcdir/test/vm/default.filter";
|
||||
if (-e "$opt_srcdir/$test.filter") {
|
||||
`$command 2>&1 | $default_filter | $opt_srcdir/$test.filter > $opt_objdir/$test.out`;
|
||||
} else {
|
||||
`$command 2>&1 | $default_filter > $opt_objdir/$test.out`;
|
||||
}
|
||||
($utime, $stime, $tutime, $tstime) = times;
|
||||
|
||||
# Subtract the before time from the after time.
|
||||
$tutime -= $cutime;
|
||||
$tstime -= $cstime;
|
||||
|
||||
if ($opt_zero)
|
||||
{
|
||||
if ($?)
|
||||
{
|
||||
$okay = 0;
|
||||
if ($opt_verbose)
|
||||
{
|
||||
print STDERR
|
||||
"\"$opt_objdir/$test > $opt_objdir/$test.out 2>&1\" returned $?\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Compare the results with the expected, and update "okay".
|
||||
if (! -e "$opt_objdir/$test.out") {
|
||||
$okay = 0;
|
||||
if ($opt_verbose)
|
||||
{
|
||||
print STDERR
|
||||
"Nonexistent output file \"$opt_objdir/$test.out\"\n";
|
||||
}
|
||||
}
|
||||
elsif (!$opt_no_exp && -e "$opt_srcdir/$test.exp")
|
||||
{
|
||||
# Diff mode.
|
||||
$diff_args = "--text -u";
|
||||
if ($opt_server)
|
||||
{
|
||||
`diff $diff_args -I HipHop $opt_srcdir/$test.exp $opt_objdir/$test.out \\
|
||||
> $opt_objdir/$test.diff 2>&1`;
|
||||
}
|
||||
else
|
||||
{
|
||||
`diff $diff_args $opt_srcdir/$test.exp $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`;
|
||||
}
|
||||
if ($?)
|
||||
{
|
||||
# diff returns non-zero if there is a difference.
|
||||
$okay = 0;
|
||||
}
|
||||
}
|
||||
elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expect")
|
||||
{
|
||||
`diff $diff_args $opt_srcdir/$test.expect $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`;
|
||||
if ($?)
|
||||
{
|
||||
# diff returns non-zero if there is a difference.
|
||||
$okay = 0;
|
||||
}
|
||||
}
|
||||
elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expectf")
|
||||
{
|
||||
open (FILE, "<$opt_objdir/$test.expectf");
|
||||
@lines = <FILE>;
|
||||
close(FILE);
|
||||
$wanted_re = join('', @lines);
|
||||
|
||||
open (FILE, "<$opt_objdir/$test.out");
|
||||
@lines = <FILE>;
|
||||
close(FILE);
|
||||
$content = join('', @lines);
|
||||
|
||||
# from run-tests.php
|
||||
$temp = "";
|
||||
$r = "%r";
|
||||
$startOffset = 0;
|
||||
$length = length($wanted_re);
|
||||
while ($startOffset < $length)
|
||||
{
|
||||
$start = index($wanted_re, $r, $startOffset);
|
||||
if ($start != -1)
|
||||
{
|
||||
$end = index($wanted_re, $r, $start+2);
|
||||
if ($end == -1)
|
||||
{
|
||||
# unbalanced tag, ignore it.
|
||||
$end = $start = $length;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$start = $end = $length;
|
||||
}
|
||||
|
||||
$temp = $temp . quotemeta(substr($wanted_re, $startOffset, $start - $startOffset));
|
||||
if ($end > $start)
|
||||
{
|
||||
$temp = $temp . '(' . substr($wanted_re, $start.2, $end - $start-2) . ')';
|
||||
}
|
||||
|
||||
$startOffset = $end + 2;
|
||||
}
|
||||
$wanted_re = $temp;
|
||||
|
||||
## perl escapes % but we want to use %s as our replacement
|
||||
$wanted_re =~ s/\\%/%/g;
|
||||
|
||||
$wanted_re =~ s/%binary_string_optional%/string/g;
|
||||
$wanted_re =~ s/%unicode_string_optional%/string/g;
|
||||
$wanted_re =~ s/%unicode\\\|string%/string/g;
|
||||
$wanted_re =~ s/%string\\\|unicode%/string/g;
|
||||
$wanted_re =~ s/%u\\\|b%//g;
|
||||
$wanted_re =~ s/%b\\\|u%//g;
|
||||
|
||||
# Stick to basics;
|
||||
$wanted_re =~ s/%e/\\\//g;
|
||||
$wanted_re =~ s/%s/[^\r\n]+/g;
|
||||
$wanted_re =~ s/%S/[^\r\n]*/g;
|
||||
$wanted_re =~ s/%a/.+/g;
|
||||
$wanted_re =~ s/%A/.*/g;
|
||||
$wanted_re =~ s/%w/\\s*/g;
|
||||
$wanted_re =~ s/%i/[+-]?\\d+/g;
|
||||
$wanted_re =~ s/%d/\\d+/g;
|
||||
$wanted_re =~ s/%x/[0-9a-fA-F]+/g;
|
||||
$wanted_re =~ s/%f/[+-]?\\.?\\d+\\.?\\d*(?:[Ee][+-]?\\d+)?/g;
|
||||
$wanted_re =~ s/%c/./g;
|
||||
|
||||
unless ($content =~ /^$wanted_re$/)
|
||||
{
|
||||
$okay = 0;
|
||||
}
|
||||
}
|
||||
elsif (!$opt_no_exp && -e "$opt_srcdir/$test.expectregex")
|
||||
{
|
||||
open (FILE, "<$opt_objdir/$test.expectregex");
|
||||
@lines = <FILE>;
|
||||
close(FILE);
|
||||
$regex = join('', @lines);
|
||||
|
||||
open (FILE, "<$opt_objdir/$test.out");
|
||||
@lines = <FILE>;
|
||||
close(FILE);
|
||||
$content = join('', @lines);
|
||||
|
||||
unless ($content =~ /$regex/)
|
||||
{
|
||||
$okay = 0;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
# Sequence mode.
|
||||
if (open (STEST_OUT, "<$opt_objdir/$test.out"))
|
||||
{
|
||||
$num_subtests = 0;
|
||||
$num_failed_subtests = 0;
|
||||
|
||||
while (defined($line = <STEST_OUT>))
|
||||
{
|
||||
if ($line =~ /1\.\.(\d+)/)
|
||||
{
|
||||
$num_subtests = $1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
if ($num_subtests == 0)
|
||||
{
|
||||
$okay = 0;
|
||||
if ($opt_verbose)
|
||||
{
|
||||
print STDERR "Malformed or missing 1..n line\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for ($subtest = 1; $subtest <= $num_subtests; $subtest++)
|
||||
{
|
||||
while (defined($line = <STEST_OUT>))
|
||||
{
|
||||
if ($line =~ /^not\s+ok\s+(\d+)?/)
|
||||
{
|
||||
$not = 1;
|
||||
$test_num = $1;
|
||||
last;
|
||||
}
|
||||
elsif ($line =~ /^ok\s+(\d+)?/)
|
||||
{
|
||||
$not = 0;
|
||||
$test_num = $1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
if (defined($line))
|
||||
{
|
||||
if (defined($test_num) && ($test_num != $subtest))
|
||||
{
|
||||
# There was no output printed for one or more tests.
|
||||
for (; $subtest < $test_num; $subtest++)
|
||||
{
|
||||
$num_failed_subtests++;
|
||||
}
|
||||
}
|
||||
if ($not)
|
||||
{
|
||||
$num_failed_subtests++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (; $subtest <= $num_subtests; $subtest++)
|
||||
{
|
||||
$num_failed_subtests++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (0 < $num_failed_subtests)
|
||||
{
|
||||
$okay = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
print STDERR "Cannot open output file \"$opt_objdir/$test.out\"\n";
|
||||
}
|
||||
exit 1;
|
||||
}
|
||||
}
|
||||
|
||||
if ($okay == 0)
|
||||
{
|
||||
# Run reduce to pinpoint the tracelet that causes the error.
|
||||
# For now, reduce has to be run sequentially, because it copies the input
|
||||
# file to the same temporary place.
|
||||
if ($opt_reduce ne "")
|
||||
{
|
||||
$cmd = $opt_reduce . " '" . $opt_hhvm . "' " . $test;
|
||||
if ($opt_cpu_time_limit)
|
||||
{
|
||||
$cmd = join(" ",
|
||||
$cmd,
|
||||
$opt_cpu_time_limit,
|
||||
$opt_real_time_limit);
|
||||
}
|
||||
$extra_dbg = `$cmd 2> /dev/null`;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
# Print the coverage.
|
||||
# Need run the command again in the tracing mode.
|
||||
if ($opt_coverage ne "")
|
||||
{
|
||||
$command = sprintf($opt_command, $opt_srcdir, $opt_objdir, $test);
|
||||
# Need calculate # of spills, reloads, and pushes/pops
|
||||
# around natives which are logged in level 3.
|
||||
$command = join("", "TRACE=tx64:3 ",
|
||||
"HPHP_TRACE_FILE=", $test, ".log ",
|
||||
$command);
|
||||
`$command 2> /dev/null`;
|
||||
$extra_dbg = `$opt_coverage < $test.log 2> /dev/null`;
|
||||
}
|
||||
}
|
||||
|
||||
return ($okay, $num_failed_subtests, $num_subtests, $tutime,
|
||||
$tstime, $extra_dbg);
|
||||
}
|
||||
|
||||
sub print_stats
|
||||
{
|
||||
my ($test, $okay, $failed_subtests, $subtests, $utime, $stime) = @_;
|
||||
my ($hutime, $hstime);
|
||||
# my (TEST_PERF);
|
||||
my (@TSTATS);
|
||||
my ($t_str, $pass_str);
|
||||
|
||||
$pass_str = $okay ? "passed" : "*** FAILED ***";
|
||||
if ((0 != $subtests) && (!$okay))
|
||||
{
|
||||
$pass_str = $pass_str . " ($failed_subtests/$subtests failed)";
|
||||
}
|
||||
$pass_str = $pass_str . ' ' x (39 - length($pass_str));
|
||||
|
||||
if (-r "$test.perf")
|
||||
{
|
||||
if (!open (TEST_PERF, "<$opt_objdir/$test.perf"))
|
||||
{
|
||||
print STDERR "Unable to open \"$opt_objdir/$test.perf\"\n";
|
||||
exit 1;
|
||||
}
|
||||
$_ = <TEST_PERF>;
|
||||
|
||||
($hutime, $hstime) = split;
|
||||
close TEST_PERF;
|
||||
|
||||
$t_str = sprintf (" %7.2f %7.2f %7.2f %7.2f\n"
|
||||
. " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
|
||||
$utime, $stime, $utime + $stime,
|
||||
($utime + $stime) - ($hutime + $hstime),
|
||||
$pass_str,
|
||||
$hutime, $hstime, $hutime + $hstime,
|
||||
(($hutime + $hstime) == 0.0) ? 0.0 :
|
||||
((($utime + $stime) - ($hutime + $hstime))
|
||||
/ ($hutime + $hstime) * 100));
|
||||
}
|
||||
else
|
||||
{
|
||||
$hutime = 0.0;
|
||||
$hstime = 0.0;
|
||||
|
||||
$t_str = sprintf (" %7.2f %7.2f %7.2f \n"
|
||||
. " %s\n",
|
||||
$utime, $stime, $utime + $stime,
|
||||
$pass_str);
|
||||
}
|
||||
@TSTATS = ($t_str);
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
foreach $line (@TSTATS)
|
||||
{
|
||||
printf STDOUT "$line";
|
||||
}
|
||||
}
|
||||
|
||||
if ($okay && $opt_ustats)
|
||||
{
|
||||
if (!open (TEST_PERF, ">$opt_objdir/$test.perf"))
|
||||
{
|
||||
if (!$opt_quiet)
|
||||
{
|
||||
print STDERR "Unable to update \"$opt_objdir/$test.perf\"\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
print TEST_PERF "$utime $stime\n";
|
||||
close TEST_PERF;
|
||||
}
|
||||
}
|
||||
|
||||
return ($hutime, $hstime);
|
||||
}
|
||||
|
||||
sub usage
|
||||
{
|
||||
print <<EOF;
|
||||
$0 usage:
|
||||
$0 [<options>] <test>+
|
||||
|
||||
Option | Description
|
||||
--------------+-------------------------------------------------------------
|
||||
-h --help | Print usage and exit.
|
||||
-v --verbose | Verbose (incompatible with quiet).
|
||||
-q --quiet | Quiet (incompatible with verbose).
|
||||
-s --srcdir | Path to source tree (default is ".").
|
||||
-o --objdir | Path to object tree (default is ".").
|
||||
-c --command | Command template, where the following substitutions are
|
||||
| performed:
|
||||
| %1\$s : srcdir (see --srcdir)
|
||||
| %2\$s : objdir (see --objdir)
|
||||
| %3\$s : <test>
|
||||
| (default is '%1\$s/%3\$s').
|
||||
-r --reduce | The path to the reduce tool.
|
||||
| Do not run reduce if it is empty.
|
||||
| (default is empty)
|
||||
--coverage | The path to the coverage tool.
|
||||
| Do not run coverage if it is empty.
|
||||
| (default is empty)
|
||||
-t | Set the time limit for each test case
|
||||
| (default is unlimited).
|
||||
-u --ustats | Update historical statistics (stored in "<test>.perf".
|
||||
-z --zero | Consider non-zero exit code to be an error.
|
||||
--no-exp | Don't diff with .exp files
|
||||
--------------+-------------------------------------------------------------
|
||||
|
||||
If <test>.exp exists and --no-exp is not specified, <test>'s output is
|
||||
diff'ed with <test>.exp. Any difference is considered failure.
|
||||
|
||||
If <test>.exp does not exist or --no-exp is specified, output to stdout of
|
||||
the following form is expected:
|
||||
|
||||
1..<n>
|
||||
{not }ok[ 1]
|
||||
{not }ok[ 2]
|
||||
...
|
||||
{not }ok[ n]
|
||||
|
||||
1 <= <n> < 2^31
|
||||
|
||||
Lines which do not match the patterns shown above are ignored.
|
||||
EOF
|
||||
}
|
||||
# vim:filetype=perl:
|
||||
@@ -6,10 +6,12 @@ then copies the good ones to test/zend/good and the bad ones to test/zend/bad.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
bad_tests = (
|
||||
@@ -74,12 +76,6 @@ parser.add_argument(
|
||||
type=str,
|
||||
help="zend path to import tests from."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--dont_run",
|
||||
action='store_true',
|
||||
help="don't run run_verify.sh. Just parse the .diff files."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--only",
|
||||
@@ -128,11 +124,15 @@ def walk(filename, source):
|
||||
for i in sections.keys():
|
||||
sections[i] = '\n'.join(sections[i])
|
||||
|
||||
unsupported_sections = ('INI', 'POST_RAW')
|
||||
for name in unsupported_sections:
|
||||
if sections.has_key(name):
|
||||
print "Unsupported test with section --%s--: " % name, filename
|
||||
return
|
||||
|
||||
if not sections.has_key('FILE'):
|
||||
print "Malformed test, no --FILE--: ", filename
|
||||
return
|
||||
|
||||
test = sections['FILE']
|
||||
|
||||
dest_filename = os.path.basename(filename).replace('.phpt', '.php')
|
||||
source_dir = source.lower().replace('/tests', '').replace('/', '-')
|
||||
@@ -151,77 +151,40 @@ def walk(filename, source):
|
||||
exp = exp.replace('\n\nWarning:', '\nWarning:')
|
||||
exp = exp.replace('\n\nNotice:', '\nNotice:')
|
||||
|
||||
match_rest_of_line = '%a'
|
||||
if key == 'EXPECTREGEX':
|
||||
match_rest_of_line = '.+'
|
||||
|
||||
exp = re.sub(r'Fatal\\? error\\?:.*', 'HipHop Fatal error: '+match_rest_of_line, exp)
|
||||
exp = re.sub(r'Warning\\?:.*', 'HipHop Warning: '+match_rest_of_line, exp)
|
||||
exp = re.sub(r'Notice\\?:.*', 'HipHop Notice: '+match_rest_of_line, exp)
|
||||
|
||||
for error in errors:
|
||||
exp = re.sub(error[0], error[1], exp)
|
||||
|
||||
sections[key] = exp
|
||||
|
||||
def kill_error_messages(exp):
|
||||
exp = re.sub(r'Fatal\\? error\\?:.*', 'HipHop Fatal error: .+', exp)
|
||||
exp = re.sub(r'Warning\\?:.*', 'HipHop Warning: .+', exp)
|
||||
exp = re.sub(r'Notice\\?:.*', 'HipHop Notice: .+', exp)
|
||||
return exp
|
||||
cur_dir = os.path.dirname(__file__)
|
||||
dest_subdir = os.path.join(cur_dir, '../test/zend/all', source_dir)
|
||||
mkdir_p(dest_subdir)
|
||||
full_dest_filename = os.path.join(dest_subdir, dest_filename)
|
||||
|
||||
if sections.has_key('EXPECT'):
|
||||
exp = sections['EXPECT']
|
||||
exp = kill_error_messages(exp)
|
||||
# we use %a for error messages so always write expectf
|
||||
file(full_dest_filename+'.expectf', 'w').write(exp)
|
||||
elif sections.has_key('EXPECTREGEX'):
|
||||
exp = sections['EXPECTREGEX']
|
||||
exp = kill_error_messages(exp)
|
||||
file(full_dest_filename+'.expectregex', 'w').write(exp)
|
||||
elif sections.has_key('EXPECTF'):
|
||||
wanted_re = sections['EXPECTF']
|
||||
|
||||
# from run-tests.php
|
||||
temp = "";
|
||||
r = "%r";
|
||||
startOffset = 0;
|
||||
length = len(wanted_re);
|
||||
while startOffset < length:
|
||||
start = wanted_re.find(r, startOffset)
|
||||
if start != -1:
|
||||
end = wanted_re.find(r, start+2);
|
||||
if end == -1:
|
||||
# unbalanced tag, ignore it.
|
||||
end = start = length;
|
||||
else:
|
||||
start = end = length;
|
||||
|
||||
temp = temp + re.escape(wanted_re[startOffset:start - startOffset])
|
||||
if (end > start):
|
||||
temp = temp + '(' + wanted_re[start+2:end - start-2] + ')'
|
||||
|
||||
startOffset = end + 2
|
||||
|
||||
wanted_re = temp
|
||||
|
||||
## different from php, since python escapes %
|
||||
wanted_re = wanted_re.replace('\\%', '%')
|
||||
wanted_re = kill_error_messages(wanted_re)
|
||||
|
||||
wanted_re = wanted_re.replace('%binary_string_optional%', 'string')
|
||||
wanted_re = wanted_re.replace('%unicode_string_optional%', 'string')
|
||||
wanted_re = wanted_re.replace('%unicode\|string%', 'string')
|
||||
wanted_re = wanted_re.replace('%string\|unicode%', 'string')
|
||||
wanted_re = wanted_re.replace('%u\|b%', '')
|
||||
wanted_re = wanted_re.replace('%b\|u%', '')
|
||||
|
||||
# Stick to basics
|
||||
wanted_re = wanted_re.replace('%e', '\\/')
|
||||
wanted_re = wanted_re.replace('%s', '[^\r\n]+')
|
||||
wanted_re = wanted_re.replace('%S', '[^\r\n]*')
|
||||
wanted_re = wanted_re.replace('%a', '.+')
|
||||
wanted_re = wanted_re.replace('%A', '.*')
|
||||
wanted_re = wanted_re.replace('%w', '\s*')
|
||||
wanted_re = wanted_re.replace('%i', '[+-]?\d+')
|
||||
wanted_re = wanted_re.replace('%d', '\d+')
|
||||
wanted_re = wanted_re.replace('%x', '[0-9a-fA-F]+')
|
||||
wanted_re = wanted_re.replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?')
|
||||
wanted_re = wanted_re.replace('%c', '.')
|
||||
exp = wanted_re
|
||||
exp = sections['EXPECTF']
|
||||
file(full_dest_filename+'.expectf', 'w').write(exp)
|
||||
else:
|
||||
print "Malformed test, no --EXPECT-- or --EXPECTF-- or --EXPECTREGEX--: ", filename
|
||||
return
|
||||
|
||||
test = sections['FILE']
|
||||
|
||||
if sections.has_key('POST'):
|
||||
test = test.replace(
|
||||
'<?php',
|
||||
@@ -238,24 +201,12 @@ def walk(filename, source):
|
||||
'<?php\n$_COOKIE = http_parse_cookie("' + sections['COOKIE'] + '");\n'
|
||||
)
|
||||
|
||||
unsupported_sections = ('INI', 'POST_RAW')
|
||||
for name in unsupported_sections:
|
||||
if sections.has_key(name):
|
||||
print "Unsupported test with section --%s--: " % name, filename
|
||||
return
|
||||
|
||||
cur_dir = os.path.dirname(__file__)
|
||||
dest_subdir = os.path.join(cur_dir, '../test/zend/all', source_dir)
|
||||
mkdir_p(dest_subdir)
|
||||
full_dest_filename = os.path.join(dest_subdir, dest_filename)
|
||||
|
||||
if 'bug60771.php' in full_dest_filename:
|
||||
test = test.replace("?>", "unlink('test.php');\n?>")
|
||||
if 'bug44805.php' in full_dest_filename:
|
||||
test = test.replace("1)) {\n\tunlink($file2", "2)) {\n\tunlink($file2")
|
||||
|
||||
file(full_dest_filename, 'w').write(test)
|
||||
file(full_dest_filename+'.exp', 'w').write(exp)
|
||||
|
||||
if args.zend_path:
|
||||
test_dirs = (('Zend/tests'), ('tests'), ('sapi'), ('ext'))
|
||||
@@ -344,98 +295,55 @@ if not os.path.isdir('test/zend/all'):
|
||||
print "Running all tests from test/zend/bad"
|
||||
shutil.copytree('test/zend/bad', 'test/zend/all')
|
||||
|
||||
if not args.dont_run:
|
||||
env = os.environ
|
||||
env.update({'VQ':'interp', 'TEST_PATH':'zend/all'})
|
||||
stdout = open(os.devnull, 'wb')
|
||||
if args.verbose:
|
||||
stdout = None
|
||||
proc = subprocess.Popen(
|
||||
['tools/run_verify.sh'],
|
||||
env=env,
|
||||
stdout=stdout,
|
||||
stderr=subprocess.STDOUT
|
||||
print "Running all tests from zend/all"
|
||||
|
||||
stdout = subprocess.Popen(
|
||||
[
|
||||
'tools/verify_to_json.php',
|
||||
'run_verify.sh',
|
||||
'interp',
|
||||
'zend/all',
|
||||
'_bin',
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT
|
||||
).communicate()[0]
|
||||
|
||||
# fbmake, you are crazy
|
||||
results = json.loads('['+stdout.strip().replace("\n", ",\n")+']')[-1]['results']
|
||||
|
||||
if args.verbose:
|
||||
print results
|
||||
|
||||
for test in results:
|
||||
filename = test['name']
|
||||
good_file = filename.replace('all', 'good', 1)
|
||||
bad_file = filename.replace('all', 'bad', 1)
|
||||
mkdir_p(os.path.dirname(good_file))
|
||||
mkdir_p(os.path.dirname(bad_file))
|
||||
|
||||
if test['status'] == 'passed':
|
||||
dest_file = good_file
|
||||
delete_file = bad_file
|
||||
subpath = 'good'
|
||||
else:
|
||||
dest_file = bad_file
|
||||
delete_file = good_file
|
||||
subpath = 'bad'
|
||||
|
||||
exps = glob.glob(filename+'.expect*')
|
||||
if not exps:
|
||||
# this file is probably generated while running tests :(
|
||||
continue
|
||||
|
||||
source_file_exp = exps[0]
|
||||
_, dest_ext = os.path.splitext(source_file_exp)
|
||||
os.rename(filename, dest_file)
|
||||
file(dest_file+dest_ext, 'w').write(
|
||||
file(source_file_exp).read().replace('/all', '/' + subpath)
|
||||
)
|
||||
proc.wait()
|
||||
|
||||
for root, dirs, files in os.walk('test/zend/all'):
|
||||
for filename in files:
|
||||
if not filename.endswith('.php'):
|
||||
continue
|
||||
filename = os.path.join(root, filename)
|
||||
|
||||
def all_exist(filename):
|
||||
extensions = ('', '.out', '.exp')
|
||||
for ext in extensions:
|
||||
if not os.path.exists(filename+ext):
|
||||
# something crazy is going on
|
||||
return False
|
||||
return True
|
||||
if not all_exist(filename):
|
||||
continue
|
||||
|
||||
good_file = filename.replace('all', 'good', 1)
|
||||
bad_file = filename.replace('all', 'bad', 1)
|
||||
mkdir_p(os.path.dirname(good_file))
|
||||
mkdir_p(os.path.dirname(bad_file))
|
||||
|
||||
def isOkDiff(original_name):
|
||||
global args
|
||||
for test in bad_tests:
|
||||
if test in original_name:
|
||||
return False
|
||||
|
||||
# no diff file or is empty
|
||||
if (not os.path.exists(original_name + '.diff') or \
|
||||
os.stat(original_name + '.diff')[6] == 0):
|
||||
if args.verbose:
|
||||
print '\n', original_name, '\nNo .diff, passed'
|
||||
return True
|
||||
|
||||
# PHP is very inconsistent with whitespace in tests
|
||||
diff = file(original_name + '.diff').read()
|
||||
diff = re.sub(r'-(.*)\n\\ No newline at end of file\n\+\1', '', diff)
|
||||
if not re.search(r'\n-', diff):
|
||||
if args.verbose:
|
||||
print '\n', original_name, '\nOnly whitespace .diff, passed'
|
||||
return True
|
||||
|
||||
# I hack a bit and store the regex in the .exp file, use that
|
||||
wanted_re = file(original_name + '.exp').read().strip()
|
||||
output = file(original_name + '.out').read().strip()
|
||||
|
||||
import sre_constants
|
||||
try:
|
||||
match = re.match(wanted_re, output)
|
||||
except (OverflowError, AssertionError, sre_constants.error) as e:
|
||||
if args.verbose:
|
||||
print '\n', original_name, '\nException', '\n', e
|
||||
return False
|
||||
|
||||
if args.verbose:
|
||||
print '\n', original_name, '\n', repr(wanted_re), '\n', repr(output), '\n', match is not None
|
||||
return match and match.group() == output
|
||||
|
||||
|
||||
if isOkDiff(filename):
|
||||
dest_file = good_file
|
||||
source_file_exp = filename+'.out'
|
||||
delete_file = bad_file
|
||||
subpath = 'good'
|
||||
else:
|
||||
dest_file = bad_file
|
||||
source_file_exp = filename+'.exp'
|
||||
delete_file = good_file
|
||||
subpath = 'bad'
|
||||
|
||||
os.rename(filename, dest_file)
|
||||
file(dest_file+'.exp', 'w').write(
|
||||
file(source_file_exp).read().replace('/all', '/' + subpath)
|
||||
)
|
||||
if os.path.exists(delete_file):
|
||||
os.unlink(delete_file)
|
||||
if os.path.exists(delete_file+'.exp'):
|
||||
os.unlink(delete_file+'.exp')
|
||||
for f in glob.glob(delete_file+"*"):
|
||||
os.unlink(f)
|
||||
|
||||
if not args.dirty:
|
||||
shutil.rmtree('test/zend/all')
|
||||
|
||||
@@ -39,7 +39,7 @@ QTESTS_SKIP='autoload5.php condinfinite.php condinfinite2.php define_b.php
|
||||
include_backtrace2.php backup_cycle_collector.php
|
||||
redeclared_class1.php redeclared_class2.php'
|
||||
|
||||
VERIFY_SCRIPT=./test/verify.sh
|
||||
VERIFY_SCRIPT=./test/verify
|
||||
|
||||
######################################################################
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
#
|
||||
: ${FBMAKE_BIN_ROOT=$HPHP_HOME/_bin}
|
||||
|
||||
VERIFY_SCRIPT=./test/verify.sh
|
||||
VERIFY_SCRIPT=./test/verify
|
||||
PARSE_TEST=$FBMAKE_BIN_ROOT/hphp/util/parser/test/parse_tester
|
||||
|
||||
# some tests are expected not to parse
|
||||
|
||||
Arquivo executável
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/env php
|
||||
<?php
|
||||
|
||||
$HPHP_HOME = $_ENV['HPHP_HOME'];
|
||||
include_once $HPHP_HOME.'/hphp/test/fbmake_test_lib.php';
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
if (count($argv) != 5) {
|
||||
echo "usage: $argv[0] test-script interp|jit|hhir TEST_PATH FBMAKE_BIN_ROOT\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$cmd = "VQ=$argv[2] TEST_PATH=$argv[3] FBMAKE_BIN_ROOT=$HPHP_HOME/$argv[4] " .
|
||||
"$HPHP_HOME/hphp/tools/$argv[1]";
|
||||
|
||||
loop_tests($cmd, function ($line) {
|
||||
if (preg_match('/^(test[^\s]*).*/', $line, &$m)) {
|
||||
start($m[1]);
|
||||
return;
|
||||
}
|
||||
if (!test_is_running()) return;
|
||||
if (preg_match('/^\s*passed.*/', $line)) {
|
||||
finish('passed');
|
||||
} else if (preg_match('/^[\s\*]*FAILED.*/', $line)) {
|
||||
finish('failed');
|
||||
}
|
||||
});
|
||||
Referência em uma Nova Issue
Bloquear um usuário