| #!/usr/bin/perl -w |
| |
| # Copyright (C) 2007 Apple Inc. All rights reserved. |
| # Copyright (C) 2007 Eric Seidel <eric@webkit.org> |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR |
| # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
| # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| use strict; |
| use Getopt::Long; |
| use File::Basename; |
| use File::Spec; |
| use Cwd; |
| use POSIX qw(strftime); |
| use Time::HiRes qw(gettimeofday tv_interval); |
| |
| my $showHelp = 0; |
| my $runInstruments = 0; |
| my $ubench = 0; |
| my $v8suite = 0; |
| my $suite = ""; |
| my $parseOnly = 0; |
| my $jsShellPath; |
| my $jsShellArgs = ""; |
| my $setBaseline = 0; |
| my $testsPattern; |
| my $testRuns = 10; |
| |
| my $programName = basename($0); |
| my $usage = <<EOF; |
| Usage: $programName --shell=[path] [options] |
| --help Show this help message |
| --set-baseline Set baseline for future comparisons |
| --shell Path to JavaScript shell |
| --args Arguments to pass to JavaScript shell |
| --runs Number of times to run tests (default: $testRuns) |
| --tests Only run tests matching provided pattern |
| --instruments Sample execution time with the Mac OS X "Instruments" tool (Time Profile) (implies --runs=1) |
| --suite Select a specific benchmark suite. The default is sunspider-1.0 |
| --ubench Use microbenchmark suite instead of regular tests. Same as --suite=ubench |
| --v8-suite Use the V8 benchmark suite. Same as --suite=v8-v4 |
| --parse-only Use the parse-only benchmark suite. Same as --suite=parse-only |
| EOF |
| |
| GetOptions('runs=i' => \$testRuns, |
| 'set-baseline' => \$setBaseline, |
| 'shell=s' => \$jsShellPath, |
| 'args=s' => \$jsShellArgs, |
| 'instruments' => \$runInstruments, |
| 'suite=s' => \$suite, |
| 'ubench' => \$ubench, |
| 'v8-suite' => \$v8suite, |
| 'parse-only' => \$parseOnly, |
| 'tests=s' => \$testsPattern, |
| 'help' => \$showHelp); |
| |
| |
| $suite = "ubench" if ($ubench); |
| $suite = "v8-v4" if ($v8suite); |
| $suite = "parse-only" if ($parseOnly); |
| $suite = "sunspider-1.0" if (!$suite); |
| |
| my $resultDirectory = "${suite}-results"; |
| |
| my $suitePath = $suite; |
| $suitePath = "tests/" . $suitePath unless ($suite =~ /\//); |
| |
| $testRuns = 1 if $runInstruments; |
| |
| if (!$jsShellPath || $showHelp) { |
| print STDERR $usage; |
| exit 1; |
| } |
| |
| sub dumpToFile($$) |
| { |
| my ($contents, $path) = @_; |
| open FILE, ">", $path or die "Failed to open $path"; |
| print FILE $contents; |
| close FILE; |
| } |
| |
| my @tests = (); |
| my @categories = (); |
| my %uniqueCategories = (); |
| |
| sub loadTestsList() |
| { |
| open TESTLIST, "<", "${suitePath}/LIST" or die "Can't find ${suitePath}/LIST"; |
| while (<TESTLIST>) { |
| chomp; |
| next unless !$testsPattern || /$testsPattern/; |
| |
| push @tests, $_; |
| my $category = $_; |
| $category =~ s/-.*//; |
| if (!$uniqueCategories{$category}) { |
| push @categories, $category; |
| $uniqueCategories{$category} = $category; |
| } |
| } |
| close TESTLIST; |
| } |
| |
| my $timeString = strftime "%Y-%m-%d-%H.%M.%S", localtime $^T; |
| my $prefixFile = "$resultDirectory/sunspider-test-prefix.js"; |
| my $resultsFile = "$resultDirectory/sunspider-results-$timeString.js"; |
| |
| sub writePrefixFile() |
| { |
| my $prefix = "var suitePath = " . '"' . $suitePath . '"' . ";\n"; |
| $prefix .= "var tests = [ " . join(", ", map { '"' . $_ . '"' } @tests) . " ];\n"; |
| $prefix .= "var categories = [ " . join(", ", map { '"' . $_ . '"' } @categories) . " ];\n"; |
| |
| mkdir "$resultDirectory"; |
| dumpToFile($prefix, $prefixFile); |
| } |
| |
| sub runTestsOnce($) |
| { |
| my ($useInstruments) = @_; |
| my $shellArgs = $jsShellArgs . " -f $prefixFile -f resources/sunspider-standalone-driver.js 2> " . File::Spec->devnull(); |
| my $output; |
| if ($useInstruments) { |
| $output = `instruments -t "resources/TimeProfile20us.tracetemplate" "$jsShellPath" $shellArgs`; |
| } else { |
| $output = `"$jsShellPath" $shellArgs | grep -v break`; |
| } |
| return $output; |
| } |
| |
| sub newestFile($$) |
| { |
| my ($dir, $pattern) = @_; |
| |
| my $newestAge; |
| my $newestFile = ""; |
| opendir DIR, $dir or die; |
| for my $file (readdir DIR) { |
| if ($file =~ $pattern) { |
| my $age = -M "$dir/$file"; |
| if (!defined $newestAge || $age < $newestAge) { |
| $newestFile = $file; |
| $newestAge = $age; |
| } |
| } |
| } |
| closedir DIR; |
| |
| return "$dir/$newestFile"; |
| } |
| |
| loadTestsList(); |
| if ($testsPattern) { |
| print STDERR "Found " . scalar(@tests) . " tests matching '" . $testsPattern . "'\n"; |
| } else { |
| print STDERR "Found " . scalar(@tests) . " tests\n"; |
| } |
| die "No tests to run" unless scalar(@tests); |
| print STDERR "Running SunSpider once for warmup, then " . ($runInstruments ? "under Instruments" : "$testRuns time" . ($testRuns == 1 ? "" : "s")) . "\n"; |
| writePrefixFile(); |
| |
| runTestsOnce(0); |
| print "Discarded first run.\n"; |
| |
| my $result; |
| my $count = 0; |
| my @results = (); |
| my $total = 0; |
| print "["; |
| while ($count++ < $testRuns) { |
| $result = runTestsOnce($runInstruments); |
| $result =~ s/\r\n/\n/g; |
| chomp $result; |
| push @results, $result; |
| print $result; |
| print ",\n" unless ($count == $testRuns); |
| } |
| print "]\n"; |
| |
| my $output = "var output = [\n" . join(",\n", @results) . "\n];\n"; |
| dumpToFile($output, $resultsFile); |
| dumpToFile(File::Spec->rel2abs($resultsFile), "$resultDirectory/baseline-filename.txt") if $setBaseline; |
| |
| system("$jsShellPath", "-f", $prefixFile, "-f", $resultsFile, "-f", "resources/sunspider-analyze-results.js"); |
| |
| print("\nResults are located at $resultsFile\n"); |
| |
| if ($runInstruments) { |
| my $newestTrace = newestFile(".", qr/\.trace$/); |
| if ($newestTrace) { |
| my $profileFile = "$resultDirectory/sunspider-profile-$timeString.trace"; |
| rename $newestTrace, $profileFile or die; |
| exec "/usr/bin/open", $profileFile; |
| } |
| } |