blob: 5c52caf3cf7a6967453097a691201d9d8a86255e [file] [log] [blame]
#!/usr/bin/env vpython
import json
import optparse
import os
import sys
from webkitpy.common.host import Host
from webkitpy.layout_tests.port import platform_options, configuration_options
def main(argv):
parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
parser.add_option('--failures', action='store_true',
help='show failing tests')
parser.add_option('--flakes', action='store_true',
help='show flaky tests')
parser.add_option('--expected', action='store_true',
help='include expected results along with unexpected')
parser.add_option('--passes', action='store_true',
help='show passing tests')
parser.add_option('--ignored-failures-path', action='store',
help='ignore failures seen in a previous run')
parser.add_options(platform_options())
parser.add_options(configuration_options())
options, args = parser.parse_args(argv)
host = Host()
if args:
if args[0] == '-':
txt = sys.stdin.read()
elif os.path.exists(args[0]):
with open(args[0], 'r') as fp:
txt = fp.read()
else:
print >> sys.stderr, "file not found: %s" % args[0]
sys.exit(1)
else:
txt = host.filesystem.read_text_file(
host.filesystem.join(
host.port_factory.get(
options=options).results_directory(),
'full_results.json'))
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
passes, failures, flakes = decode_results(results, options.expected)
tests_to_print = []
if options.passes:
tests_to_print += passes.keys()
if options.failures:
tests_to_print += failures.keys()
if options.flakes:
tests_to_print += flakes.keys()
print "\n".join(sorted(tests_to_print))
if options.ignored_failures_path:
with open(options.ignored_failures_path, 'r') as fp:
txt = fp.read()
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
_, ignored_failures, _ = decode_results(results, options.expected)
new_failures = set(failures.keys()) - set(ignored_failures.keys())
if new_failures:
print "New failures:"
print "\n".join(sorted(new_failures))
print
if ignored_failures:
print "Ignored failures:"
print "\n".join(sorted(ignored_failures.keys()))
if new_failures:
return 1
return 0
def decode_results(results, include_expected=False):
tests = convert_trie_to_flat_paths(results['tests'])
failures = {}
flakes = {}
passes = {}
for (test, result) in tests.iteritems():
if include_expected or result.get('is_unexpected'):
actual_results = result['actual'].split()
expected_results = result['expected'].split()
if len(actual_results) > 1:
if actual_results[1] in expected_results:
flakes[test] = actual_results[0]
else:
# We report the first failure type back, even if the second
# was more severe.
failures[test] = actual_results[0]
elif actual_results[0] == 'PASS':
passes[test] = result
else:
failures[test] = actual_results[0]
return (passes, failures, flakes)
def convert_trie_to_flat_paths(trie, prefix=None):
# Cloned from webkitpy.layout_tests.layout_package.json_results_generator
# so that this code can stand alone.
result = {}
for name, data in trie.iteritems():
if prefix:
name = prefix + "/" + name
if len(data) and not "actual" in data and not "expected" in data:
result.update(convert_trie_to_flat_paths(data, name))
else:
result[name] = data
return result
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))