blob: 678ba71e3b207871273143cb4a7a9ba7d4bc0a32 [file] [log] [blame]
import json
import optparse
import os
import sys
from import Host
from webkitpy.layout_tests.port import platform_options, configuration_options
def main(argv):
parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
parser.add_option('--failures', action='store_true',
help='show failing tests')
parser.add_option('--flakes', action='store_true',
help='show flaky tests')
parser.add_option('--expected', action='store_true',
help='include expected results along with unexpected')
parser.add_option('--passes', action='store_true',
help='show passing tests')
parser.add_option('--ignored-failures-path', action='store',
help='ignore failures seen in a previous run')
options, args = parser.parse_args(argv)
host = Host()
if args:
if args[0] == '-':
txt =
elif os.path.exists(args[0]):
with open(args[0], 'r') as fp:
txt =
print >> sys.stderr, "file not found: %s" % args[0]
txt = host.filesystem.read_text_file(host.filesystem.join(host.port_factory.get(options=options).results_directory(), 'full_results.json'))
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
passes, failures, flakes = decode_results(results, options.expected)
tests_to_print = []
if options.passes:
tests_to_print += passes.keys()
if options.failures:
tests_to_print += failures.keys()
if options.flakes:
tests_to_print += flakes.keys()
print "\n".join(sorted(tests_to_print))
if options.ignored_failures_path:
with open(options.ignored_failures_path, 'r') as fp:
txt =
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
txt = txt[12:-2] # ignore optional JSONP wrapper
results = json.loads(txt)
_, ignored_failures, _ = decode_results(results, options.expected)
new_failures = set(failures.keys()) - set(ignored_failures.keys())
if new_failures:
print "New failures:"
print "\n".join(sorted(new_failures))
if ignored_failures:
print "Ignored failures:"
print "\n".join(sorted(ignored_failures.keys()))
if new_failures:
return 1
return 0
def decode_results(results, include_expected=False):
tests = convert_trie_to_flat_paths(results['tests'])
failures = {}
flakes = {}
passes = {}
for (test, result) in tests.iteritems():
if include_expected or result.get('is_unexpected'):
actual_result = result['actual']
if ' PASS' in actual_result:
flakes[test] = actual_result
elif actual_result == 'PASS':
passes[test] = result
failures[test] = actual_result
return (passes, failures, flakes)
def convert_trie_to_flat_paths(trie, prefix=None):
# Cloned from webkitpy.layout_tests.layout_package.json_results_generator
# so that this code can stand alone.
result = {}
for name, data in trie.iteritems():
if prefix:
name = prefix + "/" + name
if len(data) and not "actual" in data and not "expected" in data:
result.update(convert_trie_to_flat_paths(data, name))
result[name] = data
return result
if __name__ == '__main__':