| #!/usr/bin/env vpython |
| import json |
| import optparse |
| import os |
| import sys |
| |
| from webkitpy.common.host import Host |
| from webkitpy.layout_tests.port import platform_options, configuration_options |
| |
| |
| def main(argv): |
| parser = optparse.OptionParser(usage='%prog [path-to-results.json]') |
| parser.add_option('--failures', action='store_true', |
| help='show failing tests') |
| parser.add_option('--flakes', action='store_true', |
| help='show flaky tests') |
| parser.add_option('--expected', action='store_true', |
| help='include expected results along with unexpected') |
| parser.add_option('--passes', action='store_true', |
| help='show passing tests') |
| parser.add_option('--ignored-failures-path', action='store', |
| help='ignore failures seen in a previous run') |
| parser.add_options(platform_options()) |
| parser.add_options(configuration_options()) |
| options, args = parser.parse_args(argv) |
| |
| host = Host() |
| if args: |
| if args[0] == '-': |
| txt = sys.stdin.read() |
| elif os.path.exists(args[0]): |
| with open(args[0], 'r') as fp: |
| txt = fp.read() |
| else: |
| print >> sys.stderr, "file not found: %s" % args[0] |
| sys.exit(1) |
| else: |
| txt = host.filesystem.read_text_file( |
| host.filesystem.join( |
| host.port_factory.get( |
| options=options).results_directory(), |
| 'full_results.json')) |
| |
| if txt.startswith('ADD_RESULTS(') and txt.endswith(');'): |
| txt = txt[12:-2] # ignore optional JSONP wrapper |
| results = json.loads(txt) |
| |
| passes, failures, flakes = decode_results(results, options.expected) |
| |
| tests_to_print = [] |
| if options.passes: |
| tests_to_print += passes.keys() |
| if options.failures: |
| tests_to_print += failures.keys() |
| if options.flakes: |
| tests_to_print += flakes.keys() |
| print "\n".join(sorted(tests_to_print)) |
| |
| if options.ignored_failures_path: |
| with open(options.ignored_failures_path, 'r') as fp: |
| txt = fp.read() |
| if txt.startswith('ADD_RESULTS(') and txt.endswith(');'): |
| txt = txt[12:-2] # ignore optional JSONP wrapper |
| results = json.loads(txt) |
| _, ignored_failures, _ = decode_results(results, options.expected) |
| new_failures = set(failures.keys()) - set(ignored_failures.keys()) |
| if new_failures: |
| print "New failures:" |
| print "\n".join(sorted(new_failures)) |
| print |
| if ignored_failures: |
| print "Ignored failures:" |
| print "\n".join(sorted(ignored_failures.keys())) |
| if new_failures: |
| return 1 |
| return 0 |
| |
| |
| def decode_results(results, include_expected=False): |
| tests = convert_trie_to_flat_paths(results['tests']) |
| failures = {} |
| flakes = {} |
| passes = {} |
| for (test, result) in tests.iteritems(): |
| if include_expected or result.get('is_unexpected'): |
| actual_results = result['actual'].split() |
| expected_results = result['expected'].split() |
| if len(actual_results) > 1: |
| if actual_results[1] in expected_results: |
| flakes[test] = actual_results[0] |
| else: |
| # We report the first failure type back, even if the second |
| # was more severe. |
| failures[test] = actual_results[0] |
| elif actual_results[0] == 'PASS': |
| passes[test] = result |
| else: |
| failures[test] = actual_results[0] |
| |
| return (passes, failures, flakes) |
| |
| |
| def convert_trie_to_flat_paths(trie, prefix=None): |
| # Cloned from webkitpy.layout_tests.layout_package.json_results_generator |
| # so that this code can stand alone. |
| result = {} |
| for name, data in trie.iteritems(): |
| if prefix: |
| name = prefix + "/" + name |
| |
| if len(data) and not "actual" in data and not "expected" in data: |
| result.update(convert_trie_to_flat_paths(data, name)) |
| else: |
| result[name] = data |
| |
| return result |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
