| #!/usr/bin/env vpython3 |
| # Copyright 2020 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Script for determining which GPU tests are unexpectedly passing. |
| |
| This script depends on the `bb` tool, which is available as part of depot tools, |
| and the `bq` tool, which is available as part of the Google Cloud SDK |
| https://cloud.google.com/sdk/docs/quickstarts. |
| |
| Example usage: |
| |
| unexpected_pass_finder.py \ |
| --project <BigQuery billing project> \ |
| --suite <test suite to check> \ |
| |
| Concrete example: |
| |
| unexpected_pass_finder.py \ |
| --project luci-resultdb-dev \ |
| --suite pixel |
| |
| You would typically want to pass in --remove-stale-expectations as well in order |
| to have the script automatically remove any expectations it determines are no |
| longer necessary. If a particular expectation proves to be erroneously flagged |
| and removed (e.g. due to a very low flake rate that doesn't get caught |
| consistently by the script), expectations can be omitted from automatic removal |
| using an inline `# finder:disable` comment for a single expectation or a pair of |
| `# finder:disable`/`# finder:enable` comments for a block of expectations. |
| General disables can be handled via `finder:disable-general` and |
| `finder:enable-general`. Disabling removal only if the expectation is found to |
| be unused can be handled via `finder:disable-unused` and `finder:enable-unused`. |
| Disabling removal only if the expectation is found to be stale can be handled |
| via `finder:disable-stale` and `finder:enable-stale`. |
| """ |
| |
| import argparse |
| import datetime |
| import os |
| |
| from gpu_path_util import setup_telemetry_paths # pylint: disable=unused-import |
| from gpu_path_util import setup_testing_paths # pylint: disable=unused-import |
| |
| # Must come after path setup. |
| # pylint: disable=wrong-import-order |
| from unexpected_passes_common import argument_parsing |
| from unexpected_passes_common import builders |
| from unexpected_passes_common import expectations |
| from unexpected_passes_common import result_output |
| # pylint: enable=wrong-import-order |
| |
| from gpu_tests import gpu_integration_test |
| |
| from unexpected_passes import gpu_builders |
| from unexpected_passes import gpu_expectations |
| from unexpected_passes import gpu_queries |
| |
| |
| def ParseArgs() -> argparse.Namespace: |
| name_mapping = gpu_integration_test.GenerateTestNameMapping() |
| test_suites = list(name_mapping.keys()) |
| test_suites.sort() |
| |
| parser = argparse.ArgumentParser( |
| description=('Script for finding cases of stale expectations that can ' |
| 'be removed/modified.')) |
| argument_parsing.AddCommonArguments(parser) |
| |
| input_group = parser.add_mutually_exclusive_group() |
| input_group.add_argument( |
| '--expectation-file', |
| help='A path to an expectation file to read from. If not specified and ' |
| '--test is not used, will automatically determine based off the ' |
| 'provided suite.') |
| input_group.add_argument( |
| '--test', |
| action='append', |
| dest='tests', |
| default=[], |
| help='The name of a test to check for unexpected passes. Can be passed ' |
| 'multiple times to specify multiple tests. Will be treated as if it was ' |
| 'expected to be flaky on all configurations.') |
| parser.add_argument('--suite', |
| required=True, |
| choices=test_suites, |
| help='The test suite being checked.') |
| |
| args = parser.parse_args() |
| argument_parsing.PerformCommonPostParseSetup(args) |
| suite_class = name_mapping[args.suite] |
| |
| if not (args.tests or args.expectation_file): |
| expectation_files = suite_class.ExpectationsFiles() |
| if not expectation_files: |
| raise RuntimeError( |
| f'Suite {args.suite} does not specify an expectation file and is ' |
| f'thus not compatible with this script.') |
| if len(expectation_files) > 1: |
| raise RuntimeError( |
| f'Suite {suite_class} specifies {len(expectation_files)} expectation ' |
| f'files when only 1 is supported.') |
| args.expectation_file = expectation_files[0] |
| |
| if args.remove_stale_expectations and not args.expectation_file: |
| parser.error( |
| '--remove-stale-expectations can only be used with expectation files') |
| |
| # Change to whatever repo the test suite claims the expectation file lives in. |
| # This allows the script to work for most suites if run from outside of |
| # chromium/src. Similarly, it allows suites such as WebGPU CTS that have |
| # expectation files in a different repo to be work when run from chromium/src. |
| os.chdir(suite_class.GetExpectationsFilesRepoPath()) |
| |
| return args |
| |
| |
| # pylint: disable=too-many-locals |
| def main() -> None: |
| args = ParseArgs() |
| |
| builders_instance = gpu_builders.GpuBuilders(args.suite, |
| args.include_internal_builders) |
| builders.RegisterInstance(builders_instance) |
| expectations_instance = gpu_expectations.GpuExpectations() |
| expectations.RegisterInstance(expectations_instance) |
| |
| test_expectation_map = expectations_instance.CreateTestExpectationMap( |
| args.expectation_file, args.tests, |
| datetime.timedelta(days=args.expectation_grace_period)) |
| ci_builders = builders_instance.GetCiBuilders() |
| |
| querier = gpu_queries.GpuBigQueryQuerier(args.suite, args.project, |
| args.num_samples, |
| args.keep_unmatched_results) |
| # Unmatched results are mainly useful for script maintainers, as they don't |
| # provide any additional information for the purposes of finding unexpectedly |
| # passing tests or unused expectations. |
| unmatched = querier.FillExpectationMapForBuilders(test_expectation_map, |
| ci_builders) |
| try_builders = builders_instance.GetTryBuilders(ci_builders) |
| unmatched.update( |
| querier.FillExpectationMapForBuilders(test_expectation_map, try_builders)) |
| unused_expectations = test_expectation_map.FilterOutUnusedExpectations() |
| stale, semi_stale, active = test_expectation_map.SplitByStaleness() |
| if args.result_output_file: |
| with open(args.result_output_file, 'w', encoding='utf-8') as outfile: |
| result_output.OutputResults(stale, semi_stale, active, unmatched, |
| unused_expectations, args.output_format, |
| outfile) |
| else: |
| result_output.OutputResults(stale, semi_stale, active, unmatched, |
| unused_expectations, args.output_format) |
| |
| affected_urls = set() |
| stale_message = '' |
| if args.remove_stale_expectations: |
| for expectation_file, expectation_map in stale.items(): |
| affected_urls |= expectations_instance.RemoveExpectationsFromFile( |
| expectation_map.keys(), expectation_file, |
| expectations.RemovalType.STALE) |
| stale_message += (f'Stale expectations removed from {expectation_file}. ' |
| f'Stale comments, etc. may still need to be removed.\n') |
| for expectation_file, unused_list in unused_expectations.items(): |
| affected_urls |= expectations_instance.RemoveExpectationsFromFile( |
| unused_list, expectation_file, expectations.RemovalType.UNUSED) |
| stale_message += (f'Unused expectations removed from {expectation_file}. ' |
| f'Stale comments, etc. may still need to be removed.\n') |
| |
| if args.narrow_semi_stale_expectation_scope: |
| affected_urls |= expectations_instance.NarrowSemiStaleExpectationScope( |
| semi_stale) |
| stale_message += (f'Semi-stale expectations narrowed in ' |
| f'{args.expectation_file}. Stale comments, etc. may ' |
| f'still need to be removed.\n') |
| |
| if stale_message: |
| print(stale_message) |
| if affected_urls: |
| orphaned_urls = expectations_instance.FindOrphanedBugs(affected_urls) |
| if args.bug_output_file: |
| with open(args.bug_output_file, 'w', encoding='utf-8') as bug_outfile: |
| result_output.OutputAffectedUrls(affected_urls, |
| orphaned_urls, |
| bug_outfile, |
| auto_close_bugs=args.auto_close_bugs) |
| else: |
| result_output.OutputAffectedUrls(affected_urls, |
| orphaned_urls, |
| auto_close_bugs=args.auto_close_bugs) |
| # pylint: enable=too-many-locals |
| |
| |
| if __name__ == '__main__': |
| main() |
