| # Copyright 2021 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| |
| |
| def ParseArgs(): |
| parser = argparse.ArgumentParser( |
| description=('Script for automatically suppressing flaky/failing ' |
| 'tests.')) |
| parser.add_argument('--project', |
| required=True, |
| help=('The billing project to use for BigQuery queries. ' |
| 'Must have access to the ResultDB BQ tables, e.g. ' |
| '"chrome-luci-data.chromium.gpu_ci_test_results".')) |
| parser.add_argument('--sample-period', |
| type=int, |
| default=7, |
| choices=range(1, 30), |
| help=('The number of days to sample data from.')) |
| parser.add_argument('--no-group-by-tags', |
| action='store_false', |
| default=True, |
| dest='group_by_tags', |
| help=('Append added expectations to the end of the file ' |
| 'instead of attempting to automatically group with ' |
| 'similar expectations.')) |
| parser.add_argument('--no-prompt-for-user-input', |
| action='store_false', |
| default=True, |
| dest='prompt_for_user_input', |
| help=('Generate expectations automatically based on ' |
| 'thresholds instead of prompting the user each ' |
| 'time. The user will still need to add associated ' |
| 'bugs to generated expectations afterwards.')) |
| parser.add_argument('--ignore-threshold', |
| type=float, |
| default=0.01, |
| help=('The fraction of failed tests under which flakes ' |
| 'will be ignored instead of having an expectation ' |
| 'added when --no-prompt-for-user-input is used.')) |
| parser.add_argument('--flaky-threshold', |
| type=float, |
| default=0.5, |
| help=('The fraction of failed tests under which flakes ' |
| 'will be marked as RetryOnFailure when ' |
| '--no-prompt-for-user-input is used. Above this, ' |
| 'failures will be marked as Failure.')) |
| parser.add_argument('--include-all-tags', |
| action='store_true', |
| default=False, |
| help=('Use all tags generated by a configuration when ' |
| 'creating an expectation rather than attempting ' |
| 'to only use the most specific one. This should ' |
| 'only need to be passed if the tags in the ' |
| 'expectation files are not ordered from least ' |
| 'specific to most specific.')) |
| parser.add_argument('--result-output-file', |
| help=('Output file to store the generated results. If ' |
| 'not specified, will use a temporary file.')) |
| parser.add_argument('--bypass-up-to-date-check', |
| action='store_true', |
| default=False, |
| help=('Bypasses the check that the local expectation ' |
| 'files are up to date. Only intended for use on ' |
| 'bots to avoid failures due to potential race ' |
| 'conditions between updating the checkout and ' |
| 'running the script.')) |
| parser.add_argument( |
| '--non-hidden-failures-only', |
| action='store_true', |
| default=False, |
| help= |
| ('Enable this option to only targeting visible failures on CI builders. ' |
| 'The test results will fail the builder runs, flaky results will ' |
| 'consider as pass in this option.')) |
| parser.add_argument( |
| '--build-fail-total-number-threshold', |
| type=int, |
| default=10, |
| help=('Threshold based on failed build number when ' |
| '--non-hidden-failures-only is used. A test will be ' |
| 'suppressed if its failed build number is equal to or more than ' |
| 'this threshold. All --build-fail*-thresholds must be hit in ' |
| 'order for a test to actually be suppressed.')) |
| parser.add_argument( |
| '--build-fail-consecutive-days-threshold', |
| type=int, |
| default=2, |
| choices=range(1, 30), |
| help=('Threshold based on number of consecutive days that non-hidden' |
| 'failures occur. A test will be suppressed if the number of' |
| 'consecutive days that it has non-hidden failures is equal' |
| 'to or more than this threshold. All --build-fail*-thresholds ' |
| 'must be hit in order for a test to actually be suppressed.')) |
| parser.add_argument('--build-fail-recent-days-threshold', |
| type=int, |
| default=2, |
| choices=range(1, 30), |
| help=('Suppress tests with non-hidden build failures' |
| ' within |build_fail_recent_day_threshold| days ' |
| 'when all other build-fail* thresholds meet.')) |
| parser.add_argument('--builder-name', |
| default=[], |
| action='append', |
| dest='builder_names', |
| help='CI builder list to suppress tests.') |
| args = parser.parse_args() |
| |
| if not args.prompt_for_user_input: |
| if args.ignore_threshold < 0: |
| raise ValueError('--ignore-threshold must be positive') |
| if args.flaky_threshold < 0: |
| raise ValueError('--flaky-threshold must be positive') |
| if args.flaky_threshold <= args.ignore_threshold: |
| raise ValueError( |
| '--flaky-threshold must be greater than --ignore-threshold') |
| if args.build_fail_total_number_threshold < 0: |
| raise ValueError('--build-fail-total-number-threshold must be positive') |
| |
| return args |