| # Copyright 2020 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| import glob |
| import logging |
| import os |
| import sys |
| |
| # Add src/testing/ into sys.path for importing common without pylint errors. |
| sys.path.append( |
| os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) |
| from scripts import common |
| |
| BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools') |
| CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult') |
| OUT_DIR = os.path.join(common.SRC_DIR, "out", "{}") |
| DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT = os.path.join(OUT_DIR, "results.json") |
| TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ') |
| WEB_TESTS_DIR = os.path.normpath( |
| os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests')) |
| |
| if BLINK_TOOLS_DIR not in sys.path: |
| sys.path.append(BLINK_TOOLS_DIR) |
| |
| if TYP_DIR not in sys.path: |
| sys.path.append(TYP_DIR) |
| |
| from blinkpy.common.host import Host |
| from blinkpy.common.path_finder import PathFinder |
| |
| logger = logging.getLogger(__name__) |
| |
| |
| # pylint: disable=super-with-arguments |
| class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter): |
| """The base class for script adapters that use wptrunner to execute web |
| platform tests. This contains any code shared between these scripts, such |
| as integrating output with the results viewer. Subclasses contain other |
| (usually platform-specific) logic.""" |
| |
| def __init__(self, host=None): |
| super(BaseWptScriptAdapter, self).__init__() |
| self._parser = self._override_options(self._parser) |
| if not host: |
| host = Host() |
| self.host = host |
| self.fs = host.filesystem |
| self.path_finder = PathFinder(self.fs) |
| self.port = host.port_factory.get() |
| # Path to the output of the test run. Comes from the args passed to the |
| # run, parsed after this constructor. Can be overwritten by tests. |
| self.wpt_output = None |
| self.wptreport = None |
| self._include_filename = None |
| self.layout_test_results_subdir = 'layout-test-results' |
| default_wpt_binary = os.path.join( |
| common.SRC_DIR, "third_party", "wpt_tools", "wpt", "wpt") |
| self.wpt_binary = os.environ.get("WPT_BINARY", default_wpt_binary) |
| |
| @property |
| def wpt_root_dir(self): |
| return self.path_finder.path_from_web_tests( |
| self.path_finder.wpt_prefix()) |
| |
| @property |
| def output_directory(self): |
| return self.path_finder.path_from_chromium_base('out', |
| self.options.target) |
| |
| @property |
| def mojo_js_directory(self): |
| return self.fs.join(self.output_directory, 'gen') |
| |
| def add_extra_arguments(self, parser): |
| parser.add_argument( |
| '-t', |
| '--target', |
| default='Release', |
| help='Target build subdirectory under //out') |
| parser.add_argument( |
| '--default-exclude', |
| action='store_true', |
| help=('Only run the tests explicitly given in arguments ' |
| '(can run no tests, which will exit with code 0)')) |
| self.add_mode_arguments(parser) |
| self.add_output_arguments(parser) |
| |
| def add_mode_arguments(self, parser): |
| group = parser.add_argument_group( |
| 'Mode', |
| 'Options for wptrunner modes other than running tests.') |
| # We provide an option to show wptrunner's help here because the 'wpt' |
| # executable may be inaccessible from the user's PATH. The top-level |
| # 'wpt' command also needs to have virtualenv disabled. |
| group.add_argument( |
| '--wpt-help', |
| action='store_true', |
| help='Show the wptrunner help message and exit') |
| return group |
| |
| def add_output_arguments(self, parser): |
| group = parser.add_argument_group( |
| 'Output Logging', |
| 'Options for controlling logging behavior.') |
| group.add_argument( |
| '--log-wptreport', |
| nargs='?', |
| # We cannot provide a default, since the default filename depends on |
| # the product, so we use this placeholder instead. |
| const='', |
| help=('Log a wptreport in JSON to the output directory ' |
| '(default filename: ' |
| 'wpt_reports_<product>_<shard-index>.json)')) |
| group.add_argument( |
| '-v', |
| '--verbose', |
| action='count', |
| default=0, |
| help='Increase verbosity') |
| return group |
| |
| def _override_options(self, base_parser): |
| """Create a parser that overrides existing options. |
| |
| `argument.ArgumentParser` can extend other parsers and override their |
| options, with the caveat that the child parser only inherits options |
| that the parent had at the time of the child's initialization. There is |
| not a clean way to add option overrides in `add_extra_arguments`, where |
| the provided parser is only passed up the inheritance chain, so we add |
| overridden options here at the very end. |
| |
| See Also: |
| https://docs.python.org/3/library/argparse.html#parents |
| """ |
| parser = argparse.ArgumentParser( |
| parents=[base_parser], |
| # Allow overriding existing options in the parent parser. |
| conflict_handler='resolve', |
| epilog=('All unrecognized arguments are passed through ' |
| "to wptrunner. Use '--wpt-help' to see wptrunner's usage."), |
| ) |
| parser.add_argument( |
| '--isolated-script-test-repeat', |
| '--repeat', |
| '--gtest_repeat', |
| metavar='REPEAT', |
| type=int, |
| default=1, |
| help='Number of times to run the tests') |
| parser.add_argument( |
| '--isolated-script-test-launcher-retry-limit', |
| '--test-launcher-retry-limit', |
| metavar='LIMIT', |
| type=int, |
| default=0, |
| help='Maximum number of times to rerun a failed test') |
| # `--gtest_filter` and `--isolated-script-test-filter` have slightly |
| # different formats and behavior, so keep them as separate options. |
| # See: crbug/1316164#c4 |
| return parser |
| |
| def maybe_set_default_isolated_script_test_output(self): |
| if self.options.isolated_script_test_output: |
| return |
| default_value = DEFAULT_ISOLATED_SCRIPT_TEST_OUTPUT.format( |
| self.options.target) |
| print("--isolated-script-test-output not set, defaulting to %s" % |
| default_value) |
| self.options.isolated_script_test_output = default_value |
| |
| def generate_test_output_args(self, output): |
| return ['--log-chromium=%s' % output] |
| |
| def _resolve_tests_from_isolate_filter(self, test_filter): |
| """Resolve an isolated script-style filter string into lists of tests. |
| |
| Arguments: |
| test_filter (str): Glob patterns delimited by double colons ('::'). |
| The glob is prefixed with '-' to indicate that tests matching |
| the pattern should not run. Assume a valid wpt name cannot start |
| with '-'. |
| |
| Returns: |
| tuple[list[str], list[str]]: Tests to include and exclude, |
| respectively. |
| """ |
| included_tests, excluded_tests = [], [] |
| for pattern in common.extract_filter_list(test_filter): |
| test_group = included_tests |
| if pattern.startswith('-'): |
| test_group, pattern = excluded_tests, pattern[1:] |
| pattern_on_disk = self.fs.join( |
| self.wpt_root_dir, |
| self.path_finder.strip_wpt_path(pattern), |
| ) |
| test_group.extend(glob.glob(pattern_on_disk)) |
| return included_tests, excluded_tests |
| |
| def generate_test_filter_args(self, test_filter_str): |
| included_tests, excluded_tests = \ |
| self._resolve_tests_from_isolate_filter(test_filter_str) |
| include_file, self._include_filename = self.fs.open_text_tempfile() |
| with include_file: |
| for test in included_tests: |
| include_file.write(test) |
| include_file.write('\n') |
| wpt_args = ['--include-file=%s' % self._include_filename] |
| for test in excluded_tests: |
| wpt_args.append('--exclude=%s' % test) |
| return wpt_args |
| |
| def generate_test_repeat_args(self, repeat_count): |
| return ['--repeat=%d' % repeat_count] |
| |
| # pylint: disable=unused-argument |
| def generate_test_launcher_retry_limit_args(self, retry_limit): |
| # TODO(crbug/1306222): wptrunner currently cannot rerun individual |
| # failed tests, so this flag is accepted but not used. |
| return [] |
| |
| def generate_sharding_args(self, total_shards, shard_index): |
| return ['--total-chunks=%d' % total_shards, |
| # shard_index is 0-based but WPT's this-chunk to be 1-based |
| '--this-chunk=%d' % (shard_index + 1), |
| # The default sharding strategy is to shard by directory. But |
| # we want to hash each test to determine which shard runs it. |
| # This allows running individual directories that have few |
| # tests across many shards. |
| '--chunk-type=hash'] |
| |
| def parse_args(self, args=None): |
| super(BaseWptScriptAdapter, self).parse_args(args) |
| if self.options.wpt_help: |
| self._show_wpt_help() |
| # Update the output directory and wptreport filename to defaults if not |
| # set. We cannot provide CLI option defaults because they depend on |
| # other options ('--target' and '--product'). |
| self.maybe_set_default_isolated_script_test_output() |
| report = self.options.log_wptreport |
| if report is not None: |
| if not report: |
| report = self._default_wpt_report() |
| wpt_output = self.options.isolated_script_test_output |
| self.wptreport = self.fs.join(self.fs.dirname(wpt_output), report) |
| |
| def _show_wpt_help(self): |
| command = [ |
| self.select_python_executable(), |
| ] |
| command.extend(self._wpt_run_args) |
| command.extend(['--help']) |
| exit_code = common.run_command(command) |
| self.parser.exit(exit_code) |
| |
| @property |
| def _wpt_run_args(self): |
| """The start of a 'wpt run' command.""" |
| return [ |
| self.wpt_binary, |
| # Use virtualenv packages installed by vpython, not wpt. |
| '--venv=%s' % self.path_finder.chromium_base(), |
| '--skip-venv-setup', |
| 'run', |
| ] |
| |
| @property |
| def rest_args(self): |
| unknown_args = super(BaseWptScriptAdapter, self).rest_args |
| |
| rest_args = list(self._wpt_run_args) |
| rest_args.extend([ |
| # By default, wpt will treat unexpected passes as errors, so we |
| # disable that to be consistent with Chromium CI. |
| '--no-fail-on-unexpected-pass', |
| '--no-pause-after-test', |
| '--no-capture-stdio', |
| '--no-manifest-download', |
| '--tests=%s' % self.wpt_root_dir, |
| '--mojojs-path=%s' % self.mojo_js_directory, |
| ]) |
| |
| if self.options.default_exclude: |
| rest_args.extend(['--default-exclude']) |
| |
| if self.wptreport: |
| rest_args.extend(['--log-wptreport', self.wptreport]) |
| |
| if self.options.verbose >= 3: |
| rest_args.extend([ |
| '--log-mach=-', |
| '--log-mach-level=debug', |
| '--log-mach-verbose', |
| ]) |
| if self.options.verbose >= 4: |
| rest_args.extend([ |
| '--webdriver-arg=--verbose', |
| '--webdriver-arg="--log-path=-"', |
| ]) |
| |
| rest_args.append(self.wpt_product_name()) |
| # We pass through unknown args as late as possible so that they can |
| # override earlier options. It also allows users to pass test names as |
| # positional args, which must not have option strings between them. |
| for unknown_arg in unknown_args: |
| # crbug/1274933#c14: Some developers had used the end-of-options |
| # marker '--' to pass through arguments to wptrunner. |
| # crrev.com/c/3573284 makes this no longer necessary. |
| if unknown_arg == '--': |
| logger.warning( |
| 'Unrecognized options will automatically fall through ' |
| 'to wptrunner.') |
| logger.warning( |
| "There is no need to use the end-of-options marker '--'.") |
| else: |
| rest_args.append(unknown_arg) |
| return rest_args |
| |
| def do_post_test_run_tasks(self): |
| if not self.wpt_output and self.options: |
| self.wpt_output = self.options.isolated_script_test_output |
| |
| command = [ |
| self.select_python_executable(), |
| os.path.join(BLINK_TOOLS_DIR, 'wpt_process_results.py'), |
| '--target', |
| self.options.target, |
| '--web-tests-dir', |
| WEB_TESTS_DIR, |
| '--artifacts-dir', |
| os.path.join(os.path.dirname(self.wpt_output), |
| self.layout_test_results_subdir), |
| '--wpt-results', |
| self.wpt_output, |
| ] |
| if self.options.verbose: |
| command.append('--verbose') |
| if self.wptreport: |
| command.extend(['--wpt-report', self.wptreport]) |
| common.run_command(command) |
| |
| if self._include_filename: |
| self.fs.remove(self._include_filename) |
| |
| def wpt_product_name(self): |
| raise NotImplementedError |
| |
| def _default_wpt_report(self): |
| product = self.wpt_product_name() |
| shard_index = os.environ.get('GTEST_SHARD_INDEX') |
| if shard_index is not None: |
| return 'wpt_reports_%s_%02d.json' % (product, int(shard_index)) |
| return 'wpt_reports_%s.json' % product |
