| #!/usr/bin/env python3 | |
| # Copyright (C) 2018-2021 Apple Inc. All rights reserved. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions | |
| # are met: | |
| # 1. Redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer. | |
| # 2. Redistributions in binary form must reproduce the above copyright | |
| # notice, this list of conditions and the following disclaimer in the | |
| # documentation and/or other materials provided with the distribution. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY | |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | |
| # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY | |
| # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES | |
| # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |
| # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON | |
| # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| from __future__ import print_function | |
| import logging | |
| import optparse | |
| import sys | |
| import traceback | |
| from webkitpy.port.linux_container_sdk_utils import maybe_enter_webkit_container_sdk | |
| maybe_enter_webkit_container_sdk() | |
| from webkitpy.api_tests.manager import Manager | |
| from webkitpy.api_tests.test_expectations import APITestExpectations | |
| from webkitpy.common.host import Host | |
| from webkitpy.common.program_status import ProgramStatus | |
| from webkitpy.layout_tests.views.metered_stream import MeteredStream | |
| from webkitpy.port import configuration_options, platform_options, base, win | |
| from webkitpy.results.options import upload_options | |
| EXCEPTIONAL_EXIT_STATUS = -1 | |
| INTERRUPT_EXIT_STATUS = -2 | |
| _log = logging.getLogger(__name__) | |
| def main(argv, stdout, stderr): | |
| options, args = parse_args(argv) | |
| host = Host() | |
| try: | |
| options.webkit_test_runner = True | |
| port = host.port_factory.get(options.platform, options) | |
| except NotImplementedError as e: | |
| print(str(e), file=stderr) | |
| return EXCEPTIONAL_EXIT_STATUS | |
| # Some platforms do not support API tests | |
| does_not_support_api_tests = ['ios-device'] | |
| if port.operating_system() in does_not_support_api_tests: | |
| print('{} cannot run API tests'.format(port.operating_system()), file=stderr) | |
| return EXCEPTIONAL_EXIT_STATUS | |
| if options.lint_test_files or options.fix_expectations: | |
| return lint_expectations(port, options, args, stderr) | |
| if options.print_expectations: | |
| return print_expectations(port, options, args, stderr) | |
| # Add site-isolation results report flavor for API tests when ran | |
| if options.site_isolation_enabled_by_default: | |
| if options.result_report_flavor: | |
| raise RuntimeError('--site-isolation-enabled-by-default implicitly sets the result flavor, this should not be overridden') | |
| options.result_report_flavor = 'site-isolation' | |
| try: | |
| return run(port, options, args, stderr) | |
| except KeyboardInterrupt: | |
| return INTERRUPT_EXIT_STATUS | |
| except BaseException as e: | |
| if isinstance(e, Exception): | |
| print('\n%s raised: %s' % (e.__class__.__name__, str(e)), file=stderr) | |
| traceback.print_exc(file=stderr) | |
| return EXCEPTIONAL_EXIT_STATUS | |
| def lint_expectations(port, options, args, logging_stream): | |
| expectations = APITestExpectations(port) | |
| expectations.parse_all_expectations() | |
| if options.fix_expectations: | |
| print('Fixing API test expectations...', file=logging_stream) | |
| changed = expectations.fix() | |
| for filepath in changed: | |
| print(' Fixed {}'.format(filepath), file=logging_stream) | |
| print('\n{} file(s) fixed.'.format(len(changed)) if changed else 'No fixes needed.', file=logging_stream) | |
| return 0 | |
| print('Linting API test expectations...', file=logging_stream) | |
| warnings = expectations.lint() | |
| if warnings: | |
| print('\nWarnings:', file=logging_stream) | |
| for warning in warnings: | |
| print(' {}'.format(warning), file=logging_stream) | |
| print('\n{} warning(s) found.'.format(len(warnings)), file=logging_stream) | |
| return 1 | |
| else: | |
| print('No issues found.', file=logging_stream) | |
| return 0 | |
| def print_expectations(port, options, args, logging_stream): | |
| expectations = APITestExpectations(port) | |
| expectations.parse_all_expectations() | |
| config = expectations.get_current_configuration() | |
| manager = expectations.model() | |
| if args: | |
| for test_pattern in args: | |
| exp = expectations.get_expectation(test_pattern, config) | |
| if exp: | |
| exp_names = ', '.join(str(m) for m in type(exp.expected) if m in exp.expected) | |
| print('{}: [ {} ]'.format(test_pattern, exp_names), file=logging_stream) | |
| else: | |
| print('{}: [ Pass ] (default)'.format(test_pattern), file=logging_stream) | |
| else: | |
| for exp in manager.all_expectations(): | |
| exp_names = ', '.join(str(m) for m in type(exp.expected) if m in exp.expected) | |
| bug_str = ' '.join(exp.bug_ids) + ' ' if exp.bug_ids else '' | |
| config_str = '[ {} ] '.format(' '.join(exp.configurations)) if exp.configurations else '' | |
| print('{}{}{} [ {} ]'.format(bug_str, config_str, exp.test_pattern, exp_names), file=logging_stream) | |
| return 0 | |
| def run(port, options, args, logging_stream): | |
| logger = logging.getLogger() | |
| logger.setLevel(logging.DEBUG if options.verbose else logging.ERROR if options.quiet else logging.INFO) | |
| status = ProgramStatus(logging_stream, 'run-api-tests') | |
| try: | |
| stream = MeteredStream(logging_stream, options.verbose, logger=logger, number_of_columns=port.host.platform.terminal_width(), print_timestamps=options.timestamps) | |
| if options.site_isolation_enabled_by_default: | |
| _log.info('Enabling site isolation by default') | |
| manager = Manager(port, options, stream, status=status) | |
| result = manager.run(args, json_output=options.json_output) | |
| _log.debug("Testing completed, Exit status: %d" % result) | |
| return result | |
| finally: | |
| stream.cleanup() | |
| status.close() | |
| def parse_args(args): | |
| option_group_definitions = [] | |
| option_group_definitions.append(('Platform options', platform_options())) | |
| option_group_definitions.append(('Configuration options', configuration_options())) | |
| option_group_definitions.append(('Printing Options', [ | |
| optparse.make_option('-q', '--quiet', action='store_true', default=False, | |
| help='Run quietly (errors, warnings, and progress only)'), | |
| optparse.make_option('-v', '--verbose', action='store_true', default=False, | |
| help='Enable verbose printing'), | |
| optparse.make_option('--timestamps', action='store_true', default=False, | |
| help='Print timestamps for each logged line'), | |
| optparse.make_option('--json-output', action='store', default=None, | |
| help='Save test results as JSON to file'), | |
| ])) | |
| option_group_definitions.append(('WebKit Options', [ | |
| optparse.make_option('-g', '--guard-malloc', action='store_true', default=False, | |
| help='Enable Guard Malloc (OS X only)'), | |
| optparse.make_option('--root', action='store', | |
| help='Path to a directory containing the executables needed to run tests.'), | |
| ])) | |
| option_group_definitions.append(('Testing Options', [ | |
| optparse.make_option('--wtf-only', action='store_const', const='TestWTF', dest='api_binary', | |
| help='Only build, check and run TestWTF'), | |
| optparse.make_option('--webkit-only', action='store_const', const='TestWebKitAPI', dest='api_binary', | |
| help='Only check and run TestWebKitAPI'), | |
| optparse.make_option('--ipc-only', action='store_const', const='TestIPC', dest='api_binary', | |
| help='Only check and run TestIPC (Darwin ports only)'), | |
| optparse.make_option('--web-core-only', action='store_const', const='TestWebCore', dest='api_binary', | |
| help='Only check and run TestWebCore.exe (Windows only)'), | |
| optparse.make_option('--webkit-legacy-only', action='store_const', const='TestWebKitLegacy', dest='api_binary', | |
| help='Only check and run TestWebKitLegacy.exe (Windows only)'), | |
| optparse.make_option('--wgsl-only', action='store_const', const='TestWGSL', dest='api_binary', | |
| help='Only check and run TestWGSL (Darwin ports only)'), | |
| optparse.make_option('-d', '--dump', '--list-tests', action='store_true', default=False, | |
| help='Dump all test names without running them'), | |
| optparse.make_option('--build', dest='build', action='store_true', default=False, | |
| help='Check to ensure the build is up-to-date.'), | |
| optparse.make_option('--no-build', dest='build', action='store_false', | |
| help="Don't check to see if the build is up-to-date (default)."), | |
| optparse.make_option('--timeout', default=30, | |
| help='Number of seconds to wait before a test times out'), | |
| optparse.make_option('--no-timeout', dest='timeout', action='store_false', | |
| help='Disable timeouts for all tests'), | |
| optparse.make_option('--iterations', type='int', default=1, help='Number of times to run the set of tests (e.g. ABCABCABC)'), | |
| optparse.make_option('--repeat-each', type='int', default=1, help='Number of times to run each test (e.g. AAABBBCCC)'), | |
| optparse.make_option('--child-processes', default=None, | |
| help='Number of processes to run in parallel.'), | |
| # FIXME: Default should be false, API tests should not be forced to run singly | |
| optparse.make_option('--run-singly', action='store_true', default=True, | |
| help='Run a separate process for each test'), | |
| optparse.make_option('--test-list', action='append', | |
| help='Read list of tests to run from file', metavar='FILE'), | |
| optparse.make_option('--force', action='store_true', default=False, | |
| help='Run all tests, even DISABLED tests'), | |
| optparse.make_option('-f', '--fully-parallel', action='store_true', default=False, | |
| help='Force all tests to run in parallel (without grouping shards)'), | |
| optparse.make_option('--exit-after-n-failures', type='int', default=None, | |
| help='Exit after the first N failures instead of running all tests'), | |
| optparse.make_option('--additional-env-var', type='string', action='append', default=[], | |
| help='Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)'), | |
| optparse.make_option('--test-parallel-safety', type='string', action='append', default=[], | |
| help='Run specified test(s) repeatedly while the allowlisted tests run on repeat as background load'), | |
| optparse.make_option('--parallel-safety-iterations', type='int', default=10, | |
| help='Number of times to run each --test-parallel-safety test (default: %default)'), | |
| optparse.make_option('--site-isolation-enabled-by-default', action='store_true', default=False, | |
| help='Enable Site Isolation'), | |
| optparse.make_option('--remote-layer-tree', action='store_true', default=False, | |
| help='Use the remote layer tree drawing model (macOS WebKit2 only)'), | |
| optparse.make_option('--no-remote-layer-tree', action='store_true', default=False, | |
| help='Disable the remote layer tree drawing model (macOS WebKit2 only)'), | |
| optparse.make_option('--use-gpu-process', action='store_true', default=False, | |
| help='Enable GPU process DOM rendering'), | |
| optparse.make_option('--no-use-gpu-process', action='store_true', default=False, | |
| help='Disable GPU process DOM rendering'), | |
| ])) | |
| option_group_definitions.append(('Expectation Options', [ | |
| optparse.make_option('--additional-expectations', action='append', default=[], | |
| help='Path to a test expectations file that will override previous expectations. ' | |
| 'Specify multiple times for multiple sets of overrides.'), | |
| optparse.make_option('--skip-failing-tests', action='store_true', default=False, | |
| help='Skip tests that are marked as failing or flaky.'), | |
| optparse.make_option('--skip-flaky-tests', action='store_true', default=False, | |
| help='Skip tests that are marked as flaky.'), | |
| optparse.make_option('--skipped', action='store', default='default', | |
| choices=['default', 'ignore', 'only', 'always'], | |
| help="Control how tests marked SKIP are run. " | |
| "'default' == Skip tests unless explicitly listed on the command line, " | |
| "'ignore' == Run them anyway, " | |
| "'only' == only run the SKIP tests, " | |
| "'always' == always skip, even if listed on the command line."), | |
| optparse.make_option('--lint-test-files', action='store_true', default=False, | |
| help='Validate expectations files. Does not run any tests.'), | |
| optparse.make_option('--fix-expectations', action='store_true', default=False, | |
| help='Reorganize and reformat API test expectations files (sections, alphabetical order). Does not run any tests.'), | |
| optparse.make_option('--print-expectations', action='store_true', default=False, | |
| help='Print expected outcomes for given tests. Does not run any tests.'), | |
| optparse.make_option('--check-pre-existing-failures', action='store_true', | |
| default=False, dest='check_pre_existing_failures', | |
| help='After the run, consult results.webkit.org to annotate each unexpected ' | |
| 'failure as pre-existing (historically failing on CI) or possibly new.'), | |
| optparse.make_option('--max-pre-existing-checks', type='int', default=0, | |
| dest='max_pre_existing_checks', | |
| help='Maximum number of failing tests to look up in results.webkit.org when ' | |
| '--check-pre-existing-failures is set. Use 0 for no limit. (default: 0)'), | |
| ])) | |
| option_group_definitions.append(('Upload Options', upload_options())) | |
| option_parser = optparse.OptionParser( | |
| usage='run-api-tests [options] [<test names>...]', | |
| description="""By default, run-api-tests will run all API tests. It also allows the user to specify tests of the \ | |
| format <suite>.<test> or <canonicalized binary name>.<suite>.<test>. Note that in the case where a binary is not \ | |
| specified, one will be inferred by listing all available tests. Specifying just a binary or just a suite will cause every \ | |
| test contained within to be run. The canonicalized binary name is the binary name with any filename extension \ | |
| stripped. For Unix ports, these binaries are {} and {}. For Windows ports, they are {} and {}.""".format( | |
| ', '.join(base.Port.API_TEST_BINARY_NAMES[:-1]), base.Port.API_TEST_BINARY_NAMES[-1], | |
| ', '.join(win.WinPort.API_TEST_BINARY_NAMES[:-1]), win.WinPort.API_TEST_BINARY_NAMES[-1], | |
| )) | |
| for group_name, group_options in option_group_definitions: | |
| option_group = optparse.OptionGroup(option_parser, group_name) | |
| option_group.add_options(group_options) | |
| option_parser.add_option_group(option_group) | |
| return option_parser.parse_args(args) | |
| if __name__ == '__main__': | |
| sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr)) |