| #!/usr/bin/env vpython3 |
| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Runs telemetry benchmarks and gtest perf tests. |
| |
| If optional argument --isolated-script-test-output=[FILENAME] is passed |
| to the script, json is written to that file in the format detailed in |
| //docs/testing/json-test-results-format.md. |
| |
| If optional argument --isolated-script-test-filter=[TEST_NAMES] is passed to |
| the script, it should be a double-colon-separated ("::") list of test names, |
| to run just that subset of tests. |
| |
| This script is intended to be the base command invoked by the isolate, |
| followed by a subsequent Python script. It could be generalized to |
| invoke an arbitrary executable. |
| It currently runs several benchmarks. The benchmarks it will execute are |
| based on the shard it is running on and the sharding_map_path. |
| |
| If this is executed with a gtest perf test, the flag --non-telemetry |
| has to be passed in to the script so the script knows it is running |
| an executable and not the run_benchmark command. |
| |
| This script merges test results from all the benchmarks into the one |
| output.json file. The test results and perf results are also put in separate |
| directories per benchmark. Two files will be present in each directory; |
| perf_results.json, which is the perf specific results (with unenforced format, |
| could be histogram or graph json), and test_results.json. |
| |
| TESTING: |
| To test changes to this script, please run |
| cd tools/perf |
| ./run_tests ScriptsSmokeTest.testRunPerformanceTests |
| """ |
| |
| from __future__ import print_function |
| |
| import argparse |
| import json |
| import os |
| import requests |
| import shutil |
| import sys |
| import time |
| import tempfile |
| import traceback |
| import six |
| |
| from collections import OrderedDict |
| |
| CHROMIUM_SRC_DIR = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), |
| os.path.pardir, os.path.pardir)) |
| |
| PERF_DIR = os.path.join(CHROMIUM_SRC_DIR, 'tools', 'perf') |
| sys.path.append(PERF_DIR) |
| import generate_legacy_perf_dashboard_json |
| from core import path_util |
| |
| PERF_CORE_DIR = os.path.join(PERF_DIR, 'core') |
| sys.path.append(PERF_CORE_DIR) |
| import results_merger |
| |
| # Add src/testing/ into sys.path for importing xvfb, test_env, and common. |
| sys.path.append( |
| os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) |
| import xvfb |
| import test_env |
| from scripts import common |
| |
| SHARD_MAPS_DIRECTORY = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, |
| 'tools', 'perf', 'core', 'shard_maps')) |
| |
| # See https://crbug.com/923564. |
| # We want to switch over to using histograms for everything, but converting from |
| # the format output by gtest perf tests to histograms has introduced several |
| # problems. So, only perform the conversion on tests that are whitelisted and |
| # are okay with potentially encountering issues. |
| GTEST_CONVERSION_WHITELIST = [ |
| 'angle_perftests', |
| 'base_perftests', |
| 'blink_heap_perftests', |
| 'blink_platform_perftests', |
| 'cc_perftests', |
| 'components_perftests', |
| 'command_buffer_perftests', |
| 'dawn_perf_tests', |
| 'gpu_perftests', |
| 'load_library_perf_tests', |
| 'net_perftests', |
| 'browser_tests', |
| 'services_perftests', |
| # TODO(jmadill): Remove once migrated. http://anglebug.com/5124 |
| 'standalone_angle_perftests', |
| 'sync_performance_tests', |
| 'tracing_perftests', |
| 'views_perftests', |
| 'viz_perftests', |
| 'wayland_client_perftests', |
| 'xr.vr.common_perftests', |
| ] |
| |
| # pylint: disable=useless-object-inheritance |
| |
| |
| class OutputFilePaths(object): |
| """Provide paths to where results outputs should be written. |
| |
| The process_perf_results.py merge script later will pull all of these |
| together, so that's why they aren't in the standard locations. Also, |
| note that because of the OBBS (One Build Bot Step), Telemetry |
| has multiple tests running on a single shard, so we need to prefix |
| these locations with a directory named by the benchmark name. |
| """ |
| |
| def __init__(self, isolated_out_dir, perf_test_name): |
| self.name = perf_test_name |
| self.benchmark_path = os.path.join(isolated_out_dir, perf_test_name) |
| |
| def SetUp(self): |
| os.makedirs(self.benchmark_path) |
| return self |
| |
| @property |
| def perf_results(self): |
| return os.path.join(self.benchmark_path, 'perf_results.json') |
| |
| @property |
| def test_results(self): |
| return os.path.join(self.benchmark_path, 'test_results.json') |
| |
| @property |
| def logs(self): |
| return os.path.join(self.benchmark_path, 'benchmark_log.txt') |
| |
| @property |
| def csv_perf_results(self): |
| """Path for csv perf results. |
| |
| Note that the chrome.perf waterfall uses the json histogram perf results |
| exclusively. csv_perf_results are implemented here in case a user script |
| passes --output-format=csv. |
| """ |
| return os.path.join(self.benchmark_path, 'perf_results.csv') |
| |
| |
| def print_duration(step, start): |
| print('Duration of %s: %d seconds' % (step, time.time() - start)) |
| |
| |
| def IsWindows(): |
| return sys.platform == 'cygwin' or sys.platform.startswith('win') |
| |
| |
| class GtestCommandGenerator(object): |
| def __init__(self, options, override_executable=None, additional_flags=None, |
| ignore_shard_env_vars=False): |
| self._options = options |
| self._override_executable = override_executable |
| self._additional_flags = additional_flags or [] |
| self._ignore_shard_env_vars = ignore_shard_env_vars |
| |
| def generate(self, output_dir): |
| """Generate the command to run to start the gtest perf test. |
| |
| Returns: |
| list of strings, the executable and its arguments. |
| """ |
| return ([self._get_executable()] + |
| self._generate_filter_args() + |
| self._generate_repeat_args() + |
| self._generate_also_run_disabled_tests_args() + |
| self._generate_output_args(output_dir) + |
| self._generate_shard_args() + |
| self._get_additional_flags() |
| ) |
| |
| @property |
| def executable_name(self): |
| """Gets the platform-independent name of the executable.""" |
| return self._override_executable or self._options.executable |
| |
| def _get_executable(self): |
| executable = str(self.executable_name) |
| if IsWindows(): |
| return r'.\%s.exe' % executable |
| return './%s' % executable |
| |
| def _get_additional_flags(self): |
| return self._additional_flags |
| |
| def _generate_shard_args(self): |
| """Teach the gtest to ignore the environment variables. |
| |
| GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS will confuse the gtest |
| and convince it to only run some of its tests. Instead run all |
| of them. |
| """ |
| if self._ignore_shard_env_vars: |
| return ['--test-launcher-total-shards=1', '--test-launcher-shard-index=0'] |
| return [] |
| |
| def _generate_filter_args(self): |
| if self._options.isolated_script_test_filter: |
| filter_list = common.extract_filter_list( |
| self._options.isolated_script_test_filter) |
| return ['--gtest_filter=' + ':'.join(filter_list)] |
| return [] |
| |
| def _generate_repeat_args(self): |
| # TODO(crbug.com/920002): Support --isolated-script-test-repeat. |
| return [] |
| |
| def _generate_also_run_disabled_tests_args(self): |
| # TODO(crbug.com/920002): Support |
| # --isolated-script-test-also-run-disabled-tests. |
| return [] |
| |
| def _generate_output_args(self, output_dir): |
| output_args = [] |
| if self._options.use_gtest_benchmark_script: |
| output_args.append('--output-dir=' + output_dir) |
| # These flags are to make sure that test output perf metrics in the log. |
| if not '--verbose' in self._get_additional_flags(): |
| output_args.append('--verbose') |
| if (not '--test-launcher-print-test-stdio=always' |
| in self._get_additional_flags()): |
| output_args.append('--test-launcher-print-test-stdio=always') |
| return output_args |
| |
| |
| def write_simple_test_results(return_code, output_filepath, benchmark_name): |
| # TODO(crbug.com/1115658): Fix to output |
| # https://chromium.googlesource.com/chromium/src/+/main/docs/testing/json_test_results_format.md |
| # for each test rather than this summary. |
| # Append the shard index to the end of the name so that the merge script |
| # doesn't blow up trying to merge unmergeable results. |
| benchmark_name += '_shard_%s' % os.environ.get('GTEST_SHARD_INDEX', '0') |
| output_json = { |
| 'tests': { |
| benchmark_name: { |
| 'expected': 'PASS', |
| 'actual': 'FAIL' if return_code else 'PASS', |
| 'is_unexpected': bool(return_code), |
| }, |
| }, |
| 'interrupted': False, |
| 'path_delimiter': '/', |
| 'version': 3, |
| 'seconds_since_epoch': time.time(), |
| 'num_failures_by_type': { |
| 'FAIL': 1 if return_code else 0, |
| 'PASS': 0 if return_code else 1, |
| }, |
| } |
| with open(output_filepath, 'w') as fh: |
| json.dump(output_json, fh) |
| |
| |
| def upload_simple_test_results(return_code, benchmark_name): |
| # TODO(crbug.com/1115658): Fix to upload results for each test rather than |
| # this summary. |
| try: |
| with open(os.environ['LUCI_CONTEXT']) as f: |
| sink = json.load(f)['result_sink'] |
| except KeyError: |
| return |
| |
| if return_code: |
| summary = '<p>Benchmark failed with status code %d</p>' % return_code |
| else: |
| summary = '<p>Benchmark passed</p>' |
| |
| result_json = { |
| 'testResults': [{ |
| 'testId': benchmark_name, |
| 'expected': not return_code, |
| 'status': 'FAIL' if return_code else 'PASS', |
| 'summaryHtml': summary, |
| 'tags': [{'key': 'exit_code', 'value': str(return_code)}], |
| }] |
| } |
| |
| res = requests.post( |
| url='http://%s/prpc/luci.resultsink.v1.Sink/ReportTestResults' % |
| sink['address'], |
| headers={ |
| 'Content-Type': 'application/json', |
| 'Accept': 'application/json', |
| 'Authorization': 'ResultSink %s' % sink['auth_token'], |
| }, |
| data=json.dumps(result_json)) |
| res.raise_for_status() |
| |
| |
| def execute_gtest_perf_test(command_generator, output_paths, use_xvfb=False, |
| is_unittest=False, results_label=None): |
| start = time.time() |
| |
| env = os.environ.copy() |
| env['CHROME_HEADLESS'] = '1' |
| #TODO(crbug/1138988): Some gtests do not implements the unit_test_launcher.cc. |
| # As a result, they will not respect the arguments added by |
| # _generate_shard_args() and will still use the values of GTEST_SHARD_INDEX |
| # and GTEST_TOTAL_SHARDS to run part of the tests. |
| # Removing those environment variables as a workaround. |
| if command_generator._ignore_shard_env_vars: |
| if 'GTEST_TOTAL_SHARDS' in env: |
| env.pop('GTEST_TOTAL_SHARDS') |
| if 'GTEST_SHARD_INDEX' in env: |
| env.pop('GTEST_SHARD_INDEX') |
| |
| return_code = 0 |
| try: |
| command = command_generator.generate(output_paths.benchmark_path) |
| if use_xvfb: |
| # When running with xvfb, we currently output both to stdout and to the |
| # file. It would be better to only output to the file to keep the logs |
| # clean. |
| return_code = xvfb.run_executable( |
| command, env, stdoutfile=output_paths.logs) |
| else: |
| with open(output_paths.logs, 'w') as handle: |
| try: |
| return_code = test_env.run_command_output_to_handle( |
| command, handle, env=env) |
| except OSError as e: |
| print('Command to run gtest perf test %s failed with an OSError: %s' % |
| (output_paths.name, e)) |
| return_code = 1 |
| if (not os.path.exists(output_paths.perf_results) and |
| os.path.exists(output_paths.logs)): |
| # Get the correct json format from the stdout to write to the perf |
| # results file if gtest does not generate one. |
| results_processor = generate_legacy_perf_dashboard_json.\ |
| LegacyResultsProcessor() |
| graph_json_string = results_processor.GenerateJsonResults( |
| output_paths.logs) |
| with open(output_paths.perf_results, 'w') as fh: |
| fh.write(graph_json_string) |
| except Exception: |
| traceback.print_exc() |
| return_code = 1 |
| if os.path.exists(output_paths.perf_results): |
| executable_name = command_generator.executable_name |
| if executable_name.startswith('bin/run_'): |
| # The executable is a wrapper used by Fuchsia. Remove the prefix to get |
| # the actual executable name. |
| executable_name = executable_name[8:] |
| if executable_name in GTEST_CONVERSION_WHITELIST: |
| with path_util.SysPath(path_util.GetTracingDir()): |
| # pylint: disable=no-name-in-module,import-outside-toplevel |
| from tracing.value import gtest_json_converter |
| # pylint: enable=no-name-in-module,import-outside-toplevel |
| gtest_json_converter.ConvertGtestJsonFile(output_paths.perf_results, |
| label=results_label) |
| else: |
| print('ERROR: gtest perf test %s did not generate perf output' % |
| output_paths.name) |
| return_code = 1 |
| write_simple_test_results(return_code, output_paths.test_results, |
| output_paths.name) |
| if not is_unittest: |
| upload_simple_test_results(return_code, output_paths.name) |
| |
| print_duration( |
| 'executing gtest %s' % command_generator.executable_name, start) |
| |
| return return_code |
| |
| |
| class _TelemetryFilterArgument(object): |
| def __init__(self, filter_string): |
| self.benchmark, self.story = filter_string.split('/') |
| |
| |
| class TelemetryCommandGenerator(object): |
| def __init__(self, benchmark, options, |
| story_selection_config=None, is_reference=False): |
| self.benchmark = benchmark |
| self._options = options |
| self._story_selection_config = story_selection_config |
| self._is_reference = is_reference |
| |
| def generate(self, output_dir): |
| """Generate the command to run to start the benchmark. |
| |
| Args: |
| output_dir: The directory to configure the command to put output files |
| into. |
| |
| Returns: |
| list of strings, the executable and its arguments. |
| """ |
| return ([sys.executable] + self._options.executable.split(' ') + |
| [self.benchmark] + |
| self._generate_filter_args() + |
| self._generate_also_run_disabled_tests_args() + |
| self._generate_output_args(output_dir) + |
| self._generate_story_selection_args() + |
| # passthrough args must be before reference args and repeat args: |
| # crbug.com/928928, crbug.com/894254#c78 |
| self._get_passthrough_args() + |
| self._generate_syslog_args() + |
| self._generate_repeat_args() + |
| self._generate_reference_build_args() + |
| self._generate_results_label_args() |
| ) |
| |
| def _get_passthrough_args(self): |
| return self._options.passthrough_args |
| |
| def _generate_filter_args(self): |
| if self._options.isolated_script_test_filter: |
| filter_list = common.extract_filter_list( |
| self._options.isolated_script_test_filter) |
| filter_arguments = [_TelemetryFilterArgument(f) for f in filter_list] |
| applicable_stories = [ |
| f.story for f in filter_arguments if f.benchmark == self.benchmark] |
| # Need to convert this to a valid regex. |
| filter_regex = '(' + '|'.join(applicable_stories) + ')' |
| return ['--story-filter=' + filter_regex] |
| return [] |
| |
| def _generate_repeat_args(self): |
| if self._options.isolated_script_test_repeat: |
| return ['--pageset-repeat=' + str( |
| self._options.isolated_script_test_repeat)] |
| return [] |
| |
| def _generate_also_run_disabled_tests_args(self): |
| if self._options.isolated_script_test_also_run_disabled_tests: |
| return ['--also-run-disabled-tests'] |
| return [] |
| |
| def _generate_output_args(self, output_dir): |
| return ['--output-format=json-test-results', |
| '--output-format=histograms', |
| '--output-dir=' + output_dir] |
| |
| def _generate_story_selection_args(self): |
| """Returns arguments that limit the stories to be run inside the benchmark. |
| """ |
| selection_args = [] |
| if self._story_selection_config: |
| if 'begin' in self._story_selection_config: |
| selection_args.append('--story-shard-begin-index=%d' % ( |
| self._story_selection_config['begin'])) |
| if 'end' in self._story_selection_config: |
| selection_args.append('--story-shard-end-index=%d' % ( |
| self._story_selection_config['end'])) |
| if 'sections' in self._story_selection_config: |
| range_string = self._generate_story_index_ranges( |
| self._story_selection_config['sections']) |
| if range_string: |
| selection_args.append('--story-shard-indexes=%s' % range_string) |
| if self._story_selection_config.get('abridged', True): |
| selection_args.append('--run-abridged-story-set') |
| return selection_args |
| |
| def _generate_syslog_args(self): |
| if self._options.per_test_logs_dir: |
| isolated_out_dir = os.path.dirname( |
| self._options.isolated_script_test_output) |
| return ['--logs-dir', os.path.join( |
| isolated_out_dir, |
| self.benchmark)] |
| return [] |
| |
| |
| def _generate_story_index_ranges(self, sections): |
| range_string = '' |
| for section in sections: |
| begin = section.get('begin', '') |
| end = section.get('end', '') |
| # If there only one story in the range, we only keep its index. |
| # In general, we expect either begin or end, or both. |
| if begin != '' and end != '' and end - begin == 1: |
| new_range = str(begin) |
| elif begin != '' or end != '': |
| new_range = '%s-%s' % (str(begin), str(end)) |
| else: |
| raise ValueError('Index ranges in "sections" in shard map should have' |
| 'at least one of "begin" and "end": %s' % str(section)) |
| if range_string: |
| range_string += ',%s' % new_range |
| else: |
| range_string = new_range |
| return range_string |
| |
| |
| def _generate_reference_build_args(self): |
| if self._is_reference: |
| reference_browser_flag = '--browser=reference' |
| # TODO(crbug.com/1038137): Make the logic generic once more reference |
| # settings are added |
| if '--browser=android-chrome-bundle' in self._get_passthrough_args(): |
| reference_browser_flag = '--browser=reference-android-chrome-bundle' |
| return [reference_browser_flag, |
| '--max-failures=5'] |
| return [] |
| |
| def _generate_results_label_args(self): |
| if self._options.results_label: |
| return ['--results-label=' + self._options.results_label] |
| return [] |
| |
| def execute_telemetry_benchmark( |
| command_generator, output_paths, use_xvfb=False, |
| return_exit_code_zero=False): |
| start = time.time() |
| |
| env = os.environ.copy() |
| env['CHROME_HEADLESS'] = '1' |
| |
| return_code = 1 |
| temp_dir = tempfile.mkdtemp('telemetry') |
| infra_failure = False |
| try: |
| command = command_generator.generate(temp_dir) |
| if use_xvfb: |
| # When running with xvfb, we currently output both to stdout and to the |
| # file. It would be better to only output to the file to keep the logs |
| # clean. |
| return_code = xvfb.run_executable( |
| command, env=env, stdoutfile=output_paths.logs) |
| else: |
| with open(output_paths.logs, 'w') as handle: |
| return_code = test_env.run_command_output_to_handle( |
| command, handle, env=env) |
| expected_results_filename = os.path.join(temp_dir, 'test-results.json') |
| if os.path.exists(expected_results_filename): |
| shutil.move(expected_results_filename, output_paths.test_results) |
| else: |
| common.write_interrupted_test_results_to(output_paths.test_results, start) |
| expected_perf_filename = os.path.join(temp_dir, 'histograms.json') |
| shutil.move(expected_perf_filename, output_paths.perf_results) |
| |
| csv_file_path = os.path.join(temp_dir, 'results.csv') |
| if os.path.isfile(csv_file_path): |
| shutil.move(csv_file_path, output_paths.csv_perf_results) |
| except Exception: |
| print('The following exception may have prevented the code from ' |
| 'outputing structured test results and perf results output:') |
| print(traceback.format_exc()) |
| infra_failure = True |
| finally: |
| # On swarming bots, don't remove output directory, since Result Sink might |
| # still be uploading files to Result DB. Also, swarming bots automatically |
| # clean up at the end of each task. |
| if 'SWARMING_TASK_ID' not in os.environ: |
| # Add ignore_errors=True because otherwise rmtree may fail due to leaky |
| # processes of tests are still holding opened handles to files under |
| # |tempfile_dir|. For example, see crbug.com/865896 |
| shutil.rmtree(temp_dir, ignore_errors=True) |
| |
| print_duration('executing benchmark %s' % command_generator.benchmark, start) |
| |
| if infra_failure: |
| print('There was an infrastructure error encountered during the run. ' |
| 'Please check the logs above for details') |
| return 1 |
| |
| # Telemetry sets exit code to -1 to indicate that no stories were run. This |
| # becomes 255 on linux because linux doesn't support -1 so it does modulo: |
| # -1 % 256 == 255. |
| # TODO(crbug.com/1019139): Make 111 be the exit code that means |
| # "no stories were run.". |
| if return_code in (111, -1, 255): |
| print('Exit code %s indicates that no stories were run, so we are marking ' |
| 'this as a success.' % return_code) |
| return 0 |
| if return_code: |
| if return_exit_code_zero: |
| print ('run_benchmark returned exit code ' + str(return_code) |
| + ' which indicates there were test failures in the run.') |
| return 0 |
| return return_code |
| return 0 |
| |
| def parse_arguments(args): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('executable', help='The name of the executable to run.') |
| parser.add_argument( |
| '--isolated-script-test-output', required=True) |
| # The following two flags may be passed in sometimes by Pinpoint |
| # or by the recipe, but they don't do anything. crbug.com/927482. |
| parser.add_argument( |
| '--isolated-script-test-chartjson-output', required=False) |
| parser.add_argument( |
| '--isolated-script-test-perf-output', required=False) |
| |
| parser.add_argument( |
| '--isolated-script-test-filter', type=str, required=False) |
| |
| # Note that the following three arguments are only supported by Telemetry |
| # tests right now. See crbug.com/920002. |
| parser.add_argument( |
| '--isolated-script-test-repeat', type=int, required=False) |
| parser.add_argument( |
| '--isolated-script-test-launcher-retry-limit', type=int, required=False, |
| choices=[0]) # Telemetry does not support retries. crbug.com/894254#c21 |
| parser.add_argument( |
| '--isolated-script-test-also-run-disabled-tests', |
| default=False, action='store_true', required=False) |
| parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') |
| parser.add_argument('--non-telemetry', |
| help='Type of perf test', type=bool, default=False) |
| parser.add_argument('--gtest-benchmark-name', |
| help='Name of the gtest benchmark', type=str, |
| required=False) |
| parser.add_argument('--use-gtest-benchmark-script', |
| help='Whether gtest is invoked via benchmark script.', |
| default=False, action='store_true') |
| |
| parser.add_argument('--benchmarks', |
| help='Comma separated list of benchmark names' |
| ' to run in lieu of indexing into our benchmark bot maps', |
| required=False) |
| # crbug.com/1236245: This allows for per-benchmark device logs. |
| parser.add_argument('--per-test-logs-dir', |
| help='Require --logs-dir args for test', required=False, |
| default=False, action='store_true') |
| # Some executions may have a different sharding scheme and/or set of tests. |
| # These files must live in src/tools/perf/core/shard_maps |
| parser.add_argument('--test-shard-map-filename', type=str, required=False) |
| parser.add_argument('--run-ref-build', |
| help='Run test on reference browser', action='store_true') |
| parser.add_argument('--passthrough-arg', |
| help='Arguments to pass directly through to the test ' |
| 'executable.', action='append', |
| dest='passthrough_args', |
| default=[]) |
| parser.add_argument('--use-dynamic-shards', |
| help='If set, use dynamic shardmap instead of the file.', |
| action='store_true', |
| required=False |
| ) |
| parser.add_argument('--dynamic-shardmap', |
| help='The dynamically generated shardmap string used to ' |
| 'replace the static shardmap file.', |
| type=str, |
| required=False) |
| parser.add_argument('--ignore-benchmark-exit-code', |
| help='If set, return an exit code 0 even if there' |
| + ' are benchmark failures', |
| action='store_true', |
| required=False |
| ) |
| parser.add_argument('--results-label', |
| help='If set for a non-telemetry test, adds label to' |
| + ' the result histograms.', |
| type=str, |
| required=False |
| ) |
| options, leftover_args = parser.parse_known_args(args) |
| options.passthrough_args.extend(leftover_args) |
| return options |
| |
| |
| def main(sys_args): |
| args = sys_args[1:] # Skip program name. |
| options = parse_arguments(args) |
| isolated_out_dir = os.path.dirname(options.isolated_script_test_output) |
| overall_return_code = 0 |
| # This is a list of test results files to be merged into a standard |
| # output.json file for use by infrastructure including FindIt. |
| # This list should not contain reference build runs |
| # since we do not monitor those. Also, merging test reference build results |
| # with standard build results may not work properly. |
| test_results_files = [] |
| |
| print('Running a series of performance test subprocesses. Logs, performance\n' |
| 'results, and test results JSON will be saved in a subfolder of the\n' |
| 'isolated output directory. Inside the hash marks in the following\n' |
| 'lines is the name of the subfolder to find results in.\n') |
| |
| if options.non_telemetry: |
| benchmark_name = options.gtest_benchmark_name |
| passthrough_args = options.passthrough_args |
| # crbug/1146949#c15 |
| # In the case that pinpoint passes all arguments to swarming through http |
| # request, the passthrough_args are converted into a comma-separated string. |
| if passthrough_args and isinstance(passthrough_args, six.text_type): |
| passthrough_args = passthrough_args.split(',') |
| # With --non-telemetry, the gtest executable file path will be passed in as |
| # options.executable, which is different from running on shard map. Thus, |
| # we don't override executable as we do in running on shard map. |
| command_generator = GtestCommandGenerator( |
| options, additional_flags=passthrough_args, ignore_shard_env_vars=True) |
| # Fallback to use the name of the executable if flag isn't set. |
| # TODO(crbug.com/870899): remove fallback logic and raise parser error if |
| # --non-telemetry is set but --gtest-benchmark-name is not set once pinpoint |
| # is converted to always pass --gtest-benchmark-name flag. |
| if not benchmark_name: |
| benchmark_name = options.executable |
| output_paths = OutputFilePaths(isolated_out_dir, benchmark_name).SetUp() |
| print('\n### {folder} ###'.format(folder=benchmark_name)) |
| overall_return_code = execute_gtest_perf_test( |
| command_generator, output_paths, options.xvfb, |
| results_label=options.results_label) |
| test_results_files.append(output_paths.test_results) |
| else: |
| if options.use_dynamic_shards: |
| shard_map_str = options.dynamic_shardmap |
| shard_map = json.loads(shard_map_str, object_pairs_hook=OrderedDict) |
| shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY, |
| options.test_shard_map_filename) |
| with open(shard_map_path, 'w') as f: |
| json.dump(shard_map, f, indent=4, separators=(',', ': ')) |
| shutil.copyfile( |
| shard_map_path, |
| os.path.join(isolated_out_dir, 'benchmarks_shard_map.json')) |
| overall_return_code = _run_benchmarks_on_shardmap( |
| shard_map, options, isolated_out_dir, test_results_files |
| ) |
| # If the user has supplied a list of benchmark names, execute those instead |
| # of using the shard map. |
| elif options.benchmarks: |
| benchmarks = options.benchmarks.split(',') |
| for benchmark in benchmarks: |
| output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() |
| command_generator = TelemetryCommandGenerator( |
| benchmark, options) |
| print('\n### {folder} ###'.format(folder=benchmark)) |
| return_code = execute_telemetry_benchmark( |
| command_generator, output_paths, options.xvfb, |
| options.ignore_benchmark_exit_code) |
| overall_return_code = return_code or overall_return_code |
| test_results_files.append(output_paths.test_results) |
| if options.run_ref_build: |
| print('Not running reference build. --run-ref-build argument is only ' |
| 'supported for sharded benchmarks. It is simple to support ' |
| 'this for unsharded --benchmarks if needed.') |
| elif options.test_shard_map_filename: |
| # First determine what shard we are running on to know how to |
| # index into the bot map to get list of telemetry benchmarks to run. |
| shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY, |
| options.test_shard_map_filename) |
| # Copy sharding map file to isolated_out_dir so that the merge script |
| # can collect it later. |
| shutil.copyfile( |
| shard_map_path, |
| os.path.join(isolated_out_dir, 'benchmarks_shard_map.json')) |
| with open(shard_map_path) as f: |
| shard_map = json.load(f) |
| overall_return_code = _run_benchmarks_on_shardmap( |
| shard_map, options, isolated_out_dir, test_results_files |
| ) |
| else: |
| raise Exception('Telemetry tests must provide either a shard map or a ' |
| '--benchmarks list so that we know which stories to run.') |
| |
| test_results_list = [] |
| for test_results_file in test_results_files: |
| if os.path.exists(test_results_file): |
| with open(test_results_file, 'r') as fh: |
| test_results_list.append(json.load(fh)) |
| merged_test_results = results_merger.merge_test_results(test_results_list) |
| with open(options.isolated_script_test_output, 'w') as f: |
| json.dump(merged_test_results, f) |
| |
| return overall_return_code |
| |
| def _run_benchmarks_on_shardmap( |
| shard_map, options, isolated_out_dir, test_results_files): |
| overall_return_code = 0 |
| shard_index = None |
| env = os.environ.copy() |
| if 'GTEST_SHARD_INDEX' in env: |
| shard_index = env['GTEST_SHARD_INDEX'] |
| # TODO(crbug.com/972844): shard environment variables are not specified |
| # for single-shard shard runs. |
| if not shard_index: |
| shard_map_has_multiple_shards = bool(shard_map.get('1', False)) |
| if not shard_map_has_multiple_shards: |
| shard_index = '0' |
| if not shard_index: |
| raise Exception( |
| 'Sharded Telemetry perf tests must either specify --benchmarks ' |
| 'list or have GTEST_SHARD_INDEX environment variable present.') |
| shard_configuration = shard_map[shard_index] |
| assert ('benchmarks' in shard_configuration or |
| 'executables' in shard_configuration), ( |
| 'Every shard must have benchmarks or executables associated ' |
| 'with it.') |
| if 'benchmarks' in shard_configuration: |
| benchmarks_and_configs = shard_configuration['benchmarks'] |
| for (benchmark, story_selection_config) in benchmarks_and_configs.items(): |
| # Need to run the benchmark on both latest browser and reference |
| # build. |
| output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp() |
| command_generator = TelemetryCommandGenerator( |
| benchmark, options, |
| story_selection_config=story_selection_config) |
| print('\n### {folder} ###'.format(folder=benchmark)) |
| return_code = execute_telemetry_benchmark( |
| command_generator, output_paths, options.xvfb, |
| options.ignore_benchmark_exit_code) |
| overall_return_code = return_code or overall_return_code |
| test_results_files.append(output_paths.test_results) |
| if options.run_ref_build: |
| reference_benchmark_foldername = benchmark + '.reference' |
| reference_output_paths = OutputFilePaths( |
| isolated_out_dir, reference_benchmark_foldername).SetUp() |
| reference_command_generator = TelemetryCommandGenerator( |
| benchmark, options, |
| story_selection_config=story_selection_config, |
| is_reference=True) |
| print('\n### {folder} ###'.format( |
| folder=reference_benchmark_foldername)) |
| # We intentionally ignore the return code and test results of the |
| # reference build. |
| execute_telemetry_benchmark( |
| reference_command_generator, reference_output_paths, |
| options.xvfb, options.ignore_benchmark_exit_code) |
| if 'executables' in shard_configuration: |
| names_and_configs = shard_configuration['executables'] |
| for (name, configuration) in names_and_configs.items(): |
| additional_flags = [] |
| if 'arguments' in configuration: |
| additional_flags = configuration['arguments'] |
| command_generator = GtestCommandGenerator( |
| options, override_executable=configuration['path'], |
| additional_flags=additional_flags, ignore_shard_env_vars=True) |
| output_paths = OutputFilePaths(isolated_out_dir, name).SetUp() |
| print('\n### {folder} ###'.format(folder=name)) |
| return_code = execute_gtest_perf_test( |
| command_generator, output_paths, options.xvfb) |
| overall_return_code = return_code or overall_return_code |
| test_results_files.append(output_paths.test_results) |
| |
| return overall_return_code |
| |
| # This is not really a "script test" so does not need to manually add |
| # any additional compile targets. |
| def main_compile_targets(args): |
| json.dump([], args.output) |
| |
| |
| if __name__ == '__main__': |
| # Conform minimally to the protocol defined by ScriptTest. |
| if 'compile_targets' in sys.argv: |
| funcs = { |
| 'run': None, |
| 'compile_targets': main_compile_targets, |
| } |
| sys.exit(common.run_script(sys.argv[1:], funcs)) |
| |
| sys.exit(main(sys.argv)) |
