blob: fb4751ffc6a9740d322e71337c8d93d205b648aa [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import json
import os
import sys
# This is the list of tests to run. It is a dictionary with the following
# fields:
#
# name (required): The name of the step, to show on the buildbot status page.
# path (required): The path to the executable which runs the tests.
# additional_args (optional): An array of optional arguments.
# uses_sandbox_env (optional): True if CHROME_DEVEL_SANDBOX must be in
# environment.
# disabled (optional): List of platforms the test is disabled on. May contain
# 'win', 'mac', 'linux', or 'android'.
# outputs_presentation_json (optional): If True, pass in --presentation-json
# argument to the test executable to allow it to update the buildbot status
# page. More details here:
# github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py
_DASHBOARD_TESTS = [
{
'name': 'Dashboard Dev Server Tests Stable',
'path': 'dashboard/bin/run_dev_server_tests',
'additional_args': [
'--no-install-hooks', '--no-use-local-chrome', '--channel=stable',
'--timeout-sec=120', '--timeout-retries=2'
],
'outputs_presentation_json': True,
'disabled': ['android', 'win', 'mac'],
},
{
'name': 'Dashboard Dev Server Tests Canary',
'path': 'dashboard/bin/run_dev_server_tests',
'additional_args': [
'--no-install-hooks', '--no-use-local-chrome', '--channel=canary',
'--timeout-sec=120', '--timeout-retries=2'
],
'outputs_presentation_json': True,
'disabled': ['android', 'win', 'mac'],
},
{
'name': 'Dashboard Python Tests',
'path': 'dashboard/bin/run_py_tests',
'additional_args': ['--no-install-hooks'],
'disabled': ['android', 'win', 'mac'],
},
]
_CATAPULT_TESTS = [
{
'name': 'Build Python Tests',
'path': 'catapult_build/bin/run_py_tests',
'disabled': ['android'],
},
{
'name': 'Common Tests',
'path': 'common/bin/run_tests',
},
{
'name': 'Dependency Manager Tests',
'path': 'dependency_manager/bin/run_tests',
},
{
'name': 'Devil Device Tests',
'path': 'devil/bin/run_py_devicetests',
'disabled': ['win', 'mac', 'linux']
},
{
'name': 'Devil Python Tests',
'path': 'devil/bin/run_py_tests',
'disabled': ['mac', 'win'],
},
{
'name': 'Native Heap Symbolizer Tests',
'path': 'tracing/bin/run_symbolizer_tests',
'disabled': ['android'],
},
{
'name': 'Py-vulcanize Tests',
'path': 'common/py_vulcanize/bin/run_py_tests',
'additional_args': ['--no-install-hooks'],
'disabled': ['android'],
},
{
'name': 'Systrace Tests',
'path': 'systrace/bin/run_tests',
},
{
'name': 'Snap-it Tests',
'path': 'telemetry/bin/run_snap_it_unittest',
'additional_args': ['--browser=reference',],
'uses_sandbox_env': True,
'disabled': ['android'],
},
{
'name': 'Telemetry Tests with Stable Browser (Desktop)',
'path': 'catapult_build/fetch_telemetry_deps_and_run_tests',
'additional_args': [
'--browser=reference',
'--start-xvfb',
'-v',
],
'uses_sandbox_env': True,
'disabled': ['android'],
},
{
'name': 'Telemetry Tests with Stable Browser (Android)',
'path': 'catapult_build/fetch_telemetry_deps_and_run_tests',
'additional_args': [
'--browser=reference',
'--device=android',
'--jobs=1',
'-v',
],
'uses_sandbox_env': True,
'disabled': ['win', 'mac', 'linux']
},
{
'name': 'Telemetry Integration Tests with Stable Browser',
'path': 'telemetry/bin/run_browser_tests',
'additional_args': [
'BrowserTest',
'--browser=reference',
'-v',
],
'uses_sandbox_env': True,
'disabled': ['android', 'linux'], # TODO(nedn): enable this on linux
},
{
'name': 'Tracing Dev Server Tests',
'path': 'tracing/bin/run_dev_server_tests',
'additional_args': [
'--no-install-hooks',
'--no-use-local-chrome',
'--channel=stable',
'--timeout-sec=900',
],
'outputs_presentation_json': True,
'disabled': ['android'],
},
{
'name': 'Tracing Dev Server Tests Canary',
'path': 'tracing/bin/run_dev_server_tests',
'additional_args': [
'--no-install-hooks',
'--no-use-local-chrome',
'--channel=canary',
'--timeout-sec=900',
],
'outputs_presentation_json': True,
'disabled': ['android'],
},
{
'name': 'Tracing D8 Tests',
'path': 'tracing/bin/run_vinn_tests',
'disabled': ['android'],
},
{
'name': 'Tracing Python Tests',
'path': 'tracing/bin/run_py_tests',
'additional_args': ['--no-install-hooks'],
'disabled': ['android'],
},
{
'name': 'Typ unittest',
'path': 'third_party/typ/run',
'additional_args': ['tests'],
'disabled': ['android', 'win'
], # TODO(crbug.com/851498): enable typ unittests on Win
},
{
'name': 'Vinn Tests',
'path': 'third_party/vinn/bin/run_tests',
'disabled': ['android'],
},
{
'name': 'NetLog Viewer Dev Server Tests',
'path': 'netlog_viewer/bin/run_dev_server_tests',
'additional_args': [
'--no-install-hooks',
'--no-use-local-chrome',
],
'disabled': ['android', 'win', 'mac', 'linux'],
},
]
_STALE_FILE_TYPES = ['.pyc', '.pseudo_lock']
def main(args=None):
"""Send list of test to run to recipes generator_script.
See documentation at:
github.com/luci/recipes-py/blob/master/recipe_modules/generator_script/api.py
"""
parser = argparse.ArgumentParser(description='Run catapult tests.')
parser.add_argument('--api-path-checkout', help='Path to catapult checkout')
parser.add_argument(
'--app-engine-sdk-pythonpath',
help='PYTHONPATH to include app engine SDK path')
parser.add_argument('--platform', help='Platform name (linux, mac, or win)')
parser.add_argument('--platform_arch', help='Platform arch (intel or arm)')
parser.add_argument('--output-json', help='Output for buildbot status page')
parser.add_argument(
'--run_android_tests', default=True, help='Run Android tests')
parser.add_argument(
'--dashboard_only',
default=False,
help='Run only the Dashboard and Pinpoint tests',
action='store_true')
parser.add_argument(
'--use_python3',
default=False,
help='Run Catapult Tests using vpython3',
action='store_true')
args = parser.parse_args(args)
dashboard_protos_path = os.path.join(args.api_path_checkout, 'dashboard',
'dashboard', 'proto')
dashboard_proto_files = [
os.path.join(dashboard_protos_path, p)
for p in ['sheriff.proto', 'sheriff_config.proto']
]
sheriff_proto_output_path = os.path.join(args.api_path_checkout, 'dashboard',
'dashboard', 'sheriff_config')
dashboard_proto_output_path = os.path.join(args.api_path_checkout,
'dashboard', 'dashboard')
tracing_protos_path = os.path.join(args.api_path_checkout, 'tracing',
'tracing', 'proto')
tracing_proto_output_path = tracing_protos_path
tracing_proto_files = [os.path.join(tracing_protos_path, 'histogram.proto')]
protoc_path = 'protoc'
# TODO(crbug.com/1271700): Remove this condition once protoc mac-arm build is
# released.
if args.platform == 'mac' and args.platform_arch == 'arm':
protoc_path = os.path.join(args.api_path_checkout, 'catapult_build', 'bin',
'mac-arm64', 'protoc')
steps = [
{
# Always remove stale files first. Not listed as a test above
# because it is a step and not a test, and must be first.
'name':
'Remove Stale files',
'cmd': [
'python',
os.path.join(args.api_path_checkout, 'catapult_build',
'remove_stale_files.py'),
args.api_path_checkout,
','.join(_STALE_FILE_TYPES),
]
},
# Since we might not have access to 'make', let's run the protobuf
# compiler directly. We want to run the proto compiler to generate the
# right data in the right places.
{
'name':
'Generate Sheriff Config protocol buffers',
'cmd': [
protoc_path,
'--proto_path',
dashboard_protos_path,
'--python_out',
sheriff_proto_output_path,
] + dashboard_proto_files,
},
{
'name':
'Generate Dashboard protocol buffers',
'cmd': [
protoc_path,
'--proto_path',
dashboard_protos_path,
'--python_out',
dashboard_proto_output_path,
] + dashboard_proto_files,
},
{
'name':
'Generate Tracing protocol buffers',
'cmd': [
protoc_path,
'--proto_path',
tracing_protos_path,
'--python_out',
tracing_proto_output_path,
] + tracing_proto_files,
},
]
if args.platform == 'android' and args.run_android_tests:
# On Android, we need to prepare the devices a bit before using them in
# tests. These steps are not listed as tests above because they aren't
# tests and because they must precede all tests.
steps.extend([
{
'name':
'Android: Recover Devices',
'cmd': [
'python',
os.path.join(args.api_path_checkout, 'devil', 'devil',
'android', 'tools', 'device_recovery.py')
],
},
{
'name':
'Android: Provision Devices',
'cmd': [
'python',
os.path.join(args.api_path_checkout, 'devil', 'devil',
'android', 'tools', 'provision_devices.py')
],
},
{
'name':
'Android: Device Status',
'cmd': [
'python',
os.path.join(args.api_path_checkout, 'devil', 'devil',
'android', 'tools', 'device_status.py')
],
},
])
tests = None
if args.dashboard_only:
tests = _DASHBOARD_TESTS
else:
tests = _CATAPULT_TESTS
for test in tests:
if args.platform == 'android' and not args.run_android_tests:
# Remove all the steps for the Android configuration if we're asked to not
# run the Android tests.
steps = []
break
if args.platform in test.get('disabled', []):
continue
# The test "Devil Python Tests" has two executables, run_py_tests and
# run_py3_tests. Those scripts define the vpython interpreter on shebang,
# and will quit when running on unexpected version. This script assumes one
# path for each test and thus we will conditionally replace the script name
# until python 2 is fully dropped.
# here,
test_path = test['path']
if args.use_python3 and test['name'] == 'Devil Python Tests':
test_path = 'devil/bin/run_py3_tests'
step = {'name': test['name'], 'env': {}}
if args.use_python3:
vpython_executable = "vpython3"
else:
vpython_executable = "vpython"
if sys.platform == 'win32':
vpython_executable += '.bat'
# Always add the appengine SDK path.
step['env']['PYTHONPATH'] = args.app_engine_sdk_pythonpath
step['cmd'] = [
vpython_executable,
os.path.join(args.api_path_checkout, test_path)
]
if step['name'] == 'Systrace Tests':
step['cmd'] += ['--device=' + args.platform]
if test.get('additional_args'):
step['cmd'] += test['additional_args']
if test.get('uses_sandbox_env'):
step['env']['CHROME_DEVEL_SANDBOX'] = '/opt/chromium/chrome_sandbox'
if test.get('outputs_presentation_json'):
step['outputs_presentation_json'] = True
# TODO(crbug/1221663):
# Before python 3 conversion is finished, the try jobs with use_python3 are
# experimental. We want to see all possible failure and thus we don't want
# to try job to quit before all tests are finished.
# This condition will be removed when the python 3 conversion is done.
if args.use_python3:
step['always_run'] = True
steps.append(step)
with open(args.output_json, 'w') as outfile:
json.dump(steps, outfile)
if __name__ == '__main__':
main(sys.argv[1:])