blob: 8f23893be69fb5a69e22558cd23458c35749ab57 [file] [log] [blame]
# Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import print_function
import logging
import os
import sys
from telemetry.core import util
from telemetry.core import platform as platform_module
from telemetry import decorators
from telemetry.internal.browser import browser_finder
from telemetry.internal.browser import browser_finder_exceptions
from telemetry.internal.browser import browser_options
from telemetry.internal.platform import android_device
from telemetry.internal.util import binary_manager
from telemetry.internal.util import command_line
from telemetry.internal.util import ps_util
from telemetry.testing import browser_test_case
from telemetry.testing import options_for_unittests
from py_utils import cloud_storage
from py_utils import xvfb
import typ
class RunTestsCommand(command_line.OptparseCommand):
"""Run unit tests"""
usage = (
'\n\n ./run_tests [test_name_1 test_name_2 ...] [<options>]\n\n'
'You can get a list of potential test names by running\n\n'
' ./run_tests --list-only\n\n'
'For a test that looks like \n'
'"telemetry.web_perf.timeline_based_measurement_unittest.LegacyTim'
'elineBasedMetricsTests.testDuplicateRepeatableInteractions",\n'
'You can use any substring of it, i.e.\n\n'
' ./run_tests testDuplicateRepeatableInteractions\n')
xvfb_process = None
def __init__(self):
super(RunTestsCommand, self).__init__()
self.stream = sys.stdout
@classmethod
def CreateParser(cls):
options = browser_options.BrowserFinderOptions()
options.browser_type = 'any'
parser = options.CreateParser(cls.usage)
return parser
@classmethod
def AddCommandLineArgs(cls, parser, _):
parser.add_option('--start-xvfb', action='store_true',
default=False, help='Start Xvfb display if needed.')
parser.add_option('--disable-cloud-storage-io', action='store_true',
default=False, help=('Disable cloud storage IO when '
'tests are run in parallel.'))
parser.add_option('--no-browser', action='store_true', default=False,
help='Don\'t require an actual browser to run the tests.')
parser.add_option('-d', '--also-run-disabled-tests',
dest='run_disabled_tests',
action='store_true', default=False,
help='Ignore @Disabled and @Enabled restrictions.')
parser.add_option('--client-config', dest='client_configs',
action='append', default=[])
parser.add_option('--disable-logging-config', action='store_true',
default=False, help='Configure logging (default on)')
parser.add_option('-v', '--verbose', action='count', dest='verbosity',
help='Increase verbosity level (repeat as needed)')
typ.ArgumentParser.add_option_group(parser,
"Options for running the tests",
running=True,
skip=['-d', '-v', '--verbose'])
typ.ArgumentParser.add_option_group(parser,
"Options for reporting the results",
reporting=True)
@classmethod
def ProcessCommandLineArgs(cls, parser, args, _):
if args.verbosity >= 2:
logging.getLogger().setLevel(logging.DEBUG)
elif args.verbosity:
logging.getLogger().setLevel(logging.INFO)
else:
logging.getLogger().setLevel(logging.WARNING)
# We retry failures by default unless we're running a list of tests
# explicitly.
if not args.retry_limit and not args.positional_args:
args.retry_limit = 3
if args.test_filter and args.positional_args:
parser.error(
'Cannot specify test names in positional args and use'
'--test-filter flag at the same time.')
if args.no_browser:
return
if args.start_xvfb and xvfb.ShouldStartXvfb():
cls.xvfb_process = xvfb.StartXvfb()
# Work around Mesa issues on Linux. See
# https://github.com/catapult-project/catapult/issues/3074
args.browser_options.AppendExtraBrowserArgs('--disable-gpu')
try:
possible_browser = browser_finder.FindBrowser(args)
except browser_finder_exceptions.BrowserFinderException as ex:
parser.error(ex)
if not possible_browser:
parser.error('No browser found of type %s. Cannot run tests.\n'
'Re-run with --browser=list to see '
'available browser types.' % args.browser_type)
@classmethod
def main(cls, args=None, stream=None): # pylint: disable=arguments-differ
# We override the superclass so that we can hook in the 'stream' arg.
parser = cls.CreateParser()
cls.AddCommandLineArgs(parser, None)
options, positional_args = parser.parse_args(args)
options.positional_args = positional_args
try:
# Must initialize the DependencyManager before calling
# browser_finder.FindBrowser(args)
binary_manager.InitDependencyManager(options.client_configs)
cls.ProcessCommandLineArgs(parser, options, None)
obj = cls()
if stream is not None:
obj.stream = stream
return obj.Run(options)
finally:
if cls.xvfb_process:
cls.xvfb_process.kill()
def Run(self, args):
runner = typ.Runner()
if self.stream:
runner.host.stdout = self.stream
if args.no_browser:
possible_browser = None
platform = platform_module.GetHostPlatform()
else:
possible_browser = browser_finder.FindBrowser(args)
platform = possible_browser.platform
fetch_reference_chrome_binary = False
# Fetch all binaries needed by telemetry before we run the benchmark.
if possible_browser and possible_browser.browser_type == 'reference':
fetch_reference_chrome_binary = True
binary_manager.FetchBinaryDependencies(
platform, args.client_configs, fetch_reference_chrome_binary)
# Telemetry seems to overload the system if we run one test per core,
# so we scale things back a fair amount. Many of the telemetry tests
# are long-running, so there's a limit to how much parallelism we
# can effectively use for now anyway.
#
# It should be possible to handle multiple devices if we adjust the
# browser_finder code properly, but for now we only handle one on ChromeOS.
if platform.GetOSName() == 'chromeos':
runner.args.jobs = 1
elif platform.GetOSName() == 'android':
android_devs = android_device.FindAllAvailableDevices(args)
runner.args.jobs = len(android_devs)
if runner.args.jobs == 0:
raise RuntimeError("No Android device found")
print('Running tests with %d Android device(s).' % runner.args.jobs)
elif platform.GetOSVersionName() == 'xp':
# For an undiagnosed reason, XP falls over with more parallelism.
# See crbug.com/388256
runner.args.jobs = max(int(args.jobs) // 4, 1)
else:
runner.args.jobs = max(int(args.jobs) // 2, 1)
runner.args.expectations_files = args.expectations_files
runner.args.tags = args.tags
runner.args.skip = args.skip
runner.args.metadata = args.metadata
runner.args.passthrough = args.passthrough
runner.args.path = args.path
runner.args.retry_limit = args.retry_limit
runner.args.typ_max_failures = args.typ_max_failures
runner.args.test_results_server = args.test_results_server
runner.args.partial_match_filter = args.positional_args
runner.args.test_filter = args.test_filter
runner.args.test_type = args.test_type
runner.args.top_level_dirs = args.top_level_dirs
runner.args.write_full_results_to = args.write_full_results_to
runner.args.write_trace_to = args.write_trace_to
runner.args.all = args.run_disabled_tests
runner.args.repeat = args.repeat
runner.args.list_only = args.list_only
runner.args.shard_index = args.shard_index
runner.args.test_name_prefix = args.test_name_prefix
runner.args.total_shards = args.total_shards
runner.args.repository_absolute_path = args.repository_absolute_path
runner.args.retry_only_retry_on_failure_tests = (
args.retry_only_retry_on_failure_tests)
runner.args.path.append(util.GetUnittestDataDir())
runner.args.quiet = args.quiet
# Standard verbosity will only emit output on test failure. Higher verbosity
# levels spam the output with logging, making it very difficult to figure
# out what's going on when digging into test failures.
runner.args.timing = True
runner.args.verbose = 1
runner.classifier = GetClassifier(runner, possible_browser)
runner.context = args
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print("interrupted, exiting", file=sys.stderr)
ret = 130
return ret
def GetClassifier(typ_runner, possible_browser):
def ClassifyTestWithoutBrowser(test_set, test):
if typ_runner.matches_filter(test):
if typ_runner.should_skip(test):
test_set.add_test_to_skip(
test, 'skipped because matched --skip')
return
# TODO(telemetry-team): Make sure that all telemetry unittest that invokes
# actual browser are subclasses of browser_test_case.BrowserTestCase
# (crbug.com/537428)
if issubclass(test.__class__, browser_test_case.BrowserTestCase):
test_set.add_test_to_skip(
test, 'Skip the test because it requires a browser.')
else:
test_set.add_test_to_run_in_parallel(test)
def ClassifyTestWithBrowser(test_set, test):
if typ_runner.matches_filter(test):
if typ_runner.should_skip(test):
test_set.add_test_to_skip(
test, 'skipped because matched --skip')
return
assert hasattr(test, '_testMethodName')
method = getattr(
test, test._testMethodName) # pylint: disable=protected-access
should_skip, reason = decorators.ShouldSkip(method, possible_browser)
if should_skip and not typ_runner.args.all:
test_set.add_test_to_skip(test, reason)
elif decorators.ShouldBeIsolated(method, possible_browser):
test_set.add_test_to_run_isolated(test)
else:
test_set.add_test_to_run_in_parallel(test)
if possible_browser:
return ClassifyTestWithBrowser
else:
return ClassifyTestWithoutBrowser
def _SetUpProcess(child, context): # pylint: disable=unused-argument
ps_util.EnableListingStrayProcessesUponExitHook()
# Make sure that we don't invokes cloud storage I/Os when we run the tests in
# parallel.
# TODO(https://github.com/catapult-project/catapult/issues/2192): always do
# this once telemetry tests in Chromium is updated to prefetch files.
args = context
if args.disable_cloud_storage_io:
os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1'
if binary_manager.NeedsInit():
# Typ doesn't keep the DependencyManager initialization in the child
# processes.
binary_manager.InitDependencyManager(context.client_configs)
# We need to reset the handlers in case some other parts of telemetry already
# set it to make this work.
if not args.disable_logging_config:
logging.getLogger().handlers = []
logging.basicConfig(
level=logging.INFO,
format='(%(levelname)s) %(asctime)s pid=%(process)d'
' %(module)s.%(funcName)s:%(lineno)d'
' %(message)s')
if args.remote_platform_options.device == 'android':
android_devices = android_device.FindAllAvailableDevices(args)
if not android_devices:
raise RuntimeError("No Android device found")
android_devices.sort(key=lambda device: device.name)
args.remote_platform_options.device = (
android_devices[child.worker_num-1].guid)
options_for_unittests.Push(args)
def _TearDownProcess(child, context): # pylint: disable=unused-argument
options_for_unittests.Pop()
if __name__ == '__main__':
ret_code = RunTestsCommand.main()
sys.exit(ret_code)