| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Class for running instrumentation tests on a single device.""" |
| |
| import logging |
| import os |
| import re |
| import sys |
| import time |
| |
| from pylib import constants |
| from pylib import flag_changer |
| from pylib import valgrind_tools |
| from pylib.base import base_test_result |
| from pylib.base import base_test_runner |
| from pylib.device import device_errors |
| from pylib.instrumentation import json_perf_parser |
| from pylib.instrumentation import test_result |
| |
| sys.path.append(os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', |
| 'common')) |
| import perf_tests_results_helper # pylint: disable=F0401 |
| |
| |
| _PERF_TEST_ANNOTATION = 'PerfTest' |
| |
| |
| def _GetDataFilesForTestSuite(suite_basename): |
| """Returns a list of data files/dirs needed by the test suite. |
| |
| Args: |
| suite_basename: The test suite basename for which to return file paths. |
| |
| Returns: |
| A list of test file and directory paths. |
| """ |
| test_files = [] |
| if suite_basename in ['ChromeTest', 'ContentShellTest']: |
| test_files += [ |
| 'net/data/ssl/certificates/', |
| ] |
| return test_files |
| |
| |
| class TestRunner(base_test_runner.BaseTestRunner): |
| """Responsible for running a series of tests connected to a single device.""" |
| |
| _DEVICE_DATA_DIR = 'chrome/test/data' |
| _DEVICE_COVERAGE_DIR = 'chrome/test/coverage' |
| _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
| _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
| '/chrome-profile*') |
| _DEVICE_HAS_TEST_FILES = {} |
| |
| def __init__(self, test_options, device, shard_index, test_pkg, |
| additional_flags=None): |
| """Create a new TestRunner. |
| |
| Args: |
| test_options: An InstrumentationOptions object. |
| device: Attached android device. |
| shard_index: Shard index. |
| test_pkg: A TestPackage object. |
| additional_flags: A list of additional flags to add to the command line. |
| """ |
| super(TestRunner, self).__init__(device, test_options.tool, |
| test_options.push_deps, |
| test_options.cleanup_test_files) |
| self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
| |
| self.coverage_device_file = None |
| self.coverage_dir = test_options.coverage_dir |
| self.coverage_host_file = None |
| self.options = test_options |
| self.test_pkg = test_pkg |
| # Use the correct command line file for the package under test. |
| cmdline_file = [a.cmdline_file for a in constants.PACKAGE_INFO.itervalues() |
| if a.test_package == self.test_pkg.GetPackageName()] |
| assert len(cmdline_file) < 2, 'Multiple packages have the same test package' |
| if len(cmdline_file) and cmdline_file[0]: |
| self.flags = flag_changer.FlagChanger(self.device, cmdline_file[0]) |
| if additional_flags: |
| self.flags.AddFlags(additional_flags) |
| else: |
| self.flags = None |
| |
| #override |
| def InstallTestPackage(self): |
| self.test_pkg.Install(self.device) |
| |
| #override |
| def PushDataDeps(self): |
| # TODO(frankf): Implement a general approach for copying/installing |
| # once across test runners. |
| if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): |
| logging.warning('Already copied test files to device %s, skipping.', |
| str(self.device)) |
| return |
| |
| test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) |
| if test_data: |
| # Make sure SD card is ready. |
| self.device.WaitUntilFullyBooted(timeout=20) |
| for p in test_data: |
| self.device.PushChangedFiles( |
| os.path.join(constants.DIR_SOURCE_ROOT, p), |
| os.path.join(self.device.GetExternalStoragePath(), p)) |
| |
| # TODO(frankf): Specify test data in this file as opposed to passing |
| # as command-line. |
| for dest_host_pair in self.options.test_data: |
| dst_src = dest_host_pair.split(':', 1) |
| dst_layer = dst_src[0] |
| host_src = dst_src[1] |
| host_test_files_path = os.path.join(constants.DIR_SOURCE_ROOT, |
| host_src) |
| if os.path.exists(host_test_files_path): |
| self.device.PushChangedFiles( |
| host_test_files_path, |
| '%s/%s/%s' % ( |
| self.device.GetExternalStoragePath(), |
| TestRunner._DEVICE_DATA_DIR, |
| dst_layer)) |
| self.tool.CopyFiles() |
| TestRunner._DEVICE_HAS_TEST_FILES[str(self.device)] = True |
| |
| def _GetInstrumentationArgs(self): |
| ret = {} |
| if self.options.wait_for_debugger: |
| ret['debug'] = 'true' |
| if self.coverage_dir: |
| ret['coverage'] = 'true' |
| ret['coverageFile'] = self.coverage_device_file |
| |
| return ret |
| |
| def _TakeScreenshot(self, test): |
| """Takes a screenshot from the device.""" |
| screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % test) |
| logging.info('Taking screenshot named %s', screenshot_name) |
| self.device.TakeScreenshot(screenshot_name) |
| |
| def SetUp(self): |
| """Sets up the test harness and device before all tests are run.""" |
| super(TestRunner, self).SetUp() |
| if not self.device.HasRoot(): |
| logging.warning('Unable to enable java asserts for %s, non rooted device', |
| str(self.device)) |
| else: |
| if self.device.SetJavaAsserts(True): |
| # TODO(jbudorick) How to best do shell restart after the |
| # android_commands refactor? |
| self.device.RunShellCommand('stop') |
| self.device.RunShellCommand('start') |
| |
| # We give different default value to launch HTTP server based on shard index |
| # because it may have race condition when multiple processes are trying to |
| # launch lighttpd with same port at same time. |
| self.LaunchTestHttpServer( |
| os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) |
| if self.flags: |
| self.flags.AddFlags(['--disable-fre', '--enable-test-intents']) |
| if self.options.device_flags: |
| with open(self.options.device_flags) as device_flags_file: |
| stripped_flags = (l.strip() for l in device_flags_file) |
| self.flags.AddFlags([flag for flag in stripped_flags if flag]) |
| |
| def TearDown(self): |
| """Cleans up the test harness and saves outstanding data from test run.""" |
| if self.flags: |
| self.flags.Restore() |
| super(TestRunner, self).TearDown() |
| |
| def TestSetup(self, test): |
| """Sets up the test harness for running a particular test. |
| |
| Args: |
| test: The name of the test that will be run. |
| """ |
| self.SetupPerfMonitoringIfNeeded(test) |
| self._SetupIndividualTestTimeoutScale(test) |
| self.tool.SetupEnvironment() |
| |
| # Make sure the forwarder is still running. |
| self._RestartHttpServerForwarderIfNecessary() |
| |
| if self.coverage_dir: |
| coverage_basename = '%s.ec' % test |
| self.coverage_device_file = '%s/%s/%s' % ( |
| self.device.GetExternalStoragePath(), |
| TestRunner._DEVICE_COVERAGE_DIR, coverage_basename) |
| self.coverage_host_file = os.path.join( |
| self.coverage_dir, coverage_basename) |
| |
| def _IsPerfTest(self, test): |
| """Determines whether a test is a performance test. |
| |
| Args: |
| test: The name of the test to be checked. |
| |
| Returns: |
| Whether the test is annotated as a performance test. |
| """ |
| return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) |
| |
| def SetupPerfMonitoringIfNeeded(self, test): |
| """Sets up performance monitoring if the specified test requires it. |
| |
| Args: |
| test: The name of the test to be run. |
| """ |
| if not self._IsPerfTest(test): |
| return |
| self.device.old_interface.Adb().SendCommand( |
| 'shell rm ' + TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX) |
| self.device.old_interface.StartMonitoringLogcat() |
| |
| def TestTeardown(self, test, result): |
| """Cleans up the test harness after running a particular test. |
| |
| Depending on the options of this TestRunner this might handle performance |
| tracking. This method will only be called if the test passed. |
| |
| Args: |
| test: The name of the test that was just run. |
| result: result for this test. |
| """ |
| |
| self.tool.CleanUpEnvironment() |
| |
| # The logic below relies on the test passing. |
| if not result or not result.DidRunPass(): |
| return |
| |
| self.TearDownPerfMonitoring(test) |
| |
| if self.coverage_dir: |
| self.device.PullFile( |
| self.coverage_device_file, self.coverage_host_file) |
| self.device.RunShellCommand( |
| 'rm -f %s' % self.coverage_device_file) |
| |
| def TearDownPerfMonitoring(self, test): |
| """Cleans up performance monitoring if the specified test required it. |
| |
| Args: |
| test: The name of the test that was just run. |
| Raises: |
| Exception: if there's anything wrong with the perf data. |
| """ |
| if not self._IsPerfTest(test): |
| return |
| raw_test_name = test.split('#')[1] |
| |
| # Wait and grab annotation data so we can figure out which traces to parse |
| regex = self.device.old_interface.WaitForLogMatch( |
| re.compile('\*\*PERFANNOTATION\(' + raw_test_name + '\)\:(.*)'), None) |
| |
| # If the test is set to run on a specific device type only (IE: only |
| # tablet or phone) and it is being run on the wrong device, the test |
| # just quits and does not do anything. The java test harness will still |
| # print the appropriate annotation for us, but will add --NORUN-- for |
| # us so we know to ignore the results. |
| # The --NORUN-- tag is managed by MainActivityTestBase.java |
| if regex.group(1) != '--NORUN--': |
| |
| # Obtain the relevant perf data. The data is dumped to a |
| # JSON formatted file. |
| json_string = self.device.ReadFile( |
| '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt', |
| as_root=True) |
| |
| if json_string: |
| json_string = '\n'.join(json_string) |
| else: |
| raise Exception('Perf file does not exist or is empty') |
| |
| if self.options.save_perf_json: |
| json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name |
| with open(json_local_file, 'w') as f: |
| f.write(json_string) |
| logging.info('Saving Perf UI JSON from test ' + |
| test + ' to ' + json_local_file) |
| |
| raw_perf_data = regex.group(1).split(';') |
| |
| for raw_perf_set in raw_perf_data: |
| if raw_perf_set: |
| perf_set = raw_perf_set.split(',') |
| if len(perf_set) != 3: |
| raise Exception('Unexpected number of tokens in perf annotation ' |
| 'string: ' + raw_perf_set) |
| |
| # Process the performance data |
| result = json_perf_parser.GetAverageRunInfoFromJSONString(json_string, |
| perf_set[0]) |
| perf_tests_results_helper.PrintPerfResult(perf_set[1], perf_set[2], |
| [result['average']], |
| result['units']) |
| |
| def _SetupIndividualTestTimeoutScale(self, test): |
| timeout_scale = self._GetIndividualTestTimeoutScale(test) |
| valgrind_tools.SetChromeTimeoutScale(self.device, timeout_scale) |
| |
| def _GetIndividualTestTimeoutScale(self, test): |
| """Returns the timeout scale for the given |test|.""" |
| annotations = self.test_pkg.GetTestAnnotations(test) |
| timeout_scale = 1 |
| if 'TimeoutScale' in annotations: |
| for annotation in annotations: |
| scale_match = re.match('TimeoutScale:([0-9]+)', annotation) |
| if scale_match: |
| timeout_scale = int(scale_match.group(1)) |
| if self.options.wait_for_debugger: |
| timeout_scale *= 100 |
| return timeout_scale |
| |
| def _GetIndividualTestTimeoutSecs(self, test): |
| """Returns the timeout in seconds for the given |test|.""" |
| annotations = self.test_pkg.GetTestAnnotations(test) |
| if 'Manual' in annotations: |
| return 10 * 60 * 60 |
| if 'IntegrationTest' in annotations: |
| return 30 * 60 |
| if 'External' in annotations: |
| return 10 * 60 |
| if 'EnormousTest' in annotations: |
| return 10 * 60 |
| if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: |
| return 5 * 60 |
| if 'MediumTest' in annotations: |
| return 3 * 60 |
| if 'SmallTest' in annotations: |
| return 1 * 60 |
| |
| logging.warn(("Test size not found in annotations for test '{0}', using " + |
| "1 minute for timeout.").format(test)) |
| return 1 * 60 |
| |
| def _RunTest(self, test, timeout): |
| """Runs a single instrumentation test. |
| |
| Args: |
| test: Test class/method. |
| timeout: Timeout time in seconds. |
| |
| Returns: |
| The raw output of am instrument as a list of lines. |
| """ |
| # Build the 'am instrument' command |
| instrumentation_path = ( |
| '%s/%s' % (self.test_pkg.GetPackageName(), self.options.test_runner)) |
| |
| cmd = ['am', 'instrument', '-r'] |
| for k, v in self._GetInstrumentationArgs().iteritems(): |
| cmd.extend(['-e', k, "'%s'" % v]) |
| cmd.extend(['-e', 'class', "'%s'" % test]) |
| cmd.extend(['-w', instrumentation_path]) |
| return self.device.RunShellCommand(cmd, timeout=timeout, retries=0) |
| |
| @staticmethod |
| def _ParseAmInstrumentRawOutput(raw_output): |
| """Parses the output of an |am instrument -r| call. |
| |
| Args: |
| raw_output: the output of an |am instrument -r| call as a list of lines |
| Returns: |
| A 3-tuple containing: |
| - the instrumentation code as an integer |
| - the instrumentation result as a list of lines |
| - the instrumentation statuses received as a list of 2-tuples |
| containing: |
| - the status code as an integer |
| - the bundle dump as a dict mapping string keys to a list of |
| strings, one for each line. |
| """ |
| INSTR_STATUS = 'INSTRUMENTATION_STATUS: ' |
| INSTR_STATUS_CODE = 'INSTRUMENTATION_STATUS_CODE: ' |
| INSTR_RESULT = 'INSTRUMENTATION_RESULT: ' |
| INSTR_CODE = 'INSTRUMENTATION_CODE: ' |
| |
| last = None |
| instr_code = None |
| instr_result = [] |
| instr_statuses = [] |
| bundle = {} |
| for line in raw_output: |
| if line.startswith(INSTR_STATUS): |
| instr_var = line[len(INSTR_STATUS):] |
| if '=' in instr_var: |
| k, v = instr_var.split('=', 1) |
| bundle[k] = [v] |
| last = INSTR_STATUS |
| last_key = k |
| else: |
| logging.debug('Unknown "%s" line: %s' % (INSTR_STATUS, line)) |
| |
| elif line.startswith(INSTR_STATUS_CODE): |
| instr_status = line[len(INSTR_STATUS_CODE):] |
| instr_statuses.append((int(instr_status), bundle)) |
| bundle = {} |
| last = INSTR_STATUS_CODE |
| |
| elif line.startswith(INSTR_RESULT): |
| instr_result.append(line[len(INSTR_RESULT):]) |
| last = INSTR_RESULT |
| |
| elif line.startswith(INSTR_CODE): |
| instr_code = int(line[len(INSTR_CODE):]) |
| last = INSTR_CODE |
| |
| elif last == INSTR_STATUS: |
| bundle[last_key].append(line) |
| |
| elif last == INSTR_RESULT: |
| instr_result.append(line) |
| |
| return (instr_code, instr_result, instr_statuses) |
| |
| def _GenerateTestResult(self, test, instr_statuses, start_ms, duration_ms): |
| """Generate the result of |test| from |instr_statuses|. |
| |
| Args: |
| instr_statuses: A list of 2-tuples containing: |
| - the status code as an integer |
| - the bundle dump as a dict mapping string keys to string values |
| Note that this is the same as the third item in the 3-tuple returned by |
| |_ParseAmInstrumentRawOutput|. |
| start_ms: The start time of the test in milliseconds. |
| duration_ms: The duration of the test in milliseconds. |
| Returns: |
| An InstrumentationTestResult object. |
| """ |
| INSTR_STATUS_CODE_START = 1 |
| INSTR_STATUS_CODE_OK = 0 |
| INSTR_STATUS_CODE_ERROR = -1 |
| INSTR_STATUS_CODE_FAIL = -2 |
| |
| log = '' |
| result_type = base_test_result.ResultType.UNKNOWN |
| |
| for status_code, bundle in instr_statuses: |
| if status_code == INSTR_STATUS_CODE_START: |
| pass |
| elif status_code == INSTR_STATUS_CODE_OK: |
| bundle_test = '%s#%s' % ( |
| ''.join(bundle.get('class', [''])), |
| ''.join(bundle.get('test', ['']))) |
| skipped = ''.join(bundle.get('test_skipped', [''])) |
| |
| if (test == bundle_test and |
| result_type == base_test_result.ResultType.UNKNOWN): |
| result_type = base_test_result.ResultType.PASS |
| elif skipped.lower() in ('true', '1', 'yes'): |
| result_type = base_test_result.ResultType.SKIP |
| logging.info('Skipped ' + test) |
| else: |
| if status_code not in (INSTR_STATUS_CODE_ERROR, |
| INSTR_STATUS_CODE_FAIL): |
| logging.info('Unrecognized status code %d. Handling as an error.', |
| status_code) |
| result_type = base_test_result.ResultType.FAIL |
| if 'stack' in bundle: |
| log = '\n'.join(bundle['stack']) |
| # Dismiss any error dialogs. Limit the number in case we have an error |
| # loop or we are failing to dismiss. |
| for _ in xrange(10): |
| package = self.device.old_interface.DismissCrashDialogIfNeeded() |
| if not package: |
| break |
| # Assume test package convention of ".test" suffix |
| if package in self.test_pkg.GetPackageName(): |
| result_type = base_test_result.ResultType.CRASH |
| break |
| |
| return test_result.InstrumentationTestResult( |
| test, result_type, start_ms, duration_ms, log=log) |
| |
| #override |
| def RunTest(self, test): |
| results = base_test_result.TestRunResults() |
| timeout = (self._GetIndividualTestTimeoutSecs(test) * |
| self._GetIndividualTestTimeoutScale(test) * |
| self.tool.GetTimeoutScale()) |
| |
| start_ms = 0 |
| duration_ms = 0 |
| try: |
| self.TestSetup(test) |
| |
| time_ms = lambda: int(time.time() * 1000) |
| start_ms = time_ms() |
| raw_output = self._RunTest(test, timeout) |
| duration_ms = time_ms() - start_ms |
| |
| # Parse the test output |
| _, _, statuses = self._ParseAmInstrumentRawOutput(raw_output) |
| result = self._GenerateTestResult(test, statuses, start_ms, duration_ms) |
| results.AddResult(result) |
| except device_errors.CommandTimeoutError as e: |
| results.AddResult(test_result.InstrumentationTestResult( |
| test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms, |
| log=str(e) or 'No information')) |
| except device_errors.DeviceUnreachableError as e: |
| results.AddResult(test_result.InstrumentationTestResult( |
| test, base_test_result.ResultType.CRASH, start_ms, duration_ms, |
| log=str(e) or 'No information')) |
| self.TestTeardown(test, results) |
| return (results, None if results.DidRunPass() else test) |
