| #!/usr/bin/env python |
| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # This is intended to be a very trimmed down, single-file, hackable, and easy |
| # to understand version of Telemetry. It's able to run simple user stories on |
| # Android, grab traces, and extract metrics from them. May be useful to |
| # diagnose issues with Chrome, reproduce regressions or prototype new user |
| # stories. |
| # |
| # Known limitations: Does not use WPR, so it does need to hit the live network |
| # to load pages. |
| |
| import collections |
| import contextlib |
| import functools |
| import httplib |
| import json |
| import logging |
| import os |
| import pipes |
| import posixpath |
| import re |
| import socket |
| import subprocess |
| import tempfile |
| import time |
| import websocket # pylint: disable=import-error |
| from xml.etree import ElementTree as element_tree |
| |
| |
| KEYCODE_HOME = 3 |
| KEYCODE_BACK = 4 |
| KEYCODE_APP_SWITCH = 187 |
| |
| # Parse rectangle bounds given as: '[left,top][right,bottom]'. |
| RE_BOUNDS = re.compile( |
| r'\[(?P<left>\d+),(?P<top>\d+)\]\[(?P<right>\d+),(?P<bottom>\d+)\]') |
| |
| # TODO: Maybe replace with a true on-device temp file. |
| UI_DUMP_TEMP = '/data/local/tmp/tm_ui_dump.xml' |
| |
| |
| def RetryOn(exc_type=(), returns_falsy=False, retries=5): |
| """Decorator to retry a function in case of errors or falsy values. |
| |
| Implements exponential backoff between retries. |
| |
| Args: |
| exc_type: Type of exceptions to catch and retry on. May also pass a tuple |
| of exceptions to catch and retry on any of them. Defaults to catching no |
| exceptions at all. |
| returns_falsy: If True then the function will be retried until it stops |
| returning a "falsy" value (e.g. None, False, 0, [], etc.). If equal to |
| 'raise' and the function keeps returning falsy values after all retries, |
| then the decorator will raise a ValueError. |
| retries: Max number of retry attempts. After exhausting that number of |
| attempts the function will be called with no safeguards: any exceptions |
| will be raised and falsy values returned to the caller (except when |
| returns_falsy='raise'). |
| """ |
| def Decorator(f): |
| @functools.wraps(f) |
| def Wrapper(*args, **kwargs): |
| wait = 1 |
| this_retries = kwargs.pop('retries', retries) |
| for _ in xrange(this_retries): |
| retry_reason = None |
| try: |
| value = f(*args, **kwargs) |
| except exc_type as exc: |
| retry_reason = 'raised %s' % type(exc).__name__ |
| if retry_reason is None: |
| if returns_falsy and not value: |
| retry_reason = 'returned %r' % value |
| else: |
| return value # Success! |
| logging.info('%s %s, will retry in %d second%s ...', |
| f.__name__, retry_reason, wait, '' if wait == 1 else 's') |
| time.sleep(wait) |
| wait *= 2 |
| value = f(*args, **kwargs) # Last try to run with no safeguards. |
| if returns_falsy == 'raise' and not value: |
| raise ValueError('%s returned %r' % (f.__name__, value)) |
| return value |
| return Wrapper |
| return Decorator |
| |
| |
| class AdbCommandError(Exception): |
| pass |
| |
| |
| class AdbMini(object): |
| ADB_BIN = 'adb' |
| |
| @classmethod |
| def RunBaseCommand(cls, *args): |
| cmd = [cls.ADB_BIN] |
| cmd.extend(args) |
| logging.info('$ adb %s', ' '.join(pipes.quote(a) for a in args)) |
| return subprocess.check_output(cmd, stderr=subprocess.STDOUT) |
| |
| @classmethod |
| def GetDevices(cls): |
| for line in cls.RunBaseCommand('devices').splitlines()[1:]: |
| cols = line.split() |
| if cols and cols[-1] == 'device': |
| yield cls(cols[0]) |
| |
| def __init__(self, serial): |
| self.serial = serial |
| |
| def RunCommand(self, *args): |
| return type(self).RunBaseCommand('-s', self.serial, *args) |
| |
| def RunShellCommand(self, *args): |
| return self.RunCommand('shell', *args) |
| |
| def ListPath(self, path): |
| return [ |
| line.split(' ')[-1] |
| for line in self.RunCommand('ls', path).splitlines()] |
| |
| def WriteText(self, text, path): |
| self.RunShellCommand( |
| 'echo -n %s > %s' % (pipes.quote(text), pipes.quote(path))) |
| |
| def ListPackages(self, name_filter=None, only_enabled=False): |
| """Return a list of packages available on the device.""" |
| args = ['pm', 'list', 'packages'] |
| if only_enabled: |
| args.append('-e') |
| if name_filter: |
| args.append(name_filter) |
| lines = self.RunShellCommand(*args).splitlines() |
| prefix = 'package:' |
| return [line[len(prefix):] for line in lines if line.startswith(prefix)] |
| |
| def ProcessStatus(self): |
| """Return a defaultdict mapping of {process_name: list_of_pids}.""" |
| result = collections.defaultdict(list) |
| # TODO: May not work on earlier Android verions without -e support. |
| for line in self.RunShellCommand('ps', '-e').splitlines(): |
| row = line.split(None, 8) |
| try: |
| pid = int(row[1]) |
| process_name = row[-1] |
| except StandardError: |
| continue |
| result[process_name].append(pid) |
| return result |
| |
| @RetryOn(AdbCommandError) |
| def GetUiScreenDump(self): |
| """Return the root XML node with screen captured from the device.""" |
| self.RunShellCommand('rm', '-f', UI_DUMP_TEMP) |
| output = self.RunShellCommand('uiautomator', 'dump', UI_DUMP_TEMP).strip() |
| |
| if output.startswith('ERROR:'): |
| # uiautomator may fail if device is not in idle state, e.g. animations |
| # or video playing. Retry if that's the case. |
| raise AdbCommandError(output) |
| |
| with tempfile.NamedTemporaryFile(suffix='.xml') as f: |
| f.close() |
| self.RunCommand('pull', UI_DUMP_TEMP, f.name) |
| return element_tree.parse(f.name) |
| |
| def HasUiElement(self, attr_values): |
| """Check whether a UI element is visible on the screen.""" |
| root = self.GetUiScreenDump() |
| for node in root.iter(): |
| if all(node.get(k) == v for k, v in attr_values): |
| return node |
| return None |
| |
| @RetryOn(LookupError) |
| def FindUiElement(self, *args, **kwargs): |
| """Find a UI element on the screen, retrying if not yet visible.""" |
| node = self.HasUiElement(*args, **kwargs) |
| if node is None: |
| raise LookupError('Specified UI element not found') |
| return node |
| |
| def TapUiElement(self, *args, **kwargs): |
| """Tap on a UI element found on screen.""" |
| node = self.FindUiElement(*args, **kwargs) |
| m = RE_BOUNDS.match(node.get('bounds')) |
| left, top, right, bottom = (int(v) for v in m.groups()) |
| x, y = (left + right) / 2, (top + bottom) / 2 |
| self.RunShellCommand('input', 'tap', str(x), str(y)) |
| |
| |
| def _UserAction(f): |
| """Decorator to add repeat, and action_delay options to user action methods. |
| |
| Note: It's advisable for decorated methods to include a catch-all **kwargs, |
| even if just to check it's empty. |
| |
| This is a workaround for https://github.com/PyCQA/pylint/issues/258 in which |
| decorators confuse pylint and trigger spurious 'unexpected-keyword-arg' |
| on method calls that use the extra options provided by this decorator. |
| """ |
| @functools.wraps(f) |
| def Wrapper(self, *args, **kwargs): |
| repeat = kwargs.pop('repeat', 1) |
| action_delay = kwargs.pop('action_delay', None) |
| for _ in xrange(repeat): |
| f(self, *args, **kwargs) |
| self.Idle(action_delay) |
| return Wrapper |
| |
| |
| class AndroidActions(object): |
| APP_SWITCHER_CLEAR_ALL = [ |
| ('resource-id', 'com.android.systemui:id/button'), |
| ('text', 'CLEAR ALL')] |
| APP_SWITCHER_NO_RECENT = [ |
| ('package', 'com.android.systemui'), |
| ('text', 'No recent items')] |
| |
| def __init__(self, device, user_action_delay=1): |
| self.device = device |
| self.user_action_delay = user_action_delay |
| |
| def Idle(self, duration=None): |
| if duration is None: |
| duration = self.user_action_delay |
| if duration: |
| time.sleep(duration) |
| |
| @_UserAction |
| def GoHome(self, **kwargs): |
| assert not kwargs # See @_UserAction |
| self.device.RunShellCommand('input', 'keyevent', str(KEYCODE_HOME)) |
| |
| @_UserAction |
| def GoBack(self, **kwargs): |
| assert not kwargs # See @_UserAction |
| self.device.RunShellCommand('input', 'keyevent', str(KEYCODE_BACK)) |
| |
| @_UserAction |
| def GoAppSwitcher(self, **kwargs): |
| assert not kwargs # See @_UserAction |
| self.device.RunShellCommand('input', 'keyevent', str(KEYCODE_APP_SWITCH)) |
| |
| @_UserAction |
| def StartActivity( |
| self, data_uri, action='android.intent.action.VIEW', **kwargs): |
| assert not kwargs # See @_UserAction |
| self.device.RunShellCommand('am', 'start', '-a', action, '-d', data_uri) |
| |
| @_UserAction |
| def TapUiElement(self, attr_values, **kwargs): |
| self.device.TapUiElement(attr_values, **kwargs) |
| |
| def TapHomeScreenShortcut(self, description, **kwargs): |
| self.TapUiElement([ |
| ('package', 'com.android.launcher3'), |
| ('class', 'android.widget.TextView'), |
| ('content-desc', description) |
| ], **kwargs) |
| |
| def TapAppSwitcherTitle(self, text, **kwargs): |
| self.TapUiElement([ |
| ('resource-id', 'com.android.systemui:id/title'), |
| ('text', text) |
| ], **kwargs) |
| |
| def TapAppSwitcherClearAll(self, **kwargs): |
| self.TapUiElement(self.APP_SWITCHER_CLEAR_ALL, **kwargs) |
| |
| @_UserAction |
| def SwipeUp(self, **kwargs): |
| assert not kwargs # See @_UserAction |
| # Hardcoded values for 480x854 screen size; should work reasonably on |
| # other screen sizes. |
| # Command args: swipe <x1> <y1> <x2> <y2> [duration(ms)] |
| self.device.RunShellCommand( |
| 'input', 'swipe', '240', '568', '240', '284', '400') |
| |
| @_UserAction |
| def SwipeDown(self, **kwargs): |
| assert not kwargs # See @_UserAction |
| # Hardcoded values for 480x854 screen size; should work reasonably on |
| # other screen sizes. |
| # Command args: swipe <x1> <y1> <x2> <y2> [duration(ms)] |
| self.device.RunShellCommand( |
| 'input', 'swipe', '240', '284', '240', '568', '400') |
| |
| def ClearRecentApps(self): |
| self.GoAppSwitcher() |
| if self.device.HasUiElement(self.APP_SWITCHER_NO_RECENT): |
| self.GoHome() |
| else: |
| # Sometimes we need to swipe down several times until the "Clear All" |
| # button becomes visible. |
| for _ in xrange(5): |
| try: |
| self.TapAppSwitcherClearAll(retries=0) # If not found raise error. |
| return # Success! |
| except LookupError: |
| self.SwipeDown() # Swipe down a bit more. |
| self.TapAppSwitcherClearAll() # Last try! If not found raises error. |
| |
| |
| class JavaScriptError(Exception): |
| pass |
| |
| |
| class DevToolsWebSocket(object): |
| def __init__(self, url): |
| self._url = url |
| self._socket = None |
| self._cmdid = 0 |
| |
| def __enter__(self): |
| self.Open() |
| return self |
| |
| def __exit__(self, *args, **kwargs): |
| self.Close() |
| |
| @RetryOn(socket.error) |
| def Open(self): |
| assert self._socket is None |
| self._socket = websocket.create_connection(self._url) |
| |
| def Close(self): |
| if self._socket is not None: |
| self._socket.close() |
| self._socket = None |
| |
| def Send(self, method, **kwargs): |
| logging.info( |
| '%s: %s(%s)', self._url, method, |
| ', '.join('%s=%r' % (k, v) for k, v in sorted(kwargs.iteritems()))) |
| self._cmdid += 1 |
| self._socket.send(json.dumps( |
| {'id': self._cmdid, 'method': method, 'params': kwargs})) |
| resp = self.Recv() |
| assert resp['id'] == self._cmdid |
| return resp.get('result') |
| |
| def Recv(self): |
| return json.loads(self._socket.recv()) |
| |
| def EvaluateJavaScript(self, expression): |
| resp = self.Send( |
| 'Runtime.evaluate', expression=expression, returnByValue=True) |
| if 'exceptionDetails' in resp: |
| raise JavaScriptError(resp['result']['description']) |
| return resp['result'].get('value') |
| |
| @RetryOn(returns_falsy='raise') |
| def WaitForJavaScriptCondition(self, *args, **kwargs): |
| return self.EvaluateJavaScript(*args, **kwargs) |
| |
| def RequestMemoryDump(self): |
| resp = self.Send('Tracing.requestMemoryDump') |
| assert resp['success'] |
| |
| def CollectTrace(self, trace_file): |
| """Stop tracing and collect the trace.""" |
| with open(trace_file, 'wb') as f: |
| # Call to Tracing.start is needed to update the transfer mode. |
| self.Send('Tracing.start', transferMode='ReturnAsStream', traceConfig={}) |
| self.Send('Tracing.end') |
| resp = self.Recv() |
| assert resp['method'] == 'Tracing.tracingComplete' |
| stream_handle = resp['params']['stream'] |
| try: |
| resp = {'eof': False} |
| num_bytes = 0 |
| while not resp['eof']: |
| resp = self.Send('IO.read', handle=stream_handle) |
| data = resp['data'].encode('utf-8') |
| f.write(data) |
| num_bytes += len(data) |
| logging.info( |
| 'Collected trace of %.1f MiB', float(num_bytes) / (1024 * 1024)) |
| finally: |
| self.Send('IO.close', handle=stream_handle) |
| |
| |
| class AndroidApp(object): |
| # Override this value with path to directory where APKs to install are found. |
| APKS_DIR = NotImplemented |
| |
| PACKAGE_NAME = NotImplemented |
| APK_FILENAME = None |
| |
| def __init__(self, device): |
| self.device = device |
| |
| def ForceStop(self): |
| self.device.RunShellCommand('am', 'force-stop', self.PACKAGE_NAME) |
| |
| def Install(self): |
| assert self.APK_FILENAME is not None, 'No APK to install available' |
| apk_path = os.path.join(self.APKS_DIR, self.APK_FILENAME) |
| logging.warning('Installing %s from %s', self.PACKAGE_NAME, apk_path) |
| assert os.path.isfile(apk_path), 'File not found: %s' % apk_path |
| self.device.RunCommand('install', '-r', '-d', apk_path) |
| |
| def Uninstall(self): |
| logging.warning('Uninstalling %s', self.PACKAGE_NAME) |
| self.device.RunCommand('uninstall', self.PACKAGE_NAME) |
| |
| |
| class ChromiumApp(AndroidApp): |
| PACKAGE_NAME = 'org.chromium.chrome' |
| APK_FILENAME = 'ChromePublic.apk' |
| COMMAND_LINE_FILE = '/data/local/tmp/chrome-command-line' |
| TRACE_CONFIG_FILE = '/data/local/chrome-trace-config.json' |
| |
| def __init__(self, *args, **kwargs): |
| super(ChromiumApp, self).__init__(*args, **kwargs) |
| self._devtools_local_port = None |
| self._browser_flags = None |
| self._trace_config = None |
| self.startup_time = None |
| |
| def RemoveProfile(self): |
| # TODO: Path to profile may need to be updated on newer Android versions. |
| profile_dir = posixpath.join('/data/data', self.PACKAGE_NAME) |
| filenames = self.device.ListPath(profile_dir) |
| args = ['rm', '-r'] |
| args.extend( |
| posixpath.join(profile_dir, f) |
| for f in filenames if f not in ['.', '..', 'lib']) |
| self.device.RunShellCommand(*args) |
| |
| @contextlib.contextmanager |
| def CommandLineFlags(self): |
| command_line = ' '.join(['_'] + self._browser_flags) |
| self.device.WriteText(command_line, self.COMMAND_LINE_FILE) |
| try: |
| yield |
| finally: |
| self.device.RunShellCommand('rm', '-f', self.COMMAND_LINE_FILE) |
| |
| def SetBrowserFlags(self, browser_flags): |
| self._browser_flags = browser_flags |
| |
| def SetTraceConfig(self, trace_config): |
| self._trace_config = trace_config |
| |
| def SetDevToolsLocalPort(self, port): |
| self._devtools_local_port = port |
| |
| def GetDevToolsLocalAddr(self, host='localhost'): |
| assert self._devtools_local_port is not None |
| return '%s:%d' % (host, self._devtools_local_port) |
| |
| def GetDevToolsRemoteAddr(self): |
| return 'localabstract:chrome_devtools_remote' |
| |
| @contextlib.contextmanager |
| def PortForwarding(self): |
| """Setup port forwarding to connect with DevTools on remote device.""" |
| local = self.GetDevToolsLocalAddr('tcp') |
| remote = self.GetDevToolsRemoteAddr() |
| self.device.RunCommand('forward', '--no-rebind', local, remote) |
| try: |
| yield |
| finally: |
| self.device.RunCommand('forward', '--remove', local) |
| |
| @contextlib.contextmanager |
| def StartupTracing(self): |
| self.device.WriteText( |
| json.dumps({'trace_config': self._trace_config}), |
| self.TRACE_CONFIG_FILE) |
| try: |
| yield |
| finally: |
| self.device.RunShellCommand('rm', '-f', self.TRACE_CONFIG_FILE) |
| |
| @contextlib.contextmanager |
| def Session(self): |
| """A context manager to guard the lifetime of a browser process. |
| |
| Ensures that command line flags and port forwarding are ready, the browser |
| is not alive before starting, it has a clear profile to begin with, and is |
| finally closed when done. |
| |
| It does not, however, launch the browser itself. This must be done by the |
| context managed code. |
| |
| To the extent possible, measurements from browsers launched within |
| different sessions are meant to be independent of each other. |
| """ |
| # Removing the profile breaks Chrome Shortcuts on the Home Screen. |
| # TODO: Figure out a way to automatically create the shortcuts before |
| # running the story. |
| # self.RemoveProfile() |
| with self.CommandLineFlags(): |
| with self.StartupTracing(): |
| # Ensure browser is closed after setting command line flags and |
| # trace config to ensure they are read on startup. |
| self.ForceStop() |
| with self.PortForwarding(): |
| try: |
| yield |
| finally: |
| self.ForceStop() |
| |
| def WaitForCurrentPageReady(self): |
| with self.CurrentPage() as page_dev: |
| page_dev.WaitForJavaScriptCondition('document.readyState == "complete"') |
| |
| def IterPages(self): |
| """Iterate over inspectable pages available through DevTools. |
| |
| Note: does not actually connect to the page until you "enter" it within |
| a managed context. |
| """ |
| for page in self.DevToolsRequest(): |
| if page['type'] == 'page': |
| yield self.DevToolsSocket(page['webSocketDebuggerUrl']) |
| |
| def CurrentPage(self): |
| """Get a DevToolsWebSocket to the current page on the browser.""" |
| return next(self.IterPages()) |
| |
| def CollectTrace(self, trace_file): |
| with self.DevToolsSocket() as browser_dev: |
| browser_dev.CollectTrace(trace_file) |
| |
| def DevToolsSocket(self, path='browser'): |
| # TODO(crbug.com/753842): Default browser path may need to be adjusted |
| # to include GUID. |
| if path.startswith('ws://'): |
| url = path |
| else: |
| url = ('ws://%s/devtools/' % self.GetDevToolsLocalAddr()) + path |
| return DevToolsWebSocket(url) |
| |
| @RetryOn(socket.error) |
| def DevToolsRequest(self, path=''): |
| url = '/json' |
| if path: |
| url = posixpath.join(url, path) |
| |
| conn = httplib.HTTPConnection(self.GetDevToolsLocalAddr()) |
| try: |
| conn.request('GET', url) |
| return json.load(conn.getresponse()) |
| finally: |
| conn.close() |
| |
| |
| class ChromeApp(ChromiumApp): |
| PACKAGE_NAME = 'com.google.android.apps.chrome' |
| APK_FILENAME = 'Chrome.apk' |
| |
| |
| class SystemChromeApp(ChromiumApp): |
| PACKAGE_NAME = 'com.android.chrome' |
| APK_FILENAME = None |
| |
| def Install(self): |
| # System Chrome app cannot be (un)installed, so we enable/disable instead. |
| logging.warning('Enabling %s', self.PACKAGE_NAME) |
| self.device.RunShellCommand('pm', 'enable', self.PACKAGE_NAME) |
| |
| def Uninstall(self): |
| # System Chrome app cannot be (un)installed, so we enable/disable instead. |
| logging.warning('Disabling %s', self.PACKAGE_NAME) |
| self.device.RunShellCommand('pm', 'disable', self.PACKAGE_NAME) |
| |
| |
| class UserStory(object): |
| def __init__(self, browser): |
| self.device = browser.device |
| self.browser = browser |
| self.actions = AndroidActions(self.device) |
| |
| def Run(self, trace_file): |
| with self.browser.Session(): |
| self.RunPrepareSteps() |
| try: |
| self.RunStorySteps() |
| self.browser.CollectTrace(trace_file) |
| except Exception as exc: |
| # Helps to pin point in the logs the moment where the story failed, |
| # before any of the finally blocks get to be executed. |
| logging.error('Aborting story due to %s.', type(exc).__name__) |
| raise |
| finally: |
| self.RunCleanupSteps() |
| |
| def RunPrepareSteps(self): |
| """Subclasses may override to perform actions before running the story.""" |
| pass |
| |
| def RunStorySteps(self): |
| """Subclasses should override this method to implement the story. |
| |
| The steps must: |
| - at some point cause the browser to be launched, and |
| - make sure the browser remains alive when done (even if backgrounded). |
| """ |
| raise NotImplementedError |
| |
| def RunCleanupSteps(self): |
| """Subclasses may override to perform actions after running the story. |
| |
| Note: This will be called even if an exception was raised during the |
| execution of RunStorySteps (but not for errors in RunPrepareSteps). |
| """ |
| pass |
| |
| |
| def ReadProcessMetrics(trace_file): |
| """Return a list of {"name": process_name, metric: value} dicts.""" |
| with open(trace_file) as f: |
| trace = json.load(f) |
| |
| processes = collections.defaultdict(dict) |
| for event in trace['traceEvents']: |
| if event['ph'] == 'v': |
| # Extract any metrics you may need from the trace. |
| value = event['args']['dumps']['allocators'][ |
| 'java_heap/allocated_objects']['attrs']['size'] |
| assert value['units'] == 'bytes' |
| processes[event['pid']]['java_heap'] = int(value['value'], 16) |
| elif event['ph'] == 'M' and event['name'] == 'process_name': |
| processes[event['pid']]['name'] = event['args']['name'] |
| |
| return processes.values() |
| |
| |
| def RunStories(browser, stories, repeat, output_dir): |
| for repeat_idx in range(1, repeat + 1): |
| for story_cls in stories: |
| trace_file = os.path.join( |
| output_dir, 'trace_%s_%d.json' % (story_cls.NAME, repeat_idx)) |
| print '[ RUN ]', story_cls.NAME |
| status = '[ OK ]' |
| start = time.time() |
| try: |
| story_cls(browser).Run(trace_file) |
| except Exception: # pylint: disable=broad-except |
| logging.exception('Exception raised while running story') |
| status = '[ FAILED ]' |
| finally: |
| elapsed = '(%.1f secs)' % (time.time() - start) |
| print status, story_cls.NAME, elapsed |
