| #!/usr/bin/env python |
| # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Dromaeo benchmark automation script. |
| |
| Script runs dromaeo tests in browsers specified by --browser switch and saves |
| results to a spreadsheet on docs.google.com. |
| |
| Prerequisites: |
| 1. Install Google Data APIs Python Client Library from |
| http://code.google.com/p/gdata-python-client. |
| 2. Checkout Dromaeo benchmark from |
| http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide |
| local path to it in --dromaeo_home switch. |
| 3. Create a spreadsheet at http://docs.google.com and specify its name in |
| --spreadsheet switch |
| |
| Benchmark results are presented in the following format: |
| browser | date time |
| test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error |
| test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error |
| ... |
| |
| Here mij is mean run/s in individual dromaeo test i during benchmark run j, |
| eij is error in individual dromaeo test i during benchmark run j. |
| |
| Example usage: |
| dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe" |
| -b "C:\Program Files (x86)\Safari\safari.exe" |
| -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1 |
| -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example@gmail.com |
| |
| """ |
| |
| import getpass |
| import json |
| import os |
| import re |
| import subprocess |
| import time |
| import urlparse |
| from optparse import OptionParser |
| from BaseHTTPServer import HTTPServer |
| import SimpleHTTPServer |
| import gdata.spreadsheet.service |
| |
| max_spreadsheet_columns = 20 |
| test_props = ['mean', 'error'] |
| |
| |
| def ParseArguments(): |
| parser = OptionParser() |
| parser.add_option("-b", "--browser", |
| action="append", dest="browsers", |
| help="list of browsers to test") |
| parser.add_option("-n", "--run_count", dest="run_count", type="int", |
| default=5, help="number of runs") |
| parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home", |
| help="directory with your dromaeo files") |
| parser.add_option("-p", "--port", dest="port", type="int", |
| default=8080, help="http server port") |
| parser.add_option("-f", "--filter", dest="filter", |
| default="dom", help="dromaeo suite filter") |
| parser.add_option("-e", "--email", dest="email", |
| help="your google docs account") |
| parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title", |
| default="dromaeo", |
| help="your google docs spreadsheet name") |
| |
| options = parser.parse_args()[0] |
| |
| if not options.dromaeo_home: |
| raise Exception('please specify dromaeo_home') |
| |
| return options |
| |
| |
| def KillProcessByName(process_name): |
| process = subprocess.Popen('wmic process get processid, executablepath', |
| stdout=subprocess.PIPE) |
| stdout = str(process.communicate()[0]) |
| match = re.search(re.escape(process_name) + '\s+(\d+)', stdout) |
| if match: |
| pid = match.group(1) |
| subprocess.call('taskkill /pid %s' % pid) |
| |
| |
| class SpreadsheetWriter(object): |
| "Utility class for storing benchmarking results in Google spreadsheets." |
| |
| def __init__(self, email, spreadsheet_title): |
| '''Login to google docs and search for spreadsheet''' |
| |
| self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token") |
| self.gd_client = gdata.spreadsheet.service.SpreadsheetsService() |
| |
| authenticated = False |
| if os.path.exists(self.token_file): |
| token = '' |
| try: |
| file = open(self.token_file, 'r') |
| token = file.read() |
| file.close() |
| self.gd_client.SetClientLoginToken(token) |
| self.gd_client.GetSpreadsheetsFeed() |
| authenticated = True |
| except (IOError, gdata.service.RequestError): |
| pass |
| if not authenticated: |
| self.gd_client.email = email |
| self.gd_client.password = getpass.getpass('Password for %s: ' % email) |
| self.gd_client.source = 'python robot for dromaeo' |
| self.gd_client.ProgrammaticLogin() |
| token = self.gd_client.GetClientLoginToken() |
| try: |
| file = open(self.token_file, 'w') |
| file.write(token) |
| file.close() |
| except (IOError): |
| pass |
| os.chmod(self.token_file, 0600) |
| |
| # Search for the spreadsheet with title = spreadsheet_title. |
| spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed() |
| for spreadsheet in spreadsheet_feed.entry: |
| if spreadsheet.title.text == spreadsheet_title: |
| self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1] |
| if not self.spreadsheet_key: |
| raise Exception('Spreadsheet %s not found' % spreadsheet_title) |
| |
| # Get the key of the first worksheet in spreadsheet. |
| worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key) |
| self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1] |
| |
| def _InsertRow(self, row): |
| row = dict([('c' + str(i), row[i]) for i in xrange(len(row))]) |
| self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key) |
| |
| def _InsertBlankRow(self): |
| self._InsertRow('-' * self.columns_count) |
| |
| def PrepareSpreadsheet(self, run_count): |
| """Update cells in worksheet topmost row with service information. |
| |
| Calculate column count corresponding to run_count and create worksheet |
| column titles [c0, c1, ...] in the topmost row to speed up spreadsheet |
| updates (it allows to insert a whole row with a single request) |
| """ |
| |
| # Calculate the number of columns we need to present all test results. |
| self.columns_count = (run_count + 2) * len(test_props) |
| if self.columns_count > max_spreadsheet_columns: |
| # Google spreadsheet has just max_spreadsheet_columns columns. |
| max_run_count = max_spreadsheet_columns / len(test_props) - 2 |
| raise Exception('maximum run count is %i' % max_run_count) |
| # Create worksheet column titles [c0, c1, ..., cn]. |
| for i in xrange(self.columns_count): |
| self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key, |
| self.worksheet_key) |
| |
| def WriteColumnTitles(self, run_count): |
| "Create titles for test results (mean 1, mean 2, ..., average mean, ...)" |
| row = [] |
| for prop in test_props: |
| row.append('') |
| for i in xrange(run_count): |
| row.append('%s %i' % (prop, i + 1)) |
| row.append('average ' + prop) |
| self._InsertRow(row) |
| |
| def WriteBrowserBenchmarkTitle(self, browser_name): |
| "Create browser benchmark title (browser name, date time)" |
| self._InsertBlankRow() |
| self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')]) |
| |
| def WriteBrowserBenchmarkResults(self, test_name, test_data): |
| "Insert a row with single test results" |
| row = [] |
| for prop in test_props: |
| if not row: |
| row.append(test_name) |
| else: |
| row.append('') |
| row.extend([str(x) for x in test_data[prop]]) |
| row.append(str(sum(test_data[prop]) / len(test_data[prop]))) |
| self._InsertRow(row) |
| |
| |
| class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): |
| |
| def do_POST(self): |
| self.send_response(200) |
| self.end_headers() |
| self.wfile.write("<HTML>POST OK.<BR><BR>"); |
| length = int(self.headers.getheader('content-length')) |
| parameters = urlparse.parse_qs(self.rfile.read(length)) |
| self.server.got_post = True |
| self.server.post_data = parameters['data'] |
| |
| |
| class BenchmarkResults(object): |
| "Storage class for dromaeo benchmark results" |
| |
| def __init__(self): |
| self.data = {} |
| |
| def ProcessBrowserPostData(self, data): |
| "Convert dromaeo test results in internal format" |
| tests = json.loads(data[0]) |
| for test in tests: |
| test_name = test['name'] |
| if test_name not in self.data: |
| # Test is encountered for the first time. |
| self.data[test_name] = dict([(prop, []) for prop in test_props]) |
| # Append current run results. |
| for prop in test_props: |
| value = -1 |
| if prop in test: value = test[prop] # workaround for Opera 10.5 |
| self.data[test_name][prop].append(value) |
| |
| |
| def main(): |
| options = ParseArguments() |
| |
| # Start sever with dromaeo. |
| os.chdir(options.dromaeo_home) |
| server = HTTPServer(('', options.port), DromaeoHandler) |
| |
| # Open and prepare spreadsheet on google docs. |
| spreadsheet_writer = SpreadsheetWriter(options.email, |
| options.spreadsheet_title) |
| spreadsheet_writer.PrepareSpreadsheet(options.run_count) |
| spreadsheet_writer.WriteColumnTitles(options.run_count) |
| |
| for browser in options.browsers: |
| browser_name = os.path.splitext(os.path.basename(browser))[0] |
| spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name) |
| benchmark_results = BenchmarkResults() |
| for run_number in xrange(options.run_count): |
| print '%s run %i' % (browser_name, run_number + 1) |
| # Run browser. |
| test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % ( |
| options.port, options.filter) |
| browser_process = subprocess.Popen('%s "%s"' % (browser, test_page)) |
| server.got_post = False |
| server.post_data = None |
| # Wait until POST request from browser. |
| while not server.got_post: |
| server.handle_request() |
| benchmark_results.ProcessBrowserPostData(server.post_data) |
| # Kill browser. |
| KillProcessByName(browser) |
| browser_process.wait() |
| |
| # Insert test results into spreadsheet. |
| for (test_name, test_data) in benchmark_results.data.iteritems(): |
| spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data) |
| |
| server.socket.close() |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
