| #!/usr/bin/env python |
| # Copyright 2019 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Script which gathers power measurement test results from bots. |
| |
| This is used to collect and store IPG based power measurments before they are |
| deleted. This provides data to better understand IPG. |
| """ |
| |
| import argparse |
| import ast |
| import csv |
| import json |
| import logging |
| import re |
| import sys |
| import urllib |
| import urllib2 |
| |
| |
| _TESTS = ['Basic', 'Video_720_MP4', 'Video_720_MP4_Fullscreen', |
| 'Video_720_MP4_Underlay', 'Video_720_MP4_Underlay_Fullscreen'] |
| _MEASUREMENTS = ['DRAM', 'GT', 'IA'] |
| |
| |
| def GetBuildData(method, request): |
| # Explorable via RPC explorer: |
| # https://cr-buildbucket.appspot.com/rpcexplorer/services/ |
| # buildbucket.v2.Builds/{GetBuild|SearchBuilds} |
| assert method in ['GetBuild', 'SearchBuilds'] |
| |
| # The Python docs are wrong. It's fine for this payload to be just |
| # a JSON string. |
| headers = { |
| 'content-type': 'application/json', |
| 'accept': 'application/json' |
| } |
| url = urllib2.Request( |
| 'https://cr-buildbucket.appspot.com/prpc/buildbucket.v2.Builds/' + method, |
| request, |
| headers) |
| conn = urllib2.urlopen(url) |
| result = conn.read() |
| conn.close() |
| # Result is a multi-line string the first line of which is |
| # deliberate garbage and the rest of which is a JSON payload. |
| return json.loads(''.join(result.splitlines()[1:])) |
| |
| |
| def GetJsonForBuildSteps(bot, build): |
| request = json.dumps({ 'builder': { 'project': 'chromium', |
| 'bucket': 'ci', |
| 'builder': bot }, |
| 'buildNumber': build, |
| 'fields': 'steps.*.name,steps.*.logs' }) |
| return GetBuildData('GetBuild', request) |
| |
| |
| def GetLatestGreenBuild(bot): |
| request = json.dumps({ 'predicate': { 'builder': { 'project': 'chromium', |
| 'bucket': 'ci', |
| 'builder': bot }, |
| 'status': 'SUCCESS' }, |
| 'fields': 'builds.*.number', |
| 'pageSize': 1 }) |
| builds_json = GetBuildData('SearchBuilds', request) |
| builds = builds_json['builds'] |
| assert len(builds) == 1 |
| return builds[0]['number'] |
| |
| |
| def GetJsonForLatestNBuilds(bot, build_count): |
| fields = [ |
| 'builds.*.number', |
| 'builds.*.steps.*.name', |
| 'builds.*.steps.*.logs', |
| ] |
| request = json.dumps({ 'predicate': { 'builder': { 'project': 'chromium', |
| 'bucket': 'ci', |
| 'builder': bot }}, |
| 'fields': ','.join(fields), |
| 'pageSize': build_count }) |
| builds_json = GetBuildData('SearchBuilds', request) |
| builds = builds_json['builds'] |
| if len(builds) != build_count: |
| logging.warn('Asked %d builds, got %d builds', build_count, len(builds)) |
| return builds |
| |
| |
| def FindStepLogURL(steps, step_name, log_name): |
| # The format of this JSON-encoded protobuf is defined here: |
| # https://chromium.googlesource.com/infra/luci/luci-go/+/master/ |
| # buildbucket/proto/step.proto |
| # It's easiest to just use the RPC explorer to fetch one and see |
| # what's desired to extract. |
| for step in steps: |
| if 'name' not in step or 'logs' not in step: |
| continue |
| if step['name'].startswith(step_name): |
| for log in step['logs']: |
| if log.get('name') == log_name: |
| return log.get('viewUrl') |
| return None |
| |
| |
| def ProcessStepStdout(stdout_url, entry): |
| url = urllib.unquote(stdout_url) |
| number = entry['number'] |
| logging.debug('[BUILD %d] stdout URL: %s' % (number, url)) |
| |
| # The following fails with Python 2.7.6, but succeeds with Python 2.7.14. |
| conn = urllib2.urlopen(stdout_url + '?format=raw') |
| data = conn.read() |
| conn.close() |
| |
| lines = data.splitlines() |
| pattern = re.compile('^\[(\d+)/(\d+)\]$') |
| results = None |
| bot_candidates = [] |
| for line in lines: |
| if results is not None: |
| my_results = results |
| results = None |
| # The line after results is [test/total_test] name passed {time}s |
| tokens = line.split() |
| if len(tokens) != 4: |
| logging.warn('Wrong format for test passed line: %s' % line) |
| continue |
| groups = pattern.match(tokens[0]).groups() |
| if len(groups) != 2: |
| logging.warn('Wrong format, expected [d+/d+], got %s' % tokens[0]) |
| continue |
| test_name = tokens[1] |
| if tokens[2] != 'passed': |
| logging.warn('Wrong format for test passed line: %s' % line) |
| my_results['name'] = test_name |
| entry['tests'].append(my_results) |
| elif line.startswith('Chrome Env: '): |
| chrome_env = ast.literal_eval(line[len('Chrome Env: '):]) |
| if 'COMPUTERNAME' in chrome_env: |
| bot_candidates.append(chrome_env['COMPUTERNAME']) |
| elif line.startswith('INFO:root:Chrome Env: '): |
| chrome_env = ast.literal_eval(line[len('INFO:root:Chrome Env: '):]) |
| if 'COMPUTERNAME' in chrome_env: |
| bot_candidates.append(chrome_env['COMPUTERNAME']) |
| elif line.startswith('Env: '): |
| chrome_env = ast.literal_eval(line[len('Env: '):]) |
| if 'COMPUTERNAME' in chrome_env: |
| bot_candidates.append(chrome_env['COMPUTERNAME']) |
| elif line.startswith(' COMPUTERNAME: '): |
| bot_candidates.append(line.strip()[len('COMPUTERNAME: '):]) |
| elif line.startswith('Results: '): |
| assert results is None |
| results = ast.literal_eval(line[len('Results: '):]) |
| for name in bot_candidates: |
| if name.startswith('BUILD'): |
| entry['bot'] = name |
| break |
| else: |
| logging.warn('[BUILD %d] Fail to locate the bot name' % number) |
| |
| |
| def CollectBuildData(build, data_entries): |
| if 'number' not in build: |
| logging.warn('Missing build number') |
| return False |
| if 'bot' not in build: |
| logging.warn('[BUILD %d] Missing bot name' % build['number']) |
| return False |
| if len(build['tests']) != len(_TESTS): |
| logging.warn('[BUILD %d] Measured test count should be %d, got %d' % ( |
| build['number'], len(_TESTS), len(build['tests']))) |
| return False |
| for test in build['tests']: |
| if not CollectTestData(test, data_entries): |
| return False |
| for ii in range(len(data_entries)): |
| data_entry = data_entries[ii] |
| data_entry['build'] = build['number'] |
| data_entry['bot'] = build['bot'] |
| data_entry['iteration'] = ii |
| return True |
| |
| |
| def CollectTestData(test, data_entries): |
| test_name = test['name'].split('.')[-1] |
| if test_name not in _TESTS: |
| logging.warn('Unexpected test name: %s' % test_name) |
| return False |
| for measure in _MEASUREMENTS: |
| measure_name = measure + ' Power_0' |
| if measure_name not in test: |
| logging.warn('Missing measurment: %s' % measure_name) |
| return False |
| results = test[measure_name] |
| assert results |
| if isinstance(results, list): |
| repeats = len(results) |
| else: |
| repeats = 1 |
| results = [results] |
| while len(data_entries) < repeats: |
| data_entries.append({}) |
| for ii in range(repeats): |
| data_entries[ii][test_name + '_' + measure] = results[ii] |
| return True |
| |
| |
| def SaveResultsAsCSV(results, output_filename): |
| builds = results['builds'] |
| csv_data = [] |
| for build in builds: |
| entries = [] |
| if not CollectBuildData(build, entries): |
| continue |
| csv_data.extend(entries) |
| if len(csv_data) > 0: |
| with open(output_filename, 'w') as csv_file: |
| labels = sorted(csv_data[0].keys()) |
| w = csv.DictWriter(csv_file, fieldnames=labels) |
| w.writeheader() |
| w.writerows(csv_data) |
| logging.debug('Data from %d tests saved to %s' % (len(csv_data), |
| output_filename)) |
| else: |
| logging.warn('No valid data saved to %s' % output_filename) |
| |
| |
| def main(): |
| rest_args = sys.argv[1:] |
| parser = argparse.ArgumentParser( |
| description='Gather JSON results from a run of a Swarming test.', |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| parser.add_argument('-v', '--verbose', action='store_true', default=False, |
| help='Enable verbose output') |
| parser.add_argument('--bot', default='Win10 FYI x64 Release (Intel HD 630)', |
| help='Which bot to examine.') |
| parser.add_argument('--last-build', type=int, |
| help='The last of a range of builds to fetch. If not ' |
| 'specified, use the latest build.') |
| parser.add_argument('--build-count', type=int, default=100, |
| help='How many builds to fetch. If not specified, ' |
| 'fetch 100 builds.') |
| parser.add_argument('--step', default='power_measurement_test', |
| help='Which step to fetch (treated as a prefix).') |
| parser.add_argument('--output-json', metavar='FILE', default='output.json', |
| help='Name of output json file. Default is output.json.') |
| parser.add_argument('--output-csv', metavar='FILE', default='output.csv', |
| help='Name of output csv file. Default is output.csv.') |
| |
| options = parser.parse_args(rest_args) |
| if options.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| |
| last_build = options.last_build |
| if last_build is None and options.build_count <= 100: |
| logging.debug('Start pulling latest %d builds' % options.build_count) |
| builds = GetJsonForLatestNBuilds(options.bot, options.build_count) |
| else: |
| if last_build is None: |
| last_build = GetLatestGreenBuild(options.bot) |
| first_build = last_build - options.build_count + 1 |
| builds = [] |
| for build_id in range(last_build, first_build - 1, -1): |
| logging.debug('Pull data for build %d' % build_id) |
| try: |
| build_json = GetJsonForBuildSteps(options.bot, build_id) |
| build_json['number'] = build_id |
| builds.append(build_json) |
| except urllib2.HTTPError: |
| logging.warn('HTTPError raised, failed to load data from build %d', |
| build_id) |
| |
| logging.debug('Start processing stdout data') |
| results = { 'builds': [] } |
| for ii in range(len(builds)): |
| build = builds[ii] |
| if 'number' not in build: |
| logging.warn('Missing number in build #%d' % ii) |
| continue |
| number = build['number'] |
| if 'steps' not in build: |
| logging.warn('[BUILD %d] Missing steps' % number) |
| continue |
| stdout_url = FindStepLogURL(build['steps'], options.step, 'stdout') |
| if not stdout_url: |
| logging.warn('[BUILD %d] Unable to find stdout from step %s*' % |
| (number, options.step)) |
| continue |
| results['builds'].append({ 'number': number, 'tests': [] }) |
| ProcessStepStdout(stdout_url, results['builds'][-1]) |
| |
| logging.debug('Saving output to %s', options.output_json) |
| with open(options.output_json, 'w') as f: |
| json.dump(results, f, sort_keys=True, indent=2, |
| separators=(',', ': ')) |
| |
| logging.debug('Saving output to %s', options.output_csv) |
| SaveResultsAsCSV(results, options.output_csv) |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |