| #!/usr/bin/python |
| # |
| # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """mergeoldperf.py - Merge the current and original perf data into the current |
| data. |
| """ |
| |
| import os |
| import sys |
| |
| # Prepend the buildbot pylibs directory to our import path. |
| sys.path.insert(0, os.path.join(os.path.dirname(__file__), |
| '../../buildbot/build/third_party')) |
| |
| import errno |
| import optparse |
| import re |
| import simplejson |
| import subprocess |
| |
| |
| __version__ = '1.0' |
| |
| USAGE = r"""%prog [dir1] ... [dirN]""" |
| |
| ORIGINAL_PATH = '/home/chrome-bot/saved-by-chase/original-perf' |
| CURRENT_PATH = '/home/chrome-bot/www/perf' |
| |
| |
| def Backquote(cmd, cwd=None): |
| """Like running `cmd` in a shell script.""" |
| return subprocess.Popen(cmd, |
| cwd=cwd, |
| stdout=subprocess.PIPE).communicate()[0].strip() |
| |
| |
| def ReadJson(filename, max_revision=None): |
| """Read the JSON file into memory as a Python data structure. |
| |
| Returns a list of hashes contained in the specified filename. |
| """ |
| try: |
| file = open(filename, 'r') |
| except IOError, e: |
| print >> sys.stderr, ("I/O Error reading file %s(%s): %s" % |
| (filename, e.errno, e.strerror)) |
| raise e |
| |
| if not file: |
| return None |
| |
| data = [] |
| contents = file.read() |
| file.close() |
| contentslist = contents.split("\n") |
| for jsontext in contentslist: |
| if jsontext is None or len(jsontext) == 0: |
| continue |
| |
| try: |
| json = simplejson.loads(jsontext, parse_float=str, |
| object_pairs_hook=simplejson.OrderedDict) |
| except ValueError, e: |
| print >> sys.stderr, ("Error parsing file %s: '%s'" % |
| (filename, jsontext)) |
| raise e |
| |
| # If max_revision is set and revision >= max_revision, skip this entry. |
| if max_revision and int(json['rev']) >= int(max_revision): |
| continue |
| data.append(json) |
| return data |
| |
| |
| def WriteJson(filename, data): |
| """Write a list of hashes in |data| to the file specified in |filename|.""" |
| try: |
| file = open(filename, 'w') |
| except IOError, e: |
| print >> sys.stderr, ("I/O Error writing file %s(%s): %s" % |
| (filename, e.errno, e.strerror)) |
| if file: |
| contentslist = [] |
| for json in data: |
| contentslist.append(simplejson.dumps(json)) |
| contents = "\n".join(contentslist) |
| file.write(contents + "\n") |
| file.close() |
| return True |
| |
| |
| def ProcessJson(filename): |
| """Reads, sorts, and writes each JSON data file.""" |
| original_filename = filename.replace(CURRENT_PATH, ORIGINAL_PATH) |
| if not os.path.exists(original_filename): |
| return |
| |
| tempfilename = filename + ".tmp" |
| if os.path.exists(tempfilename): |
| raise Exception("%s already exists" % tempfilename) |
| rc = subprocess.Popen(['cp', filename, tempfilename]).wait() |
| print tempfilename |
| if rc != 0: |
| raise Exception("could not copy %s to %s" % (filename, tempfilename)) |
| |
| # Read and sort (descending order) current json data. |
| current_data = ReadJson(filename) |
| current_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev']))) |
| |
| # Get the current data smallest revision. |
| smallest_current_revision = int(current_data[-1]['rev']) |
| print 'before: smallest rev = %s, highest rev = %s' % ( |
| current_data[-1]['rev'], current_data[0]['rev']) |
| |
| # Read and sort (descending order) original json file. |
| original_data = ReadJson( |
| original_filename, max_revision=smallest_current_revision) |
| original_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev']))) |
| |
| # Extend the current data with the original data. |
| current_data.extend(original_data) |
| |
| # Sort the current data one last time. |
| current_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev']))) |
| print 'after: smallest rev = %s, highest rev = %s' % ( |
| current_data[-1]['rev'], current_data[0]['rev']) |
| |
| # Check one last time that the current file matches the tmp current file. |
| rc = subprocess.Popen(['diff', filename, tempfilename]).wait() |
| if rc != 0: |
| subprocess.Popen(['rm', tempfilename]).wait() |
| print '%s changed before we could write back to it, skipping' % filename |
| return |
| |
| WriteJson(filename, current_data) |
| subprocess.Popen(['rm', tempfilename]).wait() |
| subprocess.Popen(['rm', original_filename]).wait() |
| print filename |
| |
| |
| def GetFilelist(dir, match): |
| """Finds all items in dir that match the given arg.""" |
| if not os.path.exists(dir): |
| print "Directory %s does not exist." % dir |
| return [] |
| find_cmd = ['find', dir, '-name', match] |
| find_output = Backquote(find_cmd).strip() |
| if len(find_output) == 0: |
| return [] |
| return find_output.split("\n") |
| |
| |
| def Main(args): |
| parser = optparse.OptionParser(usage=USAGE, version=__version__) |
| options, args = parser.parse_args(args) |
| |
| # Get the given directories the user wants to work in. |
| options.dir = [] |
| if len(args) > 1: |
| options.dir.extend(args[1:len(args)]) |
| # If no directories are given, assume the current working directory. |
| if len(options.dir) == 0: |
| options.dir.append('.') |
| |
| if os.getcwd() != CURRENT_PATH: |
| raise Exception('only run this from %s' % CURRENT_PATH) |
| |
| for dir in options.dir: |
| dir = os.path.abspath(dir) |
| original_dir = dir.replace(CURRENT_PATH, ORIGINAL_PATH) |
| |
| # Merge old perf data into current perf data. |
| for filename in GetFilelist(dir=dir, match='*-summary.dat'): |
| filename = os.path.abspath(filename) |
| ProcessJson(filename) |
| |
| # Move old perf summary data into current perf directories. |
| for original_filename in GetFilelist(dir=original_dir, |
| match='*-summary.dat'): |
| filename = original_filename.replace(ORIGINAL_PATH, CURRENT_PATH) |
| if not os.path.exists(filename): |
| print "%s -> %s" % (original_filename, filename) |
| subprocess.Popen(['mv', original_filename, filename]).wait() |
| |
| # Move old perf revision data into current perf directories. |
| for original_filename in GetFilelist(dir=original_dir, match='*_t*.dat'): |
| filename = original_filename.replace(ORIGINAL_PATH, CURRENT_PATH) |
| if os.path.exists(filename): |
| print 'removing %s (exists)' % original_filename |
| subprocess.Popen(['rm', original_filename]).wait() |
| continue |
| if not os.path.exists(os.path.dirname(filename)): |
| print 'skipping %s (parent does not exist)' % filename |
| continue |
| print '%s -> %s' % (original_filename, filename) |
| subprocess.Popen(['mv', original_filename, filename]).wait() |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(Main(sys.argv)) |