mergeoldperf.py

#!/usr/bin/python
#
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""mergeoldperf.py - Merge the current and original perf data into the current
data.
"""

import os
import sys

# Prepend the buildbot pylibs directory to our import path.
sys.path.insert(0, os.path.join(os.path.dirname(__file__),
                                '../../buildbot/build/third_party'))

import errno
import optparse
import re
import simplejson
import subprocess


__version__ = '1.0'

USAGE = r"""%prog [dir1] ... [dirN]"""

ORIGINAL_PATH = '/home/chrome-bot/saved-by-chase/original-perf'
CURRENT_PATH = '/home/chrome-bot/www/perf'


def Backquote(cmd, cwd=None):
  """Like running `cmd` in a shell script."""
  return subprocess.Popen(cmd,
                          cwd=cwd,
                          stdout=subprocess.PIPE).communicate()[0].strip()


def ReadJson(filename, max_revision=None):
  """Read the JSON file into memory as a Python data structure.

  Returns a list of hashes contained in the specified filename.
  """
  try:
    file = open(filename, 'r')
  except IOError, e:
    print >> sys.stderr, ("I/O Error reading file %s(%s): %s" %
                         (filename, e.errno, e.strerror))
    raise e

  if not file:
    return None

  data = []
  contents = file.read()
  file.close()
  contentslist = contents.split("\n")
  for jsontext in contentslist:
    if jsontext is None or len(jsontext) == 0:
      continue

    try:
      json = simplejson.loads(jsontext, parse_float=str,
                              object_pairs_hook=simplejson.OrderedDict)
    except ValueError, e:
      print >> sys.stderr, ("Error parsing file %s: '%s'" %
                            (filename, jsontext))
      raise e

    # If max_revision is set and revision >= max_revision, skip this entry.
    if max_revision and int(json['rev']) >= int(max_revision):
      continue
    data.append(json)
  return data


def WriteJson(filename, data):
  """Write a list of hashes in |data| to the file specified in |filename|."""
  try:
    file = open(filename, 'w')
  except IOError, e:
    print >> sys.stderr, ("I/O Error writing file %s(%s): %s" %
                          (filename, e.errno, e.strerror))
  if file:
    contentslist = []
    for json in data:
      contentslist.append(simplejson.dumps(json))
    contents = "\n".join(contentslist)
    file.write(contents + "\n")
    file.close()
  return True


def ProcessJson(filename):
  """Reads, sorts, and writes each JSON data file."""
  original_filename = filename.replace(CURRENT_PATH, ORIGINAL_PATH)
  if not os.path.exists(original_filename):
    return

  tempfilename = filename + ".tmp"
  if os.path.exists(tempfilename):
    raise Exception("%s already exists" % tempfilename)
  rc = subprocess.Popen(['cp', filename, tempfilename]).wait()
  print tempfilename
  if rc != 0:
    raise Exception("could not copy %s to %s" % (filename, tempfilename))

  # Read and sort (descending order) current json data.
  current_data = ReadJson(filename)
  current_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev'])))

  # Get the current data smallest revision.
  smallest_current_revision = int(current_data[-1]['rev'])
  print 'before: smallest rev = %s, highest rev = %s' % (
      current_data[-1]['rev'], current_data[0]['rev'])

  # Read and sort (descending order) original json file.
  original_data = ReadJson(
      original_filename, max_revision=smallest_current_revision)
  original_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev'])))

  # Extend the current data with the original data.
  current_data.extend(original_data)

  # Sort the current data one last time.
  current_data.sort(lambda x, y: cmp(int(y['rev']), int(x['rev'])))
  print 'after: smallest rev = %s, highest rev = %s' % (
      current_data[-1]['rev'], current_data[0]['rev'])

  # Check one last time that the current file matches the tmp current file.
  rc = subprocess.Popen(['diff', filename, tempfilename]).wait()
  if rc != 0:
    subprocess.Popen(['rm', tempfilename]).wait()
    print '%s changed before we could write back to it, skipping' % filename
    return

  WriteJson(filename, current_data)
  subprocess.Popen(['rm', tempfilename]).wait()
  subprocess.Popen(['rm', original_filename]).wait()
  print filename


def GetFilelist(dir, match):
  """Finds all items in dir that match the given arg."""
  if not os.path.exists(dir):
    print "Directory %s does not exist." % dir
    return []
  find_cmd = ['find', dir, '-name', match]
  find_output = Backquote(find_cmd).strip()
  if len(find_output) == 0:
    return []
  return find_output.split("\n")


def Main(args):
  parser = optparse.OptionParser(usage=USAGE, version=__version__)
  options, args = parser.parse_args(args)

  # Get the given directories the user wants to work in.
  options.dir = []
  if len(args) > 1:
    options.dir.extend(args[1:len(args)])
  # If no directories are given, assume the current working directory.
  if len(options.dir) == 0:
    options.dir.append('.')

  if os.getcwd() != CURRENT_PATH:
    raise Exception('only run this from %s' % CURRENT_PATH)

  for dir in options.dir:
    dir = os.path.abspath(dir)
    original_dir = dir.replace(CURRENT_PATH, ORIGINAL_PATH)

    # Merge old perf data into current perf data.
    for filename in GetFilelist(dir=dir, match='*-summary.dat'):
      filename = os.path.abspath(filename)
      ProcessJson(filename)

    # Move old perf summary data into current perf directories.
    for original_filename in GetFilelist(dir=original_dir,
                                         match='*-summary.dat'):
      filename = original_filename.replace(ORIGINAL_PATH, CURRENT_PATH)
      if not os.path.exists(filename):
        print "%s -> %s" % (original_filename, filename)
        subprocess.Popen(['mv', original_filename, filename]).wait()

    # Move old perf revision data into current perf directories.
    for original_filename in GetFilelist(dir=original_dir, match='*_t*.dat'):
      filename = original_filename.replace(ORIGINAL_PATH, CURRENT_PATH)
      if os.path.exists(filename):
        print 'removing %s (exists)' % original_filename
        subprocess.Popen(['rm', original_filename]).wait()
        continue
      if not os.path.exists(os.path.dirname(filename)):
        print 'skipping %s (parent does not exist)' % filename
        continue
      print '%s -> %s' % (original_filename, filename)
      subprocess.Popen(['mv', original_filename, filename]).wait()

  return 0


if __name__ == '__main__':
  sys.exit(Main(sys.argv))