tools/perf/run_and_compare_benchmarks

#!/usr/bin/env python
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runs given benchmark on two executables and determines if the results are
significantly different.

Run one of the benchmarks available in 'run_benchmark' on two different browser
executables and compare the results using a statistical hypothesis test to
determine whether or not differences in benchmark results are statistically
significant (i.e. whether or not there is an actual difference in performance).
"""

from __future__ import print_function
import os
import sys
import time

import argparse
import subprocess

from core import path_util
sys.path.insert(1, os.path.join(path_util.GetChromiumSrcDir(), 'third_party',
                                'catapult', 'experimental'))
from statistical_analysis import compare_benchmark_results


def CreateOutputDir(output_dir_name):
  """Check if specified output directory exists and create if necessary."""
  if not os.path.isdir(output_dir_name):
    os.makedirs(output_dir_name)


def RunBenchmark(benchmark_name, executable_path, output_dir, num_runs,
                 reset_results):
  """Runs a benchmark (using the run_benchmark command)."""
  run_benchmark_path = os.path.join(os.path.dirname(__file__), 'run_benchmark')

  command = ['{}'.format(run_benchmark_path),
             'run',
             '{}'.format(benchmark_name),
             '--browser-executable={}'.format(os.path.join(executable_path)),
             '--pageset-repeat={}'.format(num_runs),
             '--output-dir={}'.format(output_dir),
             '--output-format=chartjson',
             '--output-format=html']
  if reset_results:
    command = command + ['--reset-results']

  subprocess.check_call(command, shell=False)


def RenameBenchmarkResultsJson(output_dir_name, new_json_file_name):
  """Renames the JSON produced by the run_benchmark script.

  Renaming the JSON makes sure that it is not overwritten by the following
  benchmark result JSON.
  """
  current_name = os.path.join(output_dir_name, 'results-chart.json')
  new_name = os.path.join(output_dir_name, new_json_file_name)
  os.rename(current_name, new_name)


def CreateJsonPathArgs(output_dir_name, json_file_names):
  """Returns a list of the JSON path command line arguments for the
  compare_benchmark_results script.

  These two command line arguments are implicit and are not specified by the
  command line arguments of this script since the JSON files are created and
  processed by this script and are at no point handled by the user.
  """
  return [os.path.join(output_dir_name, json_file_name) for json_file_name in
          json_file_names]


def main():
  parser = argparse.ArgumentParser(description=
      'Runs the given benchmark on two different given executables and then '
      'compares the results to determine if they are significantly different '
      'from each other. Also takes any argument taken by '
      'compare_benchmark_results.py in '
      'catapult/experimental/statistical_analysis.')

  def CheckNumRuns(val):
    ret_val = int(val)
    if ret_val < 3:
      raise argparse.ArgumentTypeError('--num-runs has to be an int >= 3. '
                                       'Entered value: {}.'.format(val))

    return ret_val

  parser.add_argument(dest='benchmark_name', help='Name of the benchmark as '
                      'required by run_benchmark script. Run ./run_benchmark '
                      'list for a list of available benchmarks.')

  parser.add_argument(dest='executable_path', nargs=2,
                      help='Browser executable location.')

  parser.add_argument('--num-runs', dest='num_runs', default=30,
                      type=CheckNumRuns, metavar='NUM', help='Number of times '
                      'the benchmark is run for each executable. Has to be >= '
                      '2.')

  parser.add_argument('--output-dir', dest='output_dir', help='Output '
                      'directory where the resulting Chart JSONs (one for '
                      'each benchmark, raw data) and the output results.html '
                      'file (summarizing and visualizing results) will be '
                      'saved.')

  # forward_args will be passed to the compare_benchmark_results script.
  known_args, forward_args = parser.parse_known_args()

  if known_args.output_dir is None:
    output_dir_name = 'statistical_analysis_' + known_args.benchmark_name
  else:
    output_dir_name = known_args.output_dir

  CreateOutputDir(output_dir_name)
  new_json_file_names = ('results-1-chart.json', 'results-2-chart.json')
  reset_results_for_benchmark = (True, False)

  print('Run Benchmarks:\n')
  start_time = time.time()
  for executable_path, json_file_name, reset_results in zip(
      known_args.executable_path, new_json_file_names,
      reset_results_for_benchmark):
    RunBenchmark(known_args.benchmark_name, executable_path, output_dir_name,
                 known_args.num_runs, reset_results=reset_results)
    RenameBenchmarkResultsJson(output_dir_name, json_file_name)
  print('Running benchmarks took {} seconds.'.format(time.time() - start_time))

  forward_args = (CreateJsonPathArgs(output_dir_name, new_json_file_names) +
                  forward_args)
  compare_benchmark_results.main(forward_args)


if __name__ == '__main__':
  sys.exit(main())