testing/scripts/run_rendering_benchmark_with_gated_performance.py

#!/usr/bin/env python
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runs telemetry benchmarks on representative story tag.

This script is a wrapper around run_performance_tests.py to capture the
values of performance metrics and compare them with the acceptable limits
in order to prevent regressions.

Arguments used for this script are the same as run_performance_tests.py.

The name and some functionalities of this script should be adjusted for
use with other benchmarks.
"""

import argparse
import csv
import json
import os
import sys
import time

import common
import run_performance_tests

BENCHMARK = 'rendering.desktop'
# AVG_ERROR_MARGIN determines how much more the value of frame times can be
# compared to the recorded value
AVG_ERROR_MARGIN = 2.0
# CI stands for comfidence intrevals. "ci_095"s recorded in the data is the
# reocrded range between upper and lower CIs. CI_ERROR_MARGIN is the maximum
# acceptable ratio of calculated ci_095 to the recorded ones.
CI_ERROR_MARGIN = 2.0

class ResultRecorder(object):
  def __init__(self):
    self.fails = 0
    self.tests = 0
    self.start_time = time.time()
    self.output = {}
    self.return_code = 0

  def setTests(self, output, testNum):
    self.output = output
    self.tests = testNum
    self.fails = 0
    if 'FAIL' in output['num_failures_by_type']:
      self.fails = output['num_failures_by_type']['FAIL']

  def addFailure(self, name):
    self.output['tests'][BENCHMARK][name]['actual'] = 'FAIL'
    self.output['tests'][BENCHMARK][name]['is_unexpected'] = True
    self.fails += 1

  def getOutput(self, return_code):
    self.output['seconds_since_epoch'] = time.time() - self.start_time
    self.output['num_failures_by_type']['PASS'] = self.tests - self.fails
    if self.fails > 0:
      self.output['num_failures_by_type']['FAIL'] = self.fails
    if return_code == 1:
      self.output["interrupted"] = True

    if self.fails == 0:
      print "[  PASSED  ] " + str(self.tests) + " tests."
    else:
      print "[  FAILED  ] " + str(self.fails) + "/" + str(self.tests)+ " tests."
      self.return_code = 1

    return (self.output, self.return_code)

  def setAllToFail(self):
    self.fails = self.tests

def main():
  overall_return_code = 0

  # Linux does not have it's own specific representatives
  # and uses the representatives chosen for winodws.
  if sys.platform == 'win32':
    platform = 'win'
    story_tag = 'representative_win_desktop'
  elif sys.platform == 'darwin':
    platform = 'mac'
    story_tag = 'representative_mac_desktop'
  else:
    return 1

  options = parse_arguments()
  args = sys.argv
  args.extend(['--story-tag-filter', story_tag])

  overall_return_code = run_performance_tests.main(args)
  result_recorder = ResultRecorder()

  # The values used as the upper limit are the 99th percentile of the
  # avg and ci_095 frame_times recorded by dashboard in the past 200 revisions.
  # If the value measured here would be higher than this value at least by
  # 2ms [AVG_ERROR_MARGIN], that would be considered a failure.
  # crbug.com/953895
  with open(
    os.path.join(os.path.dirname(__file__),
    'representative_perf_test_data',
    'representatives_frame_times_upper_limit.json')
  ) as bound_data:
    upper_limit_data = json.load(bound_data)

  out_dir_path = os.path.dirname(options.isolated_script_test_output)
  test_count = len(upper_limit_data[platform])

  output_path = os.path.join(out_dir_path, BENCHMARK, 'test_results.json')

  with open(output_path, 'r+') as resultsFile:
    initialOut = json.load(resultsFile)
    result_recorder.setTests(initialOut, test_count)

    results_path = os.path.join(out_dir_path, BENCHMARK, 'perf_results.csv')
    marked_stories = set()
    with open(results_path) as csv_file:
      reader = csv.DictReader(csv_file)
      for row in reader:
        # For now only frame_times is used for testing representatives'
        # performance.
        if row['name'] != 'frame_times':
          continue
        story_name = row['stories']
        if (story_name in marked_stories or story_name not in
          upper_limit_data[platform]):
          continue
        marked_stories.add(story_name)

        if row['avg'] == '' or row['count'] == 0:
          print "No values for " + story_name
          result_recorder.addFailure(story_name)
        elif (float(row['ci_095']) >
          upper_limit_data[platform][story_name]['ci_095'] * CI_ERROR_MARGIN):
          print "Noisy data on frame_times for " + story_name + ".\n"
          result_recorder.addFailure(story_name)
        elif (float(row['avg']) >
          upper_limit_data[platform][story_name]['avg'] + AVG_ERROR_MARGIN):
          print (story_name + ": average frame_times is higher than 99th " +
            "percentile of the past 200 recorded frame_times(" +
            row['avg'] + ")" + ".\n")
          result_recorder.addFailure(story_name)

    (
      finalOut,
      overall_return_code
    ) = result_recorder.getOutput(overall_return_code)

    # Clearing the result of run_benchmark and write the gated perf results
    resultsFile.seek(0)
    resultsFile.truncate(0)
    json.dump(finalOut, resultsFile, indent=4)

    with open(options.isolated_script_test_output, 'w') as outputFile:
      json.dump(finalOut, outputFile, indent=4)

  return overall_return_code

def parse_arguments():
  parser = argparse.ArgumentParser()
  parser.add_argument('executable', help='The name of the executable to run.')
  parser.add_argument(
      '--isolated-script-test-output', required=True)
  parser.add_argument(
      '--isolated-script-test-perf-output', required=False)
  return parser.parse_known_args()[0]

def main_compile_targets(args):
  json.dump([], args.output)

if __name__ == '__main__':
  # Conform minimally to the protocol defined by ScriptTest.
  if 'compile_targets' in sys.argv:
    funcs = {
      'run': None,
      'compile_targets': main_compile_targets,
    }
    sys.exit(common.run_script(sys.argv[1:], funcs))
  sys.exit(main())