experimental/benchmark_health_report/load_benchmark_data.py

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import datetime
# Need to have dateutil module installed locally.
# pylint:disable=import-error
import dateutil.parser
import dateutil.tz

from benchmark_health_report import dashboard_api
from benchmark_health_report import drive_api
import six

def LoadBenchmarkData(
    start_date, end_date, single_benchmark):
  """Pulls benchmark data from dashboard API and sheets, and summarizes."""
  today = _GetTodayUtc()
  num_days = (today - start_date).days + 1
  alerts = dashboard_api.GetAlerts(num_days, single_benchmark)
  benchmarks = {}
  bugs = {}
  owners = _GetOwners()
  builds = _GetBuilds()
  for alert in alerts:
    utc_timestamp = _GetUtcDatetimeFromIsoTimestamp(alert['timestamp'])
    if utc_timestamp > end_date or utc_timestamp < start_date:
      continue
    benchmark_name = alert['testsuite']
    if single_benchmark and benchmark_name != single_benchmark:
      continue
    benchmark = benchmarks.setdefault(
        benchmark_name, _GetDefaultBenchmark(benchmark_name, owners, builds))
    _AddAlertDataToBenchmark(benchmark, alert, bugs)
  for bug_id in bugs:
    bugs[bug_id] = dashboard_api.GetBug(bug_id)
  for _, benchmark in six.iteritems(benchmarks):
    for bug_id in benchmark['bugs']:
      bug = bugs[bug_id]
      _AddBugDataToBenchmark(benchmark, bug)
  return benchmarks, bugs

def _GetOwners():
  """Reads the owners spreadsheet and returns a map of benchmark:owner."""
  owners = {}
  owner_values = drive_api.ReadSpreadsheet(
      '1xaAo0_SU3iDfGdqDJZX_jRV0QtkufwHUKH3kQKF3YQs', 'A4:B')
  for row in owner_values:
    if len(row) < 2:
      continue
    owners[row[0]] = row[1]
  return owners

def _GetBuilds():
  """Reads the build failure spreadsheet and returns a map of benchmark:data.

  TODO(sullivan): replace with flakiness dashboard data when available.
  """
  build_values = drive_api.ReadSpreadsheet(
      '1FG217E4V_E7Wxn1_mlMaqRAPIQyebzcGIjv4SPxddzQ', 'A2:D')
  builds = {}
  for row in build_values:
    name = row[0]
    successful = float(row[1])
    failed = float(row[2])
    builds[name] = {
        'successful': successful,
        'failed': failed,
        'total': successful + failed,
        'percent_failed': round((failed / (successful + failed)) * 100)
    }
  return builds

def _GetDefaultBenchmark(benchmark_name, owners, builds):
  return {
      'name': benchmark_name,
      'total_alerts': 0,
      'untriaged_alerts': 0,
      'invalid_alerts': 0,
      'valid_alerts': 0,
      'total_bugs': 0,
      'open_bugs': 0,
      'closed_bugs': 0,
      'fixed_bugs': 0,
      'wontfix_bugs': 0,
      'duplicate_bugs': 0,
      'total_bisects': 0,
      'successful_bisects': 0,
      'no_repro_bisects': 0,
      'infra_failure_bisects': 0,
      'bugs': set([]),
      'alerts': [],
      'owner': owners.get(benchmark_name),
      'percent_failed_builds': builds.get(
          benchmark_name, {}).get('percent_failed'),
      'total_builds': builds.get(benchmark_name, {}).get('total'),
  }

def _AddAlertDataToBenchmark(benchmark, alert, bugs):
  benchmark['alerts'].append(alert)
  if not alert['improvement']:
    benchmark['total_alerts'] += 1
    bug_id = alert['bug_id']
    if bug_id is None:
      benchmark['untriaged_alerts'] += 1
    elif bug_id < 0:
      benchmark['invalid_alerts'] += 1
    else:
      benchmark['valid_alerts'] += 1
      benchmark['bugs'].add(bug_id)
      bugs[bug_id] = {}

def _AddBugDataToBenchmark(benchmark, bug):
  benchmark['total_bugs'] += 1
  if bug['state'] == 'open':
    benchmark['open_bugs'] += 1
  else:
    benchmark['closed_bugs'] += 1
    if bug['status'] == 'Fixed' or bug['status'] == 'Verified':
      benchmark['fixed_bugs'] += 1
    elif bug['status'] == 'WontFix' or bug['status'] == 'Archived':
      benchmark['wontfix_bugs'] += 1
    elif bug['status'] == 'Duplicate':
      # TODO(sullivan): report the duplicate bug status
      benchmark['duplicate_bugs'] += 1
  for bisect in bug['legacy_bisects']:
    benchmark['total_bisects'] += 1
    if bisect['status'] == 'success':
      benchmark['successful_bisects'] += 1
    elif bisect['status'] == 'no-repro':
      benchmark['no_repro_bisects'] += 1
    elif bisect['status'] == 'failed':
      benchmark['infra_failure_bisects'] += 1

def GetTimeseriesList(benchmark):
  return dashboard_api.GetTimeseriesList(benchmark)

def _GetUtcDatetimeFromIsoTimestamp(timestamp_str):
  if not timestamp_str.endswith('Z'):
    # Dashboard keeps all dates in UTC, and always writes ISO timestamps, but
    # it does not append the 'Z'. Since we want the datetime to be UTC and
    # timezone-aware, just append the 'Z'.
    timestamp_str += 'Z'
  return dateutil.parser.parse(timestamp_str)

def _GetTodayUtc():
  return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())

def GetTimeseries(name, start_date, end_date):
  num_days = (_GetTodayUtc() - start_date).days + 1
  full_timeseries = dashboard_api.GetTimeseries(name, num_days)['timeseries']
  filtered_timeseries = [full_timeseries[0]]
  for row in full_timeseries[1:]:
    row_datetime = _GetUtcDatetimeFromIsoTimestamp(row[2])
    if row_datetime < start_date or row_datetime > end_date:
      continue
    filtered_timeseries.append(row)
  return filtered_timeseries