tools/perf/benchmarks/benchmark_smoke_unittest.py

# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the first page of one benchmark for every module.

Only benchmarks that have a composable measurement are included.
Ideally this test would be comprehensive, however, running one page
of every benchmark would run impractically long.
"""

import os
import sys
import unittest

from chrome_telemetry_build import chromium_config

from core import results_processor
from core import testing

from telemetry import benchmark as benchmark_module
from telemetry import decorators
from telemetry.testing import progress_reporter

from py_utils import discover
from py_utils import tempfile_ext

from benchmarks import jetstream
from benchmarks import jetstream2
from benchmarks import octane
from benchmarks import rasterize_and_record_micro
from benchmarks import speedometer
from benchmarks import v8_browsing


# We want to prevent benchmarks from accidentally trying to upload too much
# data to the chrome perf dashboard. So the smoke tests below cap the max
# number of values that each story tested would produce when running on the
# waterfall.
MAX_VALUES_PER_TEST_CASE = 1000


def SmokeTestGenerator(benchmark_class, num_pages=1):
  """Generates a smoke test for the first N pages from a benchmark.

  Args:
    benchmark_class: a benchmark class to smoke test.
    num_pages: only smoke test the first N pages, since smoke testing
      everything would take too long to run.
  """
  # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  #
  # This smoke test dynamically tests all benchmarks. So disabling it for one
  # failing or flaky benchmark would disable a much wider swath of coverage
  # than is usually intended. Instead, if a particular benchmark is failing,
  # disable it in tools/perf/benchmarks/*.
  @decorators.Disabled('android')  # crbug.com/641934
  def BenchmarkSmokeTest(self):
    # Some benchmarks are running multiple iterations
    # which is not needed for a smoke test
    if hasattr(benchmark_class, 'enable_smoke_test_mode'):
      benchmark_class.enable_smoke_test_mode = True

    with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
      options = testing.GetRunOptions(
          output_dir=temp_dir,
          benchmark_cls=benchmark_class,
          overrides={'story_shard_end_index': num_pages},
          environment=chromium_config.GetDefaultChromiumConfig())
      options.pageset_repeat = 1  # For smoke testing only run the page once.
      options.output_formats = ['histograms']
      options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE
      results_processor.ProcessOptions(options)

      return_code = benchmark_class().Run(options)
      # TODO(crbug.com/1019139): Make 111 be the exit code that means
      # "no stories were run.".
      if return_code in (-1, 111):
        self.skipTest('The benchmark was not run.')
      self.assertEqual(
          return_code, 0,
          msg='Benchmark run failed: %s' % benchmark_class.Name())
      return_code = results_processor.ProcessResults(options)
      self.assertEqual(
          return_code, 0,
          msg='Result processing failed: %s' % benchmark_class.Name())

  return BenchmarkSmokeTest


# The list of benchmark modules to be excluded from our smoke tests.
_BLACK_LIST_TEST_MODULES = {
    octane,  # Often fails & take long time to timeout on cq bot.
    rasterize_and_record_micro,  # Always fails on cq bot.
    speedometer,  # Takes 101 seconds.
    jetstream,  # Take 206 seconds.
    jetstream2, # Causes CQ shard to timeout, crbug.com/992837
    v8_browsing, # Flaky on Android, crbug.com/628368.
}

# The list of benchmark names to be excluded from our smoke tests.
_BLACK_LIST_TEST_NAMES = [
    'memory.long_running_idle_gmail_background_tbmv2',
    'desktop_ui',  # crbug.com/1159462
    'tab_switching.typical_25',
    'UNSCHEDULED_oortonline_tbmv2',
    'webrtc',  # crbug.com/932036
    'v8.runtime_stats.top_25',  # Fails in Windows, crbug.com/1043048
    'wasmpspdfkit',  # Fails in Chrome OS, crbug.com/1191938
]


def MergeDecorators(method, method_attribute, benchmark, benchmark_attribute):
  # Do set union of attributes to eliminate duplicates.
  merged_attributes = getattr(method, method_attribute, set()).union(
      getattr(benchmark, benchmark_attribute, set()))
  if merged_attributes:
    setattr(method, method_attribute, merged_attributes)


def load_tests(loader, standard_tests, pattern):
  del loader, standard_tests, pattern  # unused
  suite = progress_reporter.TestSuite()

  benchmarks_dir = os.path.dirname(__file__)
  top_level_dir = os.path.dirname(benchmarks_dir)

  # Using the default of |index_by_class_name=False| means that if a module
  # has multiple benchmarks, only the last one is returned.
  all_benchmarks = discover.DiscoverClasses(
      benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
      index_by_class_name=False).values()
  for benchmark in all_benchmarks:
    if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
      continue
    if benchmark.Name() in _BLACK_LIST_TEST_NAMES:
      continue

    class BenchmarkSmokeTest(unittest.TestCase):
      pass

    # tab_switching needs more than one page to test correctly.
    if 'tab_switching' in benchmark.Name():
      method = SmokeTestGenerator(benchmark, num_pages=2)
    else:
      method = SmokeTestGenerator(benchmark)

    # Make sure any decorators are propagated from the original declaration.
    # (access to protected members) pylint: disable=protected-access
    # TODO(dpranke): Since we only pick the first test from every class
    # (above), if that test is disabled, we'll end up not running *any*
    # test from the class. We should probably discover all of the tests
    # in a class, and then throw the ones we don't need away instead.

    disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark)
    disabled_method_attr = decorators.DisabledAttributeName(method)
    enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark)
    enabled_method_attr = decorators.EnabledAttributeName(method)

    MergeDecorators(method, disabled_method_attr, benchmark,
                    disabled_benchmark_attr)
    MergeDecorators(method, enabled_method_attr, benchmark,
                    enabled_benchmark_attr)

    setattr(BenchmarkSmokeTest, benchmark.Name(), method)

    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))

  return suite