tools/perf/benchmarks/benchmark_smoke_unittest.py

# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the first page of one benchmark for every module.

Only benchmarks that have a composable measurement are included.
Ideally this test would be comprehensive, however, running one page
of every benchmark would run impractically long.
"""

import os
import sys
import unittest

from core import path_util

from telemetry import benchmark as benchmark_module
from telemetry import decorators
from telemetry.testing import options_for_unittests
from telemetry.testing import progress_reporter
from py_utils import discover

from benchmarks import jetstream
from benchmarks import kraken
from benchmarks import octane
from benchmarks import rasterize_and_record_micro
from benchmarks import speedometer
from benchmarks import v8_browsing


MAX_NUM_VALUES = 50000


def SmokeTestGenerator(benchmark, num_pages=1):
  """Generates a benchmark that includes first N pages from pageset.

  Args:
    benchmark: benchmark object to make smoke test.
    num_pages: use the first N pages to run smoke test.
  """
  # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
  #
  # This smoke test dynamically tests all benchmarks. So disabling it for one
  # failing or flaky benchmark would disable a much wider swath of coverage
  # than is usually intended. Instead, if a particular benchmark is failing,
  # disable it in tools/perf/benchmarks/*.
  @decorators.Disabled('chromeos')  # crbug.com/351114
  @decorators.Disabled('android')  # crbug.com/641934
  def BenchmarkSmokeTest(self):
    class SinglePageBenchmark(benchmark):  # pylint: disable=no-init
      # Only measure a single page so that this test cycles reasonably quickly.
      options = benchmark.options.copy()
      options['pageset_repeat'] = 1

      def CreateStorySet(self, options):
        # pylint: disable=super-on-old-class
        story_set = super(SinglePageBenchmark, self).CreateStorySet(options)

        # Only smoke test the first story since smoke testing everything takes
        # too long.
        for s in story_set.stories[num_pages:]:
          story_set.RemoveStory(s)
        return story_set

    # Some benchmarks are running multiple iterations
    # which is not needed for a smoke test
    if hasattr(SinglePageBenchmark, 'enable_smoke_test_mode'):
      SinglePageBenchmark.enable_smoke_test_mode = True

    # Set the benchmark's default arguments.
    options = options_for_unittests.GetCopy()
    options.output_formats = ['none']
    parser = options.CreateParser()

    SinglePageBenchmark.AddCommandLineArgs(parser)
    benchmark_module.AddCommandLineArgs(parser)
    SinglePageBenchmark.SetArgumentDefaults(parser)
    options.MergeDefaultValues(parser.get_default_values())

    # Prevent benchmarks from accidentally trying to upload too much data to the
    # chromeperf dashboard. The number of values uploaded is equal to (the
    # average number of values produced by a single story) * (1 + (the number of
    # stories)). The "1 + " accounts for values summarized across all stories.
    # We can approximate "the average number of values produced by a single
    # story" as the number of values produced by the first story.
    # pageset_repeat doesn't matter because values are summarized across
    # repetitions before uploading.
    story_set = benchmark().CreateStorySet(options)
    SinglePageBenchmark.MAX_NUM_VALUES = MAX_NUM_VALUES / len(story_set.stories)

    SinglePageBenchmark.ProcessCommandLineArgs(None, options)
    benchmark_module.ProcessCommandLineArgs(None, options)

    single_page_benchmark = SinglePageBenchmark()
    with open(path_util.GetExpectationsPath()) as fp:
      single_page_benchmark.AugmentExpectationsWithParser(fp.read())

    self.assertEqual(0, single_page_benchmark.Run(options),
                     msg='Failed: %s' % benchmark)

  return BenchmarkSmokeTest


# The list of benchmark modules to be excluded from our smoke tests.
_BLACK_LIST_TEST_MODULES = {
    octane,  # Often fails & take long time to timeout on cq bot.
    rasterize_and_record_micro,  # Always fails on cq bot.
    speedometer,  # Takes 101 seconds.
    jetstream,  # Take 206 seconds.
    kraken,  # Flaky on Android, crbug.com/626174.
    v8_browsing, # Flaky on Android, crbug.com/628368.
}

# The list of benchmark names to be excluded from our smoke tests.
_BLACK_LIST_TEST_NAMES = [
   'memory.long_running_idle_gmail_background_tbmv2',
   'tab_switching.typical_25',
   'UNSCHEDULED_oortonline_tbmv2',
   'webrtc',  # crbug.com/932036
]


def MergeDecorators(method, method_attribute, benchmark, benchmark_attribute):
  # Do set union of attributes to eliminate duplicates.
  merged_attributes = getattr(method, method_attribute, set()).union(
      getattr(benchmark, benchmark_attribute, set()))
  if merged_attributes:
    setattr(method, method_attribute, merged_attributes)


def load_tests(loader, standard_tests, pattern):
  del loader, standard_tests, pattern  # unused
  suite = progress_reporter.TestSuite()

  benchmarks_dir = os.path.dirname(__file__)
  top_level_dir = os.path.dirname(benchmarks_dir)

  # Using the default of |index_by_class_name=False| means that if a module
  # has multiple benchmarks, only the last one is returned.
  all_benchmarks = discover.DiscoverClasses(
      benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
      index_by_class_name=False).values()
  for benchmark in all_benchmarks:
    if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
      continue
    if benchmark.Name() in _BLACK_LIST_TEST_NAMES:
      continue

    class BenchmarkSmokeTest(unittest.TestCase):
      pass

    # tab_switching needs more than one page to test correctly.
    if 'tab_switching' in benchmark.Name():
      method = SmokeTestGenerator(benchmark, num_pages=2)
    else:
      method = SmokeTestGenerator(benchmark)

    # Make sure any decorators are propagated from the original declaration.
    # (access to protected members) pylint: disable=protected-access
    # TODO(dpranke): Since we only pick the first test from every class
    # (above), if that test is disabled, we'll end up not running *any*
    # test from the class. We should probably discover all of the tests
    # in a class, and then throw the ones we don't need away instead.

    disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark)
    disabled_method_attr = decorators.DisabledAttributeName(method)
    enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark)
    enabled_method_attr = decorators.EnabledAttributeName(method)

    MergeDecorators(method, disabled_method_attr, benchmark,
                    disabled_benchmark_attr)
    MergeDecorators(method, enabled_method_attr, benchmark,
                    enabled_benchmark_attr)

    setattr(BenchmarkSmokeTest, benchmark.Name(), method)

    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))

  return suite