blob: ef34d3e96fc1b1c0c729f488fa6753fdc1a70423 [file] [log] [blame]
# Copyright 2022 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import abc
import pathlib
import pyfakefs.fake_filesystem_unittest
try:
from . import mockbenchmark
except ImportError:
# VSCode has issues discovering tests code
from tests import mockbenchmark
import crossbench as cb
import crossbench.benchmarks as bm
class BaseRunnerTest(
pyfakefs.fake_filesystem_unittest.TestCase, metaclass=abc.ABCMeta):
def setUp(self):
self.setUpPyfakefs(modules_to_reload=[cb, mockbenchmark])
mockbenchmark.MockBrowserDev.setup_fs(self.fs)
mockbenchmark.MockBrowserStable.setup_fs(self.fs)
self.platform = mockbenchmark.mock_platform
self.out_dir = pathlib.Path("tmp/results/test")
self.out_dir.parent.mkdir(parents=True)
self.browsers = [
mockbenchmark.MockBrowserDev("dev", platform=self.platform),
mockbenchmark.MockBrowserStable("stable", platform=self.platform)
]
class TestPageLoadBenchmark(BaseRunnerTest):
BENCHMARK = bm.loading.PageLoadBenchmark
def test_default_stories(self):
stories = bm.loading.LivePage.from_names(["all"])
self.assertGreater(len(stories), 1)
for story in stories:
self.assertIsInstance(story, bm.loading.LivePage)
def test_combined_stories(self):
stories = bm.loading.LivePage.from_names(["all"], separate=False)
self.assertEqual(len(stories), 1)
combined = stories[0]
self.assertIsInstance(combined, bm.loading.CombinedPage)
def test_filter_by_name(self):
for page in bm.loading.PAGE_LIST:
stories = bm.loading.LivePage.from_names([page.name])
self.assertListEqual(stories, [page])
self.assertListEqual(bm.loading.LivePage.from_names([]), [])
def test_filter_by_name_with_duration(self):
pages = bm.loading.PAGE_LIST
filtered_pages = bm.loading.LivePage.from_names(
[pages[0].name, pages[1].name, '1001'])
self.assertListEqual(filtered_pages, [pages[0], pages[1]])
self.assertEqual(filtered_pages[0].duration, pages[0].duration)
self.assertEqual(filtered_pages[1].duration, 1001)
def test_page_by_url(self):
url1 = "http:://example.com/test1"
url2 = "http:://example.com/test2"
stories = bm.loading.LivePage.from_names([url1, url2])
self.assertEqual(len(stories), 2)
self.assertEqual(stories[0].url, url1)
self.assertEqual(stories[1].url, url2)
def test_page_by_url_combined(self):
url1 = "http:://example.com/test1"
url2 = "http:://example.com/test2"
stories = bm.loading.LivePage.from_names([url1, url2], separate=False)
self.assertEqual(len(stories), 1)
combined = stories[0]
self.assertIsInstance(combined, bm.loading.CombinedPage)
def test_run(self):
stories = bm.loading.PAGE_LIST
benchmark = self.BENCHMARK(stories)
self.assertTrue(len(benchmark.describe()) > 0)
runner = cb.runner.Runner(
self.out_dir,
self.browsers,
benchmark,
use_checklist=False,
platform=self.platform)
runner.run()
self.assertEqual(self.browsers[0].url_list,
[story.url for story in stories])
self.assertEqual(self.browsers[1].url_list,
[story.url for story in stories])
self.assertTrue(self.browsers[0].did_run)
self.assertTrue(self.browsers[1].did_run)
class Speedometer2Test(BaseRunnerTest):
BENCHMARK = bm.speedometer.Speedometer20Benchmark
def test_story_filtering(self):
stories = bm.speedometer.Speedometer20Story.from_names([])
self.assertEqual(len(stories), 1)
stories = bm.speedometer.Speedometer20Story.from_names([], separate=True)
self.assertEqual(len(stories),
len(bm.speedometer.Speedometer20Story.SUBSTORIES))
stories_b = bm.speedometer.Speedometer20Story.from_names(".*",
separate=True)
self.assertListEqual(
[story.name for story in stories],
[story.name for story in stories_b],
)
stories_c = bm.speedometer.Speedometer20Story.from_names([".*"],
separate=True)
self.assertListEqual(
[story.name for story in stories],
[story.name for story in stories_c],
)
def test_invalid_story_names(self):
with self.assertRaises(Exception):
# Only one regexp entry will work
bm.speedometer.Speedometer20Story.from_names([".*", 'a'], separate=True)
def test_run(self):
repetitions = 3
iterations = 2
stories = bm.speedometer.Speedometer20Story.from_names(['VanillaJS-TodoMVC'])
example_story_data = {
"tests": {
"Adding100Items": {
"tests": {
"Sync": 74.6000000089407,
"Async": 6.299999997019768
},
"total": 80.90000000596046
},
"CompletingAllItems": {
"tests": {
"Sync": 22.600000008940697,
"Async": 5.899999991059303
},
"total": 28.5
},
"DeletingItems": {
"tests": {
"Sync": 11.800000011920929,
"Async": 0.19999998807907104
},
"total": 12
}
},
"total": 121.40000000596046
}
speedometer_probe_results = [{
"tests": {story.name: example_story_data
for story in stories},
"total": 1000,
"mean": 2000,
"geomean": 3000,
"score": 10
} for i in range(iterations)]
for browser in self.browsers:
browser.js_side_effect = [
True, # Page is ready
None, # filter benchmarks
None, # Start running benchmark
False, # Wait, ...
True, # until done
speedometer_probe_results,
]
benchmark = self.BENCHMARK(stories)
self.assertTrue(len(benchmark.describe()) > 0)
runner = cb.runner.Runner(
self.out_dir,
self.browsers,
benchmark,
use_checklist=False,
platform=self.platform,
repetitions=repetitions)
runner.run()
for browser in self.browsers:
self.assertEqual(len(browser.url_list), repetitions)
self.assertIn(bm.speedometer.Speedometer20Probe.JS, browser.js_list)
class JetStream2Test(BaseRunnerTest):
BENCHMARK = bm.jetstream.JetStream2Benchmark
def test_run(self):
stories = bm.jetstream.JetStream2Story.from_names(['WSL'])
example_story_data = {'firstIteration': 1, 'average': 0.1, 'worst4': 1.1}
jetstream_probe_results = {
story.name: example_story_data for story in stories
}
for browser in self.browsers:
browser.js_side_effect = [
True, # Page is ready
None, # filter benchmarks
True, # UI is updated and ready,
None, # Start running benchmark
True, # Wait until done
jetstream_probe_results,
]
repetitions = 3
benchmark = self.BENCHMARK(stories)
self.assertTrue(len(benchmark.describe()) > 0)
runner = cb.runner.Runner(
self.out_dir,
self.browsers,
benchmark,
use_checklist=False,
platform=self.platform,
repetitions=repetitions)
runner.run()
for browser in self.browsers:
self.assertEqual(len(browser.url_list), repetitions)
self.assertIn(bm.jetstream.JetStream2Probe.JS, browser.js_list)
class MotionMark2Test(BaseRunnerTest):
BENCHMARK = bm.motionmark.MotionMark12Benchmark