Add basic speedometer2 and motionnmark unittests
- Rename Speedometer20Runner to Speedometer20Benchmark
- Add some more types
- Fix import errors when running tests under vscode
Change-Id: I7dee54af13b9e0167a7862ce5fe6aa1c93099d17
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/3909528
Reviewed-by: Alexander Schulze <alexschulze@chromium.org>
diff --git a/crossbench/benchmarks/__init__.py b/crossbench/benchmarks/__init__.py
index 1d1e32f..afcdbf5 100644
--- a/crossbench/benchmarks/__init__.py
+++ b/crossbench/benchmarks/__init__.py
@@ -6,4 +6,4 @@
from crossbench.benchmarks.jetstream import JetStream2Benchmark
from crossbench.benchmarks.loading import PageLoadBenchmark
from crossbench.benchmarks.motionmark import MotionMark12Benchmark
-from crossbench.benchmarks.speedometer import Speedometer20Runner
+from crossbench.benchmarks.speedometer import Speedometer20Benchmark
diff --git a/crossbench/benchmarks/speedometer.py b/crossbench/benchmarks/speedometer.py
index d595090..44abe6b 100644
--- a/crossbench/benchmarks/speedometer.py
+++ b/crossbench/benchmarks/speedometer.py
@@ -148,7 +148,7 @@
2 * len(self._substories) * self.iterations))
-class Speedometer20Runner(benchmarks.PressBenchmark):
+class Speedometer20Benchmark(benchmarks.PressBenchmark):
"""
Benchmark runner for Speedometer 2.0
"""
diff --git a/crossbench/cli.py b/crossbench/cli.py
index 4315891..45e5f89 100644
--- a/crossbench/cli.py
+++ b/crossbench/cli.py
@@ -223,7 +223,7 @@
class CrossBenchCLI:
BENCHMARKS = (
- cb.benchmarks.Speedometer20Runner,
+ cb.benchmarks.Speedometer20Benchmark,
cb.benchmarks.JetStream2Benchmark,
cb.benchmarks.MotionMark12Benchmark,
cb.benchmarks.PageLoadBenchmark,
diff --git a/crossbench/probes/json.py b/crossbench/probes/json.py
index 770de60..4a6afcb 100644
--- a/crossbench/probes/json.py
+++ b/crossbench/probes/json.py
@@ -30,7 +30,7 @@
FLATTEN = True
@property
- def results_file_name(self):
+ def results_file_name(self) -> str:
return f"{self.name}.json"
@abc.abstractmethod
@@ -144,14 +144,14 @@
return sum(self.values) / len(self.values)
@property
- def geomean(self):
+ def geomean(self) -> float:
product = 1
for value in self.values:
product *= value
return product**(1 / len(self.values))
@property
- def stddev(self):
+ def stddev(self) -> float:
"""
We're ignoring here any actual distribution of the data and use this as a
rough estimate of the quality of the data
diff --git a/crossbench/stories.py b/crossbench/stories.py
index e482e22..1095137 100644
--- a/crossbench/stories.py
+++ b/crossbench/stories.py
@@ -7,7 +7,7 @@
import logging
import re
from abc import ABC, ABCMeta, abstractmethod
-from typing import Tuple
+from typing import List, Optional, Sequence, Tuple, Union
class Story(ABC):
@@ -15,7 +15,7 @@
@classmethod
@abstractmethod
- def story_names(cls):
+ def story_names(cls) -> Sequence[str]:
pass
@classmethod
@@ -23,7 +23,7 @@
def from_names(cls, names, separate=False):
pass
- def __init__(self, name: str, duration=15):
+ def __init__(self, name: str, duration: float = 15):
assert name, "Invalid page name"
self._name = name
assert duration > 0, (
@@ -31,13 +31,13 @@
self.duration = duration
@property
- def name(self):
+ def name(self) -> str:
return self._name
def details_json(self):
return dict(name=self.name, duration=self.duration)
- def is_done(self, _):
+ def is_done(self, _) -> bool:
return True
@abstractmethod
@@ -59,9 +59,12 @@
return cls.SUBSTORIES
@classmethod
- def from_names(cls, names, separate=False, live=True):
- if len(names) == 1:
- first = names[0]
+ def from_names(cls, names:Union[Sequence[str], str], separate=False, live=True):
+ if len(names) == 1 or isinstance(names, str):
+ if isinstance(names, str):
+ first = names
+ else:
+ first = names[0]
if first == "all":
names = cls.SUBSTORIES
elif first not in cls.SUBSTORIES:
@@ -97,13 +100,21 @@
]
@classmethod
- def get_substories(cls, separate, substories):
+ def get_substories(cls, separate:bool, substories:Sequence[str]):
substories = substories or cls.SUBSTORIES
if separate:
return substories
return [substories]
- def __init__(self, *args, is_live=True, substories=None, **kwargs):
+
+ _substories : List[str]
+ is_live : bool
+
+ def __init__(self,
+ *args,
+ is_live: bool = True,
+ substories: Optional[Union[str, List[str]]] = None,
+ **kwargs):
cls = self.__class__
assert self.SUBSTORIES, f"{cls}.SUBSTORIES is not set."
assert self.NAME is not None, f"{cls}.NAME is not set."
@@ -125,7 +136,7 @@
self._url = self.URL_LOCAL
assert self._url is not None, f"Invalid URL for {self.NAME}"
- def _verify_url(self, url, property_name):
+ def _verify_url(self, url:str, property_name:str):
cls = self.__class__
assert url is not None, f"{cls}.{property_name} is not set."
diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py
index ec02634..ef34d3e 100644
--- a/tests/test_benchmarks.py
+++ b/tests/test_benchmarks.py
@@ -6,13 +6,16 @@
import pathlib
import pyfakefs.fake_filesystem_unittest
-from . import mockbenchmark
+try:
+ from . import mockbenchmark
+except ImportError:
+ # VSCode has issues discovering tests code
+ from tests import mockbenchmark
import crossbench as cb
import crossbench.benchmarks as bm
-
class BaseRunnerTest(
pyfakefs.fake_filesystem_unittest.TestCase, metaclass=abc.ABCMeta):
@@ -93,19 +96,110 @@
self.assertTrue(self.browsers[1].did_run)
+class Speedometer2Test(BaseRunnerTest):
+ BENCHMARK = bm.speedometer.Speedometer20Benchmark
+
+ def test_story_filtering(self):
+ stories = bm.speedometer.Speedometer20Story.from_names([])
+ self.assertEqual(len(stories), 1)
+ stories = bm.speedometer.Speedometer20Story.from_names([], separate=True)
+ self.assertEqual(len(stories),
+ len(bm.speedometer.Speedometer20Story.SUBSTORIES))
+ stories_b = bm.speedometer.Speedometer20Story.from_names(".*",
+ separate=True)
+ self.assertListEqual(
+ [story.name for story in stories],
+ [story.name for story in stories_b],
+ )
+ stories_c = bm.speedometer.Speedometer20Story.from_names([".*"],
+ separate=True)
+ self.assertListEqual(
+ [story.name for story in stories],
+ [story.name for story in stories_c],
+ )
+
+ def test_invalid_story_names(self):
+ with self.assertRaises(Exception):
+ # Only one regexp entry will work
+ bm.speedometer.Speedometer20Story.from_names([".*", 'a'], separate=True)
+
+
+ def test_run(self):
+ repetitions = 3
+ iterations = 2
+ stories = bm.speedometer.Speedometer20Story.from_names(['VanillaJS-TodoMVC'])
+ example_story_data = {
+ "tests": {
+ "Adding100Items": {
+ "tests": {
+ "Sync": 74.6000000089407,
+ "Async": 6.299999997019768
+ },
+ "total": 80.90000000596046
+ },
+ "CompletingAllItems": {
+ "tests": {
+ "Sync": 22.600000008940697,
+ "Async": 5.899999991059303
+ },
+ "total": 28.5
+ },
+ "DeletingItems": {
+ "tests": {
+ "Sync": 11.800000011920929,
+ "Async": 0.19999998807907104
+ },
+ "total": 12
+ }
+ },
+ "total": 121.40000000596046
+ }
+ speedometer_probe_results = [{
+ "tests": {story.name: example_story_data
+ for story in stories},
+ "total": 1000,
+ "mean": 2000,
+ "geomean": 3000,
+ "score": 10
+ } for i in range(iterations)]
+
+ for browser in self.browsers:
+ browser.js_side_effect = [
+ True, # Page is ready
+ None, # filter benchmarks
+ None, # Start running benchmark
+ False, # Wait, ...
+ True, # until done
+ speedometer_probe_results,
+ ]
+ benchmark = self.BENCHMARK(stories)
+ self.assertTrue(len(benchmark.describe()) > 0)
+ runner = cb.runner.Runner(
+ self.out_dir,
+ self.browsers,
+ benchmark,
+ use_checklist=False,
+ platform=self.platform,
+ repetitions=repetitions)
+ runner.run()
+ for browser in self.browsers:
+ self.assertEqual(len(browser.url_list), repetitions)
+ self.assertIn(bm.speedometer.Speedometer20Probe.JS, browser.js_list)
+
+
class JetStream2Test(BaseRunnerTest):
BENCHMARK = bm.jetstream.JetStream2Benchmark
def test_run(self):
stories = bm.jetstream.JetStream2Story.from_names(['WSL'])
- example_story_data = {'firstItertaion': 1, 'average': 0.1, 'worst4': 1.1}
+ example_story_data = {'firstIteration': 1, 'average': 0.1, 'worst4': 1.1}
jetstream_probe_results = {
story.name: example_story_data for story in stories
}
for browser in self.browsers:
browser.js_side_effect = [
True, # Page is ready
- None, # filter benchmnarks
+ None, # filter benchmarks
True, # UI is updated and ready,
None, # Start running benchmark
True, # Wait until done
@@ -125,3 +219,6 @@
for browser in self.browsers:
self.assertEqual(len(browser.url_list), repetitions)
self.assertIn(bm.jetstream.JetStream2Probe.JS, browser.js_list)
+
+class MotionMark2Test(BaseRunnerTest):
+ BENCHMARK = bm.motionmark.MotionMark12Benchmark
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d4236e5..eb1d41f 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -10,7 +10,11 @@
import pyfakefs.fake_filesystem_unittest
-from . import mockbenchmark
+try:
+ from . import mockbenchmark
+except ImportError:
+ # VSCode has issues discovering tests code
+ from tests import mockbenchmark
import crossbench as cb
from crossbench.cli import BrowserConfig, CrossBenchCLI, FlagGroupConfig
