blob: c75f8fe654647947bea328931a596e5d3c782d72 [file] [log] [blame]
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import annotations
import argparse
import datetime as dt
import enum
import logging
from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple,
Type, Union, cast)
from crossbench import cli_helper, compat, helper
from crossbench.benchmarks.speedometer.speedometer import (
ProbeClsTupleT, SpeedometerBenchmark, SpeedometerBenchmarkStoryFilter,
SpeedometerProbe, SpeedometerStory)
from crossbench.browsers import viewport as vp
from crossbench.stories.story import Story
if TYPE_CHECKING:
from crossbench.runner.run import Run
ShuffleSeedT = Optional[Union[str, int]]
from crossbench.runner.actions import Actions
from crossbench.types import JSON
class Speedometer30Probe(SpeedometerProbe):
"""
Speedometer3-specific probe (compatible with v3.0).
Extracts all speedometer times and scores.
"""
NAME: str = "speedometer_3.0"
JS: str = "return window.benchmarkClient.metrics"
@property
def speedometer(self) -> Speedometer30Benchmark:
return cast(Speedometer30Benchmark, self.benchmark)
def to_json(self, actions: Actions) -> JSON:
return actions.js(self.JS)
def flatten_json_data(self, json_data: Any) -> JSON:
result: Dict[str, float] = dict()
assert isinstance(json_data, dict), f"Expected dict, got {type(json_data)}"
for name, metric in json_data.items():
result[name] = metric["mean"]
return result
def _valid_metric_key(self, metric_key: str) -> bool:
parts = metric_key.split("/")
if len(parts) != 1:
return False
if self.speedometer.detailed_metrics:
return True
if metric_key.startswith("Iteration-"):
return False
if metric_key == "Geomean":
return False
return True
@enum.unique
class MeasurementMethod(compat.StrEnumWithHelp):
RAF = ("raf", "requestAnimationFrame-based measurement")
TIMER = ("timer", "setTimeout-based measurement")
def to_ms(duration: dt.timedelta) -> int:
return int(round(duration.total_seconds() * 1000))
def parse_shuffle_seed(value: Optional[Any]) -> ShuffleSeedT:
if value in (None, "off", "generate"):
return value
if isinstance(value, int):
return value
return cli_helper.parse_int(value, "shuffle-seed")
# Generated by running this JS snippet and updating the bools:
# JSON.stringify(
# Suites.reduce((data, e) => {
# data[e.name]={ tags:e.tags, enabled:!e.disabled};
# return data}, {}))
# .replaceAll(":true", ":True")
# .replaceAll(":false", ":False");
SPEEDOMETER_3_STORY_DATA = {
"TodoMVC-JavaScript-ES5": {
"tags": ["all", "default", "todomvc"],
"enabled": True
},
"TodoMVC-JavaScript-ES5-Complex-DOM": {
"tags": ["all", "todomvc", "complex"],
"enabled": False
},
"TodoMVC-JavaScript-ES6-Webpack": {
"tags": ["all", "todomvc"],
"enabled": False
},
"TodoMVC-JavaScript-ES6-Webpack-Complex-DOM": {
"tags": ["all", "default", "todomvc", "complex", "complex-default"],
"enabled": True
},
"TodoMVC-WebComponents": {
"tags": ["all", "default", "todomvc", "webcomponents"],
"enabled": True
},
"TodoMVC-WebComponents-Complex-DOM": {
"tags": ["all", "todomvc", "webcomponents", "complex"],
"enabled": False
},
"TodoMVC-React": {
"tags": ["all", "todomvc"],
"enabled": False
},
"TodoMVC-React-Complex-DOM": {
"tags": ["all", "default", "todomvc", "complex", "complex-default"],
"enabled": True
},
"TodoMVC-React-Redux": {
"tags": ["all", "default", "todomvc"],
"enabled": True
},
"TodoMVC-React-Redux-Complex-DOM": {
"tags": ["all", "todomvc", "complex"],
"enabled": False
},
"TodoMVC-Backbone": {
"tags": ["all", "default", "todomvc"],
"enabled": True
},
"TodoMVC-Backbone-Complex-DOM": {
"tags": ["all", "todomvc", "complex"],
"enabled": False
},
"TodoMVC-Angular": {
"tags": ["all", "todomvc"],
"enabled": False
},
"TodoMVC-Angular-Complex-DOM": {
"tags": ["all", "default", "todomvc", "complex", "complex-default"],
"enabled": True
},
"TodoMVC-Vue": {
"tags": ["all", "default", "todomvc"],
"enabled": True
},
"TodoMVC-Vue-Complex-DOM": {
"tags": ["all", "todomvc", "complex", "complex-default"],
"enabled": False
},
"TodoMVC-jQuery": {
"tags": ["all", "default", "todomvc"],
"enabled": True
},
"TodoMVC-jQuery-Complex-DOM": {
"tags": ["all", "todomvc", "complex"],
"enabled": False
},
"TodoMVC-Preact": {
"tags": ["all", "todomvc"],
"enabled": False
},
"TodoMVC-Preact-Complex-DOM": {
"tags": ["all", "default", "todomvc", "complex", "complex-default"],
"enabled": True
},
"TodoMVC-Svelte": {
"tags": ["all", "todomvc"],
"enabled": False
},
"TodoMVC-Svelte-Complex-DOM": {
"tags": ["all", "default", "todomvc", "complex", "complex-default"],
"enabled": True
},
"TodoMVC-Lit": {
"tags": ["all", "todomvc", "webcomponents"],
"enabled": False
},
"TodoMVC-Lit-Complex-DOM": {
"tags": [
"all", "default", "todomvc", "webcomponents", "complex",
"complex-default"
],
"enabled": True
},
"NewsSite-Next": {
"tags": ["all", "default", "newssite", "language"],
"enabled": True
},
"NewsSite-Nuxt": {
"tags": ["all", "default", "newssite"],
"enabled": True
},
"Editor-CodeMirror": {
"tags": ["all", "default", "editor"],
"enabled": True
},
"Editor-TipTap": {
"tags": ["all", "default", "editor"],
"enabled": True
},
"Charts-observable-plot": {
"tags": ["all", "default", "chart"],
"enabled": True
},
"Charts-chartjs": {
"tags": ["all", "default", "chart"],
"enabled": True
},
"React-Stockcharts-SVG": {
"tags": ["all", "default", "chart", "svg"],
"enabled": True
},
"Perf-Dashboard": {
"tags": ["all", "default", "chart", "webcomponents"],
"enabled": True
}
}
class Speedometer30Story(SpeedometerStory):
__doc__ = SpeedometerStory.__doc__
NAME: str = "speedometer_3.0"
# TODO: Update once public version is available
URL: str = "https://sp3-alpha-testing.netlify.app/"
URL_LOCAL: str = "http://127.0.0.1:7000"
SUBSTORIES: Tuple[str, ...] = tuple(SPEEDOMETER_3_STORY_DATA.keys())
@classmethod
def default_story_names(cls) -> Tuple[str, ...]:
return tuple(
tuple(name for name, data in SPEEDOMETER_3_STORY_DATA.items()
if data["enabled"]))
def __init__(self,
substories: Sequence[str] = (),
iterations: Optional[int] = None,
sync_wait: Optional[dt.timedelta] = None,
sync_warmup: Optional[dt.timedelta] = None,
measurement_method: Optional[MeasurementMethod] = None,
viewport: Optional[vp.Viewport] = None,
shuffle_seed: ShuffleSeedT = None,
url: Optional[str] = None):
self._sync_wait = cli_helper.Duration.parse_zero(
sync_wait or dt.timedelta(0), "sync_wait")
self._sync_warmup = cli_helper.Duration.parse_zero(
sync_warmup or dt.timedelta(0), "sync_warmup")
self._measurement_method: MeasurementMethod = (
measurement_method or MeasurementMethod.RAF)
self._viewport = None
if viewport:
self._viewport = vp.Viewport.parse_sized(viewport)
self._shuffle_seed: ShuffleSeedT = parse_shuffle_seed(shuffle_seed)
super().__init__(url=url, substories=substories, iterations=iterations)
@property
def sync_wait(self) -> dt.timedelta:
return self._sync_wait
@property
def sync_warmup(self) -> dt.timedelta:
return self._sync_warmup
@property
def measurement_method(self) -> MeasurementMethod:
return self._measurement_method
@property
def viewport(self) -> Optional[vp.Viewport]:
return self._viewport
@property
def shuffle_seed(self) -> ShuffleSeedT:
return self._shuffle_seed
@property
def url_params(self) -> Dict[str, str]:
url_params = super().url_params
if sync_wait := self.sync_wait:
url_params["waitBeforeSync"] = str(to_ms(sync_wait))
if sync_warmup := self.sync_warmup:
url_params["warmupBeforeSync"] = str(to_ms(sync_warmup))
if self.measurement_method != MeasurementMethod.RAF:
url_params["measurementMethod"] = str(self.measurement_method)
if viewport := self.viewport:
url_params["viewport"] = f"{viewport.width}x{viewport.height}"
if self.shuffle_seed is not None:
url_params["shuffleSeed"] = str(self.shuffle_seed)
return url_params
def log_run_test_url(self, run: Run) -> None:
del run
params = self.url_params
params["suites"] = ",".join(self.substories)
params["developerMode"] = "true"
params["startAutomatically"] = "true"
official_test_url = helper.update_url_query(self.URL, params)
logging.info("STORY PUBLIC TEST URL: %s", official_test_url)
class Speedometer3BenchmarkStoryFilter(SpeedometerBenchmarkStoryFilter):
__doc__ = SpeedometerBenchmarkStoryFilter.__doc__
@classmethod
def add_cli_parser(
cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
parser = super().add_cli_parser(parser)
parser.add_argument(
"--sync-wait",
default=dt.timedelta(0),
type=cli_helper.Duration.parse_zero,
help="Add a custom wait timeout before each sync step.")
parser.add_argument(
"--sync-warmup",
default=dt.timedelta(0),
type=cli_helper.Duration.parse_zero,
help="Run a warmup loop for the given duration before each sync step.")
measurement_method_group = parser.add_argument_group(
"Measurement Method Option")
measurement_method_group = parser.add_mutually_exclusive_group()
measurement_method_group.add_argument(
"--raf",
dest="measurement_method",
default=MeasurementMethod.RAF,
const=MeasurementMethod.RAF,
action="store_const",
help=("Use the default requestAnimationFrame-based approach "
"for async time measurement."))
measurement_method_group.add_argument(
"--timer",
dest="measurement_method",
const=MeasurementMethod.TIMER,
action="store_const",
help=("Use the 'classical' setTimeout-based approach "
"for async time measurement. "
"This might omit measuring some async work."))
parser.add_argument(
"--story-viewport",
type=vp.Viewport.parse_sized,
help="Specify the speedometer workload viewport size.")
parser.add_argument(
"--shuffle-seed",
type=parse_shuffle_seed,
help=("Set a shuffle seed to run the stories in a"
"non-default order."))
return parser
@classmethod
def kwargs_from_cli(cls, args: argparse.Namespace) -> Dict[str, Any]:
kwargs = super().kwargs_from_cli(args)
kwargs["iterations"] = args.iterations
kwargs["measurement_method"] = args.measurement_method
kwargs["sync_wait"] = args.sync_wait
kwargs["sync_warmup"] = args.sync_warmup
kwargs["viewport"] = args.story_viewport
kwargs["shuffle_seed"] = args.shuffle_seed
return kwargs
def __init__(self,
story_cls: Type[SpeedometerStory],
patterns: Sequence[str],
separate: bool = False,
url: Optional[str] = None,
iterations: Optional[int] = None,
measurement_method: Optional[MeasurementMethod] = None,
sync_wait: Optional[dt.timedelta] = None,
sync_warmup: Optional[dt.timedelta] = None,
viewport: Optional[vp.Viewport] = None,
shuffle_seed: ShuffleSeedT = None):
self.measurement_method = measurement_method
self.sync_wait = sync_wait
self.sync_warmup = sync_warmup
self.viewport = viewport
self.shuffle_seed: ShuffleSeedT = shuffle_seed
assert issubclass(story_cls, Speedometer30Story)
super().__init__(story_cls, patterns, separate, url, iterations=iterations)
def create_stories_from_names(self, names: List[str],
separate: bool) -> Sequence[SpeedometerStory]:
return self.story_cls.from_names(
names,
separate=separate,
url=self.url,
iterations=self.iterations,
measurement_method=self.measurement_method,
sync_wait=self.sync_wait,
sync_warmup=self.sync_warmup,
viewport=self.viewport,
shuffle_seed=self.shuffle_seed)
class Speedometer30Benchmark(SpeedometerBenchmark):
"""
Benchmark runner for Speedometer 3.0
"""
NAME: str = "speedometer_3.0"
DEFAULT_STORY_CLS = Speedometer30Story
STORY_FILTER_CLS = Speedometer3BenchmarkStoryFilter
PROBES: ProbeClsTupleT = (Speedometer30Probe,)
@classmethod
def version(cls) -> Tuple[int, ...]:
return (3, 0)
@classmethod
def aliases(cls) -> Tuple[str, ...]:
return ("sp3", "speedometer_3") + super().aliases()
@classmethod
def add_cli_parser(
cls, subparsers: argparse.ArgumentParser, aliases: Sequence[str] = ()
) -> cli_helper.CrossBenchArgumentParser:
parser = super().add_cli_parser(subparsers, aliases)
parser.add_argument(
"--detailed-metrics",
default=False,
action="store_true",
help="Report more detailed internal metrics.")
return parser
@classmethod
def kwargs_from_cli(cls, args: argparse.Namespace) -> Dict[str, Any]:
kwargs = super().kwargs_from_cli(args)
kwargs["detailed_metrics"] = args.detailed_metrics
return kwargs
def __init__(self,
stories: Sequence[Story],
custom_url: Optional[str] = None,
detailed_metrics: bool = False):
self._detailed_metrics = detailed_metrics
super().__init__(stories, custom_url)
@property
def detailed_metrics(self) -> bool:
return self._detailed_metrics