Add JetStream v3.0 support
For now JetStream3* classes are all based on the v2 versions as
there are no runner or metrics changes yet.
- Add jetstream.py base file
- Add jestream_3.py base file
- Use more direct imports in test_jetstream.py
Bug: 358290763
Change-Id: Id89117c905726af5f68e0a21b5e55068969601cc
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/5785818
Reviewed-by: Patrick Thier <pthier@chromium.org>
Commit-Queue: Camillo Bruni <cbruni@chromium.org>
diff --git a/crossbench/benchmarks/all.py b/crossbench/benchmarks/all.py
index 31acfb8..56e4cf9 100644
--- a/crossbench/benchmarks/all.py
+++ b/crossbench/benchmarks/all.py
@@ -7,7 +7,8 @@
from crossbench.benchmarks.experimental.power import PowerBenchmark
from crossbench.benchmarks.jetstream import (JetStream20Benchmark,
JetStream21Benchmark,
- JetStream22Benchmark)
+ JetStream22Benchmark,
+ JetStream30Benchmark)
from crossbench.benchmarks.loading.loading_benchmark import PageLoadBenchmark
from crossbench.benchmarks.manual import ManualBenchmark
from crossbench.benchmarks.motionmark import (MotionMark10Benchmark,
diff --git a/crossbench/benchmarks/jetstream/__init__.py b/crossbench/benchmarks/jetstream/__init__.py
index 91696b0..229e21f 100644
--- a/crossbench/benchmarks/jetstream/__init__.py
+++ b/crossbench/benchmarks/jetstream/__init__.py
@@ -7,3 +7,4 @@
from crossbench.benchmarks.jetstream.jetstream_2_0 import JetStream20Benchmark
from crossbench.benchmarks.jetstream.jetstream_2_1 import JetStream21Benchmark
from crossbench.benchmarks.jetstream.jetstream_2_2 import JetStream22Benchmark
+from crossbench.benchmarks.jetstream.jetstream_3_0 import JetStream30Benchmark
diff --git a/crossbench/benchmarks/jetstream/jetstream.py b/crossbench/benchmarks/jetstream/jetstream.py
new file mode 100644
index 0000000..ab3c468
--- /dev/null
+++ b/crossbench/benchmarks/jetstream/jetstream.py
@@ -0,0 +1,20 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import annotations
+
+import abc
+
+from crossbench.benchmarks.base import PressBenchmark
+
+
+class JetStreamBenchmark(PressBenchmark, metaclass=abc.ABCMeta):
+
+ @classmethod
+ def short_base_name(cls) -> str:
+ return "js"
+
+ @classmethod
+ def base_name(cls) -> str:
+ return "jetstream"
diff --git a/crossbench/benchmarks/jetstream/jetstream_2.py b/crossbench/benchmarks/jetstream/jetstream_2.py
index 5c1d339..51e0dfe 100644
--- a/crossbench/benchmarks/jetstream/jetstream_2.py
+++ b/crossbench/benchmarks/jetstream/jetstream_2.py
@@ -11,16 +11,17 @@
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Type
-from crossbench.benchmarks.base import BenchmarkProbeMixin, PressBenchmark
+from crossbench.benchmarks.base import BenchmarkProbeMixin
+from crossbench.benchmarks.jetstream.jetstream import JetStreamBenchmark
from crossbench.probes import metric as cb_metric
from crossbench.probes.json import JsonResultProbe
from crossbench.probes.results import ProbeResult, ProbeResultDict
from crossbench.stories.press_benchmark import PressBenchmarkStory
if TYPE_CHECKING:
+ from crossbench.path import LocalPath
from crossbench.runner.actions import Actions
from crossbench.runner.groups import BrowsersRunGroup, StoriesRunGroup
- from crossbench.path import LocalPath
from crossbench.runner.run import Run
@@ -229,16 +230,5 @@
ProbeClsTupleT = Tuple[Type[JetStream2Probe], ...]
-class JetStreamBenchmark(PressBenchmark, metaclass=abc.ABCMeta):
-
- @classmethod
- def short_base_name(cls) -> str:
- return "js"
-
- @classmethod
- def base_name(cls) -> str:
- return "jetstream"
-
-
class JetStream2Benchmark(JetStreamBenchmark):
pass
diff --git a/crossbench/benchmarks/jetstream/jetstream_3.py b/crossbench/benchmarks/jetstream/jetstream_3.py
new file mode 100644
index 0000000..0750f74
--- /dev/null
+++ b/crossbench/benchmarks/jetstream/jetstream_3.py
@@ -0,0 +1,33 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import annotations
+
+import abc
+from typing import Tuple, Type
+
+from crossbench.benchmarks.jetstream.jetstream_2 import (JetStream2Benchmark,
+ JetStream2Probe,
+ JetStream2Story)
+
+
+# TODO: introduce JetStreamProbe
+class JetStream3Probe(JetStream2Probe, metaclass=abc.ABCMeta):
+ """
+ JetStream3-specific Probe.
+ Extracts all JetStream 3 times and scores.
+ """
+
+
+# TODO: introduce JetStreamStory
+class JetStream3Story(JetStream2Story, metaclass=abc.ABCMeta):
+ SUBSTORIES: Tuple[str, ...] = ()
+
+
+ProbeClsTupleT = Tuple[Type[JetStream3Probe], ...]
+
+
+# TODO: introduce JetStreamBenchmark
+class JetStream3Benchmark(JetStream2Benchmark):
+ pass
diff --git a/crossbench/benchmarks/jetstream/jetstream_3_0.py b/crossbench/benchmarks/jetstream/jetstream_3_0.py
new file mode 100644
index 0000000..67fdf62
--- /dev/null
+++ b/crossbench/benchmarks/jetstream/jetstream_3_0.py
@@ -0,0 +1,127 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import annotations
+
+from typing import Tuple
+
+from crossbench.benchmarks.jetstream.jetstream_3 import (JetStream3Benchmark,
+ JetStream3Probe,
+ JetStream3Story,
+ ProbeClsTupleT)
+
+
+class JetStream30Probe(JetStream3Probe):
+ __doc__ = JetStream3Probe.__doc__
+ NAME: str = "jetstream_3.0"
+
+
+class JetStream30Story(JetStream3Story):
+ __doc__ = JetStream3Story.__doc__
+ NAME: str = "jetstream_3.0"
+ URL: str = "https://chromium-workloads.web.app/jetstream/v3.0/"
+ URL_OFFICIAL: str = "https://browserbench.org/JetStream3.0/"
+ SUBSTORIES: Tuple[str, ...] = (
+ "WSL",
+ "UniPoker",
+ "uglify-js-wtb",
+ "typescript",
+ "tsf-wasm",
+ "tfjs-wasm-simd",
+ "tfjs-wasm",
+ "tagcloud-SP",
+ "sync-fs",
+ "string-unpack-code-SP",
+ "stanford-crypto-sha256",
+ "stanford-crypto-pbkdf2",
+ "stanford-crypto-aes",
+ "splay",
+ "segmentation",
+ "richards-wasm",
+ "richards",
+ "regexp",
+ "regex-dna-SP",
+ "raytrace-public-class-fields",
+ "raytrace-private-class-fields",
+ "raytrace",
+ "quicksort-wasm",
+ "proxy-vue",
+ "proxy-mobx",
+ "prepack-wtb",
+ "pdfjs",
+ "OfflineAssembler",
+ "octane-zlib",
+ "octane-code-load",
+ "navier-stokes",
+ "n-body-SP",
+ "multi-inspector-code-load",
+ "ML",
+ "mandreel",
+ "lebab-wtb",
+ "lazy-collections",
+ "json-stringify-inspector",
+ "json-parse-inspector",
+ "jshint-wtb",
+ "js-tokens",
+ "HashSet-wasm",
+ "hash-map",
+ "gcc-loops-wasm",
+ "gbemu",
+ "gaussian-blur",
+ "float-mm.c",
+ "FlightPlanner",
+ "first-inspector-code-load",
+ "espree-wtb",
+ "earley-boyer",
+ "doxbee-promise",
+ "doxbee-async",
+ "delta-blue",
+ "date-format-xparb-SP",
+ "date-format-tofte-SP",
+ "crypto-sha1-SP",
+ "crypto-md5-SP",
+ "crypto-aes-SP",
+ "crypto",
+ "coffeescript-wtb",
+ "chai-wtb",
+ "cdjs",
+ "Box2D",
+ "bomb-workers",
+ "bigint-paillier",
+ "bigint-noble-secp256k1",
+ "bigint-noble-ed25519",
+ "bigint-noble-bls12-381",
+ "bigint-bigdenary",
+ "Basic",
+ "base64-SP",
+ "babylon-wtb",
+ "Babylon",
+ "async-fs",
+ "argon2-wasm-simd",
+ "argon2-wasm",
+ "Air",
+ "ai-astar",
+ "acorn-wtb",
+ "8bitbench-wasm",
+ "3d-raytrace-SP",
+ "3d-cube-SP",
+ )
+
+
+class JetStream30Benchmark(JetStream3Benchmark):
+ """
+ Benchmark runner for JetStream 3.0.
+ """
+
+ NAME: str = "jetstream_3.0"
+ DEFAULT_STORY_CLS = JetStream30Story
+ PROBES: ProbeClsTupleT = (JetStream30Probe,)
+
+ @classmethod
+ def version(cls) -> Tuple[int, ...]:
+ return (3, 0)
+
+ @classmethod
+ def aliases(cls) -> Tuple[str, ...]:
+ return ("js3", "jetstream_3") + super().aliases()
diff --git a/crossbench/cbb/cbb_adapter.py b/crossbench/cbb/cbb_adapter.py
index 562328f..b229e46 100644
--- a/crossbench/cbb/cbb_adapter.py
+++ b/crossbench/cbb/cbb_adapter.py
@@ -36,6 +36,7 @@
benchmarks.JetStream20Benchmark,
benchmarks.JetStream21Benchmark,
benchmarks.JetStream22Benchmark,
+ benchmarks.JetStream30Benchmark,
]
press_benchmarks_dict = {cls.NAME: cls for cls in press_benchmarks}
diff --git a/crossbench/cli/cli.py b/crossbench/cli/cli.py
index ca17fa3..3a8629b 100644
--- a/crossbench/cli/cli.py
+++ b/crossbench/cli/cli.py
@@ -147,6 +147,7 @@
benchmarks.Speedometer30Benchmark,
benchmarks.Speedometer21Benchmark,
benchmarks.Speedometer20Benchmark,
+ benchmarks.JetStream30Benchmark,
benchmarks.JetStream22Benchmark,
benchmarks.JetStream21Benchmark,
benchmarks.JetStream20Benchmark,
diff --git a/tests/crossbench/benchmarks/jetstream_helper.py b/tests/crossbench/benchmarks/jetstream_helper.py
index 1ecbc97..13299ef 100644
--- a/tests/crossbench/benchmarks/jetstream_helper.py
+++ b/tests/crossbench/benchmarks/jetstream_helper.py
@@ -126,3 +126,8 @@
self.assertIn("JetStream results", output)
self.assertIn("102.22.33.44", output)
self.assertIn("100.22.33.44", output)
+
+
+# TODO: introduce JetStreamBaseTestCase
+class JetStream3BaseTestCase(JetStream2BaseTestCase, metaclass=abc.ABCMeta):
+ pass
diff --git a/tests/crossbench/benchmarks/test_all.py b/tests/crossbench/benchmarks/test_all.py
index b87a837..b9e7574 100644
--- a/tests/crossbench/benchmarks/test_all.py
+++ b/tests/crossbench/benchmarks/test_all.py
@@ -13,6 +13,7 @@
from crossbench.benchmarks.jetstream.jetstream_2_0 import JetStream20Benchmark
from crossbench.benchmarks.jetstream.jetstream_2_1 import JetStream21Benchmark
from crossbench.benchmarks.jetstream.jetstream_2_2 import JetStream22Benchmark
+from crossbench.benchmarks.jetstream.jetstream_3_0 import JetStream30Benchmark
from crossbench.benchmarks.loading.loading_benchmark import PageLoadBenchmark
from crossbench.benchmarks.manual.manual_benchmark import ManualBenchmark
from crossbench.benchmarks.motionmark.motionmark_1_0 import \
@@ -36,6 +37,7 @@
JetStream20Benchmark,
JetStream21Benchmark,
JetStream22Benchmark,
+ JetStream30Benchmark,
PageLoadBenchmark,
ManualBenchmark,
MotionMark10Benchmark,
diff --git a/tests/crossbench/benchmarks/test_jetstream.py b/tests/crossbench/benchmarks/test_jetstream.py
index 822d765..9b5b8cd 100644
--- a/tests/crossbench/benchmarks/test_jetstream.py
+++ b/tests/crossbench/benchmarks/test_jetstream.py
@@ -2,9 +2,20 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-from crossbench.benchmarks.jetstream import (jetstream_2_0, jetstream_2_1,
- jetstream_2_2)
+from crossbench.benchmarks.jetstream.jetstream_2_0 import (JetStream20Benchmark,
+ JetStream20Probe,
+ JetStream20Story)
+from crossbench.benchmarks.jetstream.jetstream_2_1 import (JetStream21Benchmark,
+ JetStream21Probe,
+ JetStream21Story)
+from crossbench.benchmarks.jetstream.jetstream_2_2 import (JetStream22Benchmark,
+ JetStream22Probe,
+ JetStream22Story)
+from crossbench.benchmarks.jetstream.jetstream_3_0 import (JetStream30Benchmark,
+ JetStream30Probe,
+ JetStream30Story)
from tests import test_helper
+# Only import module to avoid exposing the abstract test classes to the runner.
from tests.crossbench.benchmarks import jetstream_helper
@@ -12,15 +23,15 @@
@property
def benchmark_cls(self):
- return jetstream_2_0.JetStream20Benchmark
+ return JetStream20Benchmark
@property
def story_cls(self):
- return jetstream_2_0.JetStream20Story
+ return JetStream20Story
@property
def probe_cls(self):
- return jetstream_2_0.JetStream20Probe
+ return JetStream20Probe
@property
def name(self):
@@ -31,15 +42,15 @@
@property
def benchmark_cls(self):
- return jetstream_2_1.JetStream21Benchmark
+ return JetStream21Benchmark
@property
def story_cls(self):
- return jetstream_2_1.JetStream21Story
+ return JetStream21Story
@property
def probe_cls(self):
- return jetstream_2_1.JetStream21Probe
+ return JetStream21Probe
@property
def name(self):
@@ -50,20 +61,39 @@
@property
def benchmark_cls(self):
- return jetstream_2_2.JetStream22Benchmark
+ return JetStream22Benchmark
@property
def story_cls(self):
- return jetstream_2_2.JetStream22Story
+ return JetStream22Story
@property
def probe_cls(self):
- return jetstream_2_2.JetStream22Probe
+ return JetStream22Probe
@property
def name(self):
return "jetstream_2.2"
+class JetStream30TestCase(jetstream_helper.JetStream3BaseTestCase):
+
+ @property
+ def benchmark_cls(self):
+ return JetStream30Benchmark
+
+ @property
+ def story_cls(self):
+ return JetStream30Story
+
+ @property
+ def probe_cls(self):
+ return JetStream30Probe
+
+ @property
+ def name(self):
+ return "jetstream_3.0"
+
+
if __name__ == "__main__":
test_helper.run_pytest(__file__)
