Update URLs handling
- Use chrome-controlled https://chromium-workloads.web.app/ by default
- Add --official option for using the offically hosted version
- Add more url option alternatives
- Add more motionmark versions, mostly for consistency
Change-Id: I064805024fa998416c7878cafe83168b54a1d18c
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/5344754
Reviewed-by: Patrick Thier <pthier@chromium.org>
Commit-Queue: Camillo Bruni <cbruni@chromium.org>
diff --git a/crossbench/benchmarks/all.py b/crossbench/benchmarks/all.py
index bfc0ec2..c0f412c 100644
--- a/crossbench/benchmarks/all.py
+++ b/crossbench/benchmarks/all.py
@@ -9,7 +9,9 @@
JetStream21Benchmark)
from crossbench.benchmarks.loading.loading_benchmark import PageLoadBenchmark
from crossbench.benchmarks.manual import ManualBenchmark
-from crossbench.benchmarks.motionmark import (MotionMark12Benchmark,
+from crossbench.benchmarks.motionmark import (MotionMark10Benchmark,
+ MotionMark11Benchmark,
+ MotionMark12Benchmark,
MotionMark13Benchmark)
from crossbench.benchmarks.speedometer import (Speedometer20Benchmark,
Speedometer21Benchmark,
diff --git a/crossbench/benchmarks/base.py b/crossbench/benchmarks/base.py
index 236c5ee..e382fde 100644
--- a/crossbench/benchmarks/base.py
+++ b/crossbench/benchmarks/base.py
@@ -397,23 +397,36 @@
parser = super().add_cli_parser(subparsers, aliases)
# TODO: Move story-related args to dedicated PressBenchmarkStoryFilter class
benchmark_url_group = parser.add_mutually_exclusive_group()
- default_live_url = cls.DEFAULT_STORY_CLS.URL
- default_local_url = cls.DEFAULT_STORY_CLS.URL_LOCAL
+ live_url = cls.DEFAULT_STORY_CLS.URL
+ local_url = cls.DEFAULT_STORY_CLS.URL_LOCAL
+ official_url = cls.DEFAULT_STORY_CLS.URL_OFFICIAL
benchmark_url_group.add_argument(
"--live",
+ "--live-url",
+ "--browser-ben",
dest="custom_benchmark_url",
const=None,
action="store_const",
- help=f"Use live/online benchmark url ({default_live_url}).")
+ help=(f"Use chrome live benchmark url ({live_url}) "
+ "on https://browserben.ch."))
+ benchmark_url_group.add_argument(
+ "--official",
+ "--official-url",
+ dest="custom_benchmark_url",
+ const=official_url,
+ action="store_const",
+ help=(f"Use officially hosted live/online benchmark url "
+ f"({official_url})."))
benchmark_url_group.add_argument(
"--local",
+ "--local-url",
"--url",
"--custom-benchmark-url",
type=cli_helper.parse_httpx_url_str,
nargs="?",
dest="custom_benchmark_url",
- const=default_local_url,
- help=(f"Use custom or locally (default={default_local_url}) "
+ const=local_url,
+ help=(f"Use custom or locally (default={local_url}) "
"hosted benchmark url."))
cls.STORY_FILTER_CLS.add_cli_parser(parser)
return parser
@@ -429,6 +442,7 @@
data = super().describe()
assert issubclass(cls.DEFAULT_STORY_CLS, PressBenchmarkStory)
data["url"] = cls.DEFAULT_STORY_CLS.URL
+ data["url-official"] = cls.DEFAULT_STORY_CLS.URL_OFFICIAL
data["url-local"] = cls.DEFAULT_STORY_CLS.URL_LOCAL
return data
diff --git a/crossbench/benchmarks/jetstream/jetstream_2_0.py b/crossbench/benchmarks/jetstream/jetstream_2_0.py
index a224fff..f98287c 100644
--- a/crossbench/benchmarks/jetstream/jetstream_2_0.py
+++ b/crossbench/benchmarks/jetstream/jetstream_2_0.py
@@ -20,7 +20,8 @@
class JetStream20Story(JetStream2Story):
__doc__ = JetStream2Story.__doc__
NAME: str = "jetstream_2.0"
- URL: str = "https://browserbench.org/JetStream2.0/"
+ URL: str = "https://chromium-workloads.web.app/jetstream/v2.0/"
+ URL_OFFICIAL: str = "https://browserbench.org/JetStream2.0/"
class JetStream20Benchmark(JetStream2Benchmark):
diff --git a/crossbench/benchmarks/jetstream/jetstream_2_1.py b/crossbench/benchmarks/jetstream/jetstream_2_1.py
index dd42698..bed8a0d 100644
--- a/crossbench/benchmarks/jetstream/jetstream_2_1.py
+++ b/crossbench/benchmarks/jetstream/jetstream_2_1.py
@@ -20,7 +20,8 @@
class JetStream21Story(JetStream2Story):
__doc__ = JetStream2Story.__doc__
NAME: str = "jetstream_2.1"
- URL: str = "https://browserbench.org/JetStream2.1/"
+ URL: str = "https://chromium-workloads.web.app/jetstream/v2.1/"
+ URL_OFFICIAL: str = "https://browserbench.org/JetStream2.1/"
class JetStream21Benchmark(JetStream2Benchmark):
diff --git a/crossbench/benchmarks/motionmark/__init__.py b/crossbench/benchmarks/motionmark/__init__.py
index b8f1975..16bcede 100644
--- a/crossbench/benchmarks/motionmark/__init__.py
+++ b/crossbench/benchmarks/motionmark/__init__.py
@@ -4,12 +4,17 @@
from __future__ import annotations
+from crossbench.benchmarks.motionmark.motionmark_1_0 import \
+ MotionMark10Benchmark
+from crossbench.benchmarks.motionmark.motionmark_1_1 import \
+ MotionMark11Benchmark
from crossbench.benchmarks.motionmark.motionmark_1_2 import \
MotionMark12Benchmark
from crossbench.benchmarks.motionmark.motionmark_1_3 import \
MotionMark13Benchmark
-benchmark_classes = (MotionMark12Benchmark, MotionMark13Benchmark)
+benchmark_classes = (MotionMark10Benchmark, MotionMark11Benchmark,
+ MotionMark12Benchmark, MotionMark13Benchmark)
_versions = set()
for benchmark_cls in benchmark_classes:
diff --git a/crossbench/benchmarks/motionmark/motionmark_1.py b/crossbench/benchmarks/motionmark/motionmark_1.py
index d692f7a..1814cc3 100644
--- a/crossbench/benchmarks/motionmark/motionmark_1.py
+++ b/crossbench/benchmarks/motionmark/motionmark_1.py
@@ -62,7 +62,7 @@
class MotionMark1Story(PressBenchmarkStory):
- URL_LOCAL = "http://localhost:8000/"
+ URL_LOCAL: str = "http://localhost:8000/"
ALL_STORIES = {
"MotionMark": (
"Multiply",
diff --git a/crossbench/benchmarks/motionmark/motionmark_1_0.py b/crossbench/benchmarks/motionmark/motionmark_1_0.py
new file mode 100644
index 0000000..e9a1b1a
--- /dev/null
+++ b/crossbench/benchmarks/motionmark/motionmark_1_0.py
@@ -0,0 +1,38 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import annotations
+
+from typing import Tuple
+
+from crossbench.benchmarks.motionmark.motionmark_1 import (MotionMark1Benchmark,
+ MotionMark1Probe,
+ MotionMark1Story)
+
+
+class MotionMark10Probe(MotionMark1Probe):
+ __doc__ = MotionMark1Probe.__doc__
+ NAME = "motionmark_1.0"
+
+
+class MotionMark10Story(MotionMark1Story):
+ NAME = "motionmark_1.0"
+ URL: str = "https://chromium-workloads.web.app/motionmark/v1.0/"
+ URL_OFFICIAL: str = "https://browserbench.org/MotionMark1.0/"
+
+
+class MotionMark10Benchmark(MotionMark1Benchmark):
+ """
+ Benchmark runner for MotionMark 1.0.
+
+ See https://browserbench.org/MotionMark1.0/ for more details.
+ """
+
+ NAME = "motionmark_1.0"
+ DEFAULT_STORY_CLS = MotionMark10Story
+ PROBES = (MotionMark10Probe,)
+
+ @classmethod
+ def version(cls) -> Tuple[int, ...]:
+ return (1, 0)
diff --git a/crossbench/benchmarks/motionmark/motionmark_1_1.py b/crossbench/benchmarks/motionmark/motionmark_1_1.py
new file mode 100644
index 0000000..f997833
--- /dev/null
+++ b/crossbench/benchmarks/motionmark/motionmark_1_1.py
@@ -0,0 +1,38 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import annotations
+
+from typing import Tuple
+
+from crossbench.benchmarks.motionmark.motionmark_1 import (MotionMark1Benchmark,
+ MotionMark1Probe,
+ MotionMark1Story)
+
+
+class MotionMark11Probe(MotionMark1Probe):
+ __doc__ = MotionMark1Probe.__doc__
+ NAME = "motionmark_1.1"
+
+
+class MotionMark11Story(MotionMark1Story):
+ NAME = "motionmark_1.1"
+ URL: str = "https://chromium-workloads.web.app/motionmark/v1.1/"
+ URL_OFFICIAL: str = "https://browserbench.org/MotionMark1.1/"
+
+
+class MotionMark11Benchmark(MotionMark1Benchmark):
+ """
+ Benchmark runner for MotionMark 1.1.
+
+ See https://browserbench.org/MotionMark1.1/ for more details.
+ """
+
+ NAME = "motionmark_1.1"
+ DEFAULT_STORY_CLS = MotionMark11Story
+ PROBES = (MotionMark11Probe,)
+
+ @classmethod
+ def version(cls) -> Tuple[int, ...]:
+ return (1, 1)
diff --git a/crossbench/benchmarks/motionmark/motionmark_1_2.py b/crossbench/benchmarks/motionmark/motionmark_1_2.py
index c782f5b..2623dbe 100644
--- a/crossbench/benchmarks/motionmark/motionmark_1_2.py
+++ b/crossbench/benchmarks/motionmark/motionmark_1_2.py
@@ -18,7 +18,8 @@
class MotionMark12Story(MotionMark1Story):
NAME = "motionmark_1.2"
- URL = "https://browserbench.org/MotionMark1.2"
+ URL: str = "https://chromium-workloads.web.app/motionmark/v1.2/"
+ URL_OFFICIAL: str = "https://browserbench.org/MotionMark1.2/"
class MotionMark12Benchmark(MotionMark1Benchmark):
diff --git a/crossbench/benchmarks/motionmark/motionmark_1_3.py b/crossbench/benchmarks/motionmark/motionmark_1_3.py
index b6d2505..9632d5f 100644
--- a/crossbench/benchmarks/motionmark/motionmark_1_3.py
+++ b/crossbench/benchmarks/motionmark/motionmark_1_3.py
@@ -18,7 +18,8 @@
class MotionMark13Story(MotionMark1Story):
NAME = "motionmark_1.3"
- URL = "https://browserbench.org/MotionMark1.3"
+ URL: str = "https://chromium-workloads.web.app/motionmark/v1.3/"
+ URL_OFFICIAL: str = "https://browserbench.org/MotionMark1.3/"
DEVELOPER_READY_JS: str = (
"return !(document.querySelector('#frame-rate-detection span'));")
READY_JS: str = (
diff --git a/crossbench/benchmarks/speedometer/speedometer_2_0.py b/crossbench/benchmarks/speedometer/speedometer_2_0.py
index 29a240c..1f29b89 100644
--- a/crossbench/benchmarks/speedometer/speedometer_2_0.py
+++ b/crossbench/benchmarks/speedometer/speedometer_2_0.py
@@ -18,7 +18,8 @@
class Speedometer20Story(Speedometer2Story):
NAME: str = "speedometer_2.0"
- URL: str = "https://browserbench.org/Speedometer2.0"
+ URL: str = "https://chromium-workloads.web.app/speedometer/v2.0/"
+ URL_OFFICIAL: str = "https://browserbench.org/Speedometer2.0/"
class Speedometer20Benchmark(SpeedometerBenchmark):
diff --git a/crossbench/benchmarks/speedometer/speedometer_2_1.py b/crossbench/benchmarks/speedometer/speedometer_2_1.py
index d54898d..da709a5 100644
--- a/crossbench/benchmarks/speedometer/speedometer_2_1.py
+++ b/crossbench/benchmarks/speedometer/speedometer_2_1.py
@@ -18,7 +18,8 @@
class Speedometer21Story(Speedometer2Story):
NAME: str = "speedometer_2.1"
- URL: str = "https://browserbench.org/Speedometer2.1/"
+ URL: str = "https://chromium-workloads.web.app/speedometer/v2.1/"
+ URL_OFFICIAL: str = "https://browserbench.org/Speedometer2.1/"
class Speedometer21Benchmark(SpeedometerBenchmark):
diff --git a/crossbench/benchmarks/speedometer/speedometer_3_0.py b/crossbench/benchmarks/speedometer/speedometer_3_0.py
index c75f8fe..502d1b9 100644
--- a/crossbench/benchmarks/speedometer/speedometer_3_0.py
+++ b/crossbench/benchmarks/speedometer/speedometer_3_0.py
@@ -223,8 +223,8 @@
class Speedometer30Story(SpeedometerStory):
__doc__ = SpeedometerStory.__doc__
NAME: str = "speedometer_3.0"
- # TODO: Update once public version is available
- URL: str = "https://sp3-alpha-testing.netlify.app/"
+ URL: str = "https://chromium-workloads.web.app/speedometer/v3.0/"
+ URL_OFFICIAL: str = "https://browserbench.org/Speedometer3.0/"
URL_LOCAL: str = "http://127.0.0.1:7000"
SUBSTORIES: Tuple[str, ...] = tuple(SPEEDOMETER_3_STORY_DATA.keys())
diff --git a/crossbench/cli/cli.py b/crossbench/cli/cli.py
index 92da4e6..6ca4392 100644
--- a/crossbench/cli/cli.py
+++ b/crossbench/cli/cli.py
@@ -97,8 +97,10 @@
benchmarks.Speedometer20Benchmark,
benchmarks.JetStream21Benchmark,
benchmarks.JetStream20Benchmark,
- benchmarks.MotionMark12Benchmark,
benchmarks.MotionMark13Benchmark,
+ benchmarks.MotionMark12Benchmark,
+ benchmarks.MotionMark11Benchmark,
+ benchmarks.MotionMark10Benchmark,
benchmarks.PageLoadBenchmark,
benchmarks.PowerBenchmark,
benchmarks.ManualBenchmark,
diff --git a/crossbench/runner/run.py b/crossbench/runner/run.py
index c782d1a..f3e5c86 100644
--- a/crossbench/runner/run.py
+++ b/crossbench/runner/run.py
@@ -222,6 +222,7 @@
self._cool_down(is_dry_run)
with ChangeCWD(self._out_dir), self.exception_info(*self.info_stack):
self._probe_context_manager.setup(self.probes, is_dry_run)
+ self._log_setup()
def setup_selenium_options(self, options: ArgOptions):
# TODO: move explicitly to session.
diff --git a/crossbench/stories/press_benchmark.py b/crossbench/stories/press_benchmark.py
index c1cdcb7..b7aa011 100644
--- a/crossbench/stories/press_benchmark.py
+++ b/crossbench/stories/press_benchmark.py
@@ -19,6 +19,7 @@
class PressBenchmarkStory(Story, metaclass=abc.ABCMeta):
NAME: str = ""
URL: str = ""
+ URL_OFFICIAL: str = ""
URL_LOCAL: str = ""
SUBSTORIES: Tuple[str, ...] = ()
@@ -82,6 +83,7 @@
assert self.SUBSTORIES, f"{cls}.SUBSTORIES is not set."
assert self.NAME is not None, f"{cls}.NAME is not set."
self._verify_url(self.URL, "URL")
+ self._verify_url(self.URL_OFFICIAL, "URL_OFFICIAL")
self._verify_url(self.URL_LOCAL, "URL_LOCAL")
assert substories, f"No substories provided for {cls}"
self._substories = substories
