Avoid TestCase mixins that break pytype type inference
- Move abstract TestCase classe to tests/benchmarks/helper.py
- Add more type annotations to mockbenchmark.py
Change-Id: Iaea227afdc1be74d720697bf3c3f95bfb9d3577e
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/3963549
Reviewed-by: Victor Gomes <victorgomes@chromium.org>
diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py
index 736eb17..005a4a1 100644
--- a/tests/benchmarks/__init__.py
+++ b/tests/benchmarks/__init__.py
@@ -1,75 +1,3 @@
# Copyright 2022 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-
-import abc
-
-from .. import mockbenchmark
-
-import crossbench as cb
-
-
-class BaseBenchmarkTestCase(
- mockbenchmark.BaseCrossbenchTestCase, metaclass=abc.ABCMeta):
-
- @property
- @abc.abstractmethod
- def benchmark_cls(self):
- pass
-
- @property
- def story_cls(self):
- return self.benchmark_cls.DEFAULT_STORY_CLS
-
- def setUp(self):
- super().setUp()
- self.assertTrue(
- issubclass(self.benchmark_cls, cb.benchmarks.Benchmark),
- f"Expected Benchmark subclass, but got: BENCHMARK={self.benchmark_cls}")
-
-
-class BenchmarkTestCaseMixin:
-
- def test_stories_creation(self):
- for name in self.story_cls.story_names():
- stories = self.story_cls.from_names([name])
- self.assertTrue(len(stories) == 1)
- story = stories[0]
- self.assertIsInstance(story, self.story_cls)
- self.assertIsInstance(story.details_json(), dict)
- self.assertTrue(len(str(story)) > 0)
-
- def test_instantiate_no_stories(self):
- with self.assertRaises(AssertionError):
- self.benchmark_cls(stories=[])
- with self.assertRaises(AssertionError):
- self.benchmark_cls(stories="")
- with self.assertRaises(AssertionError):
- self.benchmark_cls(stories=["", ""])
-
- def test_instantiate_single_story(self):
- any_story_name = self.story_cls.story_names()[0]
- any_story = self.story_cls.from_names([any_story_name])[0]
- # Instantiate with single story,
- with self.assertRaises(TypeError):
- self.benchmark_cls(any_story)
- # with single story array
- self.benchmark_cls([any_story])
- with self.assertRaises(AssertionError):
- # Accidentally nested array.
- self.benchmark_cls([[any_story]])
-
- def test_instantiate_all_stories(self):
- stories = self.story_cls.from_names(self.story_cls.story_names())
- self.benchmark_cls(stories)
-
- def test_describe(self):
- self.assertIsInstance(self.benchmark_cls.describe(), dict)
-
-
-class PressBenchmarkTestCaseMixin(BenchmarkTestCaseMixin):
-
- def test_invalid_story_names(self):
- with self.assertRaises(Exception):
- # Only one regexp entry will work
- self.story_cls.from_names([".*", 'a'], separate=True)
diff --git a/tests/benchmarks/helper.py b/tests/benchmarks/helper.py
new file mode 100644
index 0000000..33df577
--- /dev/null
+++ b/tests/benchmarks/helper.py
@@ -0,0 +1,88 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import abc
+from typing import Sequence, Type
+
+from .. import mockbenchmark
+
+import crossbench as cb
+
+
+class BaseBenchmarkTestCase(
+ mockbenchmark.BaseCrossbenchTestCase, metaclass=abc.ABCMeta):
+
+ @property
+ @abc.abstractmethod
+ def benchmark_cls(self):
+ pass
+
+ @property
+ def story_cls(self):
+ return self.benchmark_cls.DEFAULT_STORY_CLS
+
+ def setUp(self):
+ super().setUp()
+ self.assertTrue(
+ issubclass(self.benchmark_cls, cb.benchmarks.Benchmark),
+ f"Expected Benchmark subclass, but got: BENCHMARK={self.benchmark_cls}")
+
+ def test_instantiate_no_stories(self):
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories=[])
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories="")
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories=["", ""])
+
+ def test_describe(self):
+ self.assertIsInstance(self.benchmark_cls.describe(), dict)
+
+
+class SubStoryTestCase(BaseBenchmarkTestCase, metaclass=abc.ABCMeta):
+
+ def test_stories_creation(self):
+ for name in self.story_cls.story_names():
+ stories = self.story_cls.from_names([name])
+ self.assertTrue(len(stories) == 1)
+ story = stories[0]
+ self.assertIsInstance(story, self.story_cls)
+ self.assertIsInstance(story.details_json(), dict)
+ self.assertTrue(len(str(story)) > 0)
+
+ def test_instantiate_no_stories(self):
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories=[])
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories="")
+ with self.assertRaises(AssertionError):
+ self.benchmark_cls(stories=["", ""])
+
+ def test_instantiate_single_story(self):
+ any_story_name = self.story_cls.story_names()[0]
+ any_story = self.story_cls.from_names([any_story_name])[0]
+ # Instantiate with single story,
+ with self.assertRaises(Exception):
+ self.benchmark_cls(any_story)
+ # with single story array
+ self.benchmark_cls([any_story])
+ with self.assertRaises(AssertionError):
+ # Accidentally nested array.
+ self.benchmark_cls([[any_story]])
+
+ def test_instantiate_all_stories(self):
+ stories = self.story_cls.from_names(self.story_cls.story_names())
+ self.benchmark_cls(stories)
+
+ def test_describe(self):
+ self.assertIsInstance(self.benchmark_cls.describe(), dict)
+
+
+
+class PressBaseBenchmarkTestCase(SubStoryTestCase, metaclass=abc.ABCMeta):
+
+ def test_invalid_story_names(self):
+ with self.assertRaises(Exception):
+ # Only one regexp entry will work
+ self.story_cls.from_names([".*", 'a'], separate=True)
diff --git a/tests/benchmarks/test_jetstream.py b/tests/benchmarks/test_jetstream.py
index d4af12c..ded3a04 100644
--- a/tests/benchmarks/test_jetstream.py
+++ b/tests/benchmarks/test_jetstream.py
@@ -5,10 +5,10 @@
import crossbench as cb
import crossbench.benchmarks as bm
-from . import BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin
+from . import helper
-class JetStream2Test(BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin):
+class JetStream2Test(helper.PressBaseBenchmarkTestCase):
@property
def benchmark_cls(self):
diff --git a/tests/benchmarks/test_loading.py b/tests/benchmarks/test_loading.py
index d62674c..b9e464a 100644
--- a/tests/benchmarks/test_loading.py
+++ b/tests/benchmarks/test_loading.py
@@ -5,10 +5,10 @@
import crossbench as cb
import crossbench.benchmarks as bm
-from . import BaseBenchmarkTestCase
+from . import helper
-class TestPageLoadBenchmark(BaseBenchmarkTestCase):
+class TestPageLoadBenchmark(helper.SubStoryTestCase):
@property
def benchmark_cls(self):
diff --git a/tests/benchmarks/test_motionmark.py b/tests/benchmarks/test_motionmark.py
index efcbc4e..88800cc 100644
--- a/tests/benchmarks/test_motionmark.py
+++ b/tests/benchmarks/test_motionmark.py
@@ -5,10 +5,10 @@
import crossbench as cb
import crossbench.benchmarks as bm
-from . import BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin
+from . import helper
-class MotionMark2Test(BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin):
+class MotionMark2Test(helper.PressBaseBenchmarkTestCase):
@property
def benchmark_cls(self):
diff --git a/tests/benchmarks/test_speedometer.py b/tests/benchmarks/test_speedometer.py
index 78c4d5d..554a4ea 100644
--- a/tests/benchmarks/test_speedometer.py
+++ b/tests/benchmarks/test_speedometer.py
@@ -5,10 +5,11 @@
import crossbench as cb
import crossbench.benchmarks as bm
-from . import BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin
+from . import helper
-class Speedometer2Test(BaseBenchmarkTestCase, PressBenchmarkTestCaseMixin):
+class Speedometer2Test(helper.PressBaseBenchmarkTestCase):
+
@property
def benchmark_cls(self):
diff --git a/tests/mockbenchmark.py b/tests/mockbenchmark.py
index 583d673..7190d94 100644
--- a/tests/mockbenchmark.py
+++ b/tests/mockbenchmark.py
@@ -47,7 +47,7 @@
class MockBrowser(cb.browsers.Browser):
- BIN_PATH = None
+ BIN_PATH: pathlib.Path = pathlib.Path("/")
VERSION = "100.22.33.44"
@classmethod
@@ -69,11 +69,11 @@
super().__init__(label, path, *args, **kwargs)
self.url_list: List[str] = []
self.js_list: List[str] = []
- self.js_side_effect = []
- self.run_js_side_effect = []
- self.did_run = False
- self.clear_cache_dir = False
- self.js_flags = cb.flags.JSFlags()
+ self.js_side_effect: List[str] = []
+ self.run_js_side_effect: List[str] = []
+ self.did_run: bool = False
+ self.clear_cache_dir: bool = False
+ self.js_flags: cb.flags.JSFlags = cb.flags.JSFlags()
def clear_cache(self, runner: cb.runner.Runner):
pass
