Add basic speedometer2 and motionnmark unittests

- Rename Speedometer20Runner to Speedometer20Benchmark
- Add some more types
- Fix import errors when running tests under vscode

Change-Id: I7dee54af13b9e0167a7862ce5fe6aa1c93099d17
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/3909528
Reviewed-by: Alexander Schulze <alexschulze@chromium.org>
diff --git a/crossbench/benchmarks/__init__.py b/crossbench/benchmarks/__init__.py
index 1d1e32f..afcdbf5 100644
--- a/crossbench/benchmarks/__init__.py
+++ b/crossbench/benchmarks/__init__.py
@@ -6,4 +6,4 @@
 from crossbench.benchmarks.jetstream import JetStream2Benchmark
 from crossbench.benchmarks.loading import PageLoadBenchmark
 from crossbench.benchmarks.motionmark import MotionMark12Benchmark
-from crossbench.benchmarks.speedometer import Speedometer20Runner
+from crossbench.benchmarks.speedometer import Speedometer20Benchmark
diff --git a/crossbench/benchmarks/speedometer.py b/crossbench/benchmarks/speedometer.py
index d595090..44abe6b 100644
--- a/crossbench/benchmarks/speedometer.py
+++ b/crossbench/benchmarks/speedometer.py
@@ -148,7 +148,7 @@
                                2 * len(self._substories) * self.iterations))
 
 
-class Speedometer20Runner(benchmarks.PressBenchmark):
+class Speedometer20Benchmark(benchmarks.PressBenchmark):
   """
   Benchmark runner for Speedometer 2.0
   """
diff --git a/crossbench/cli.py b/crossbench/cli.py
index 4315891..45e5f89 100644
--- a/crossbench/cli.py
+++ b/crossbench/cli.py
@@ -223,7 +223,7 @@
 class CrossBenchCLI:
 
   BENCHMARKS = (
-      cb.benchmarks.Speedometer20Runner,
+      cb.benchmarks.Speedometer20Benchmark,
       cb.benchmarks.JetStream2Benchmark,
       cb.benchmarks.MotionMark12Benchmark,
       cb.benchmarks.PageLoadBenchmark,
diff --git a/crossbench/probes/json.py b/crossbench/probes/json.py
index 770de60..4a6afcb 100644
--- a/crossbench/probes/json.py
+++ b/crossbench/probes/json.py
@@ -30,7 +30,7 @@
   FLATTEN = True
 
   @property
-  def results_file_name(self):
+  def results_file_name(self) -> str:
     return f"{self.name}.json"
 
   @abc.abstractmethod
@@ -144,14 +144,14 @@
     return sum(self.values) / len(self.values)
 
   @property
-  def geomean(self):
+  def geomean(self) -> float:
     product = 1
     for value in self.values:
       product *= value
     return product**(1 / len(self.values))
 
   @property
-  def stddev(self):
+  def stddev(self) -> float:
     """
     We're ignoring here any actual distribution of the data and use this as a
     rough estimate of the quality of the data
diff --git a/crossbench/stories.py b/crossbench/stories.py
index e482e22..1095137 100644
--- a/crossbench/stories.py
+++ b/crossbench/stories.py
@@ -7,7 +7,7 @@
 import logging
 import re
 from abc import ABC, ABCMeta, abstractmethod
-from typing import Tuple
+from typing import List, Optional, Sequence, Tuple, Union
 
 
 class Story(ABC):
@@ -15,7 +15,7 @@
 
   @classmethod
   @abstractmethod
-  def story_names(cls):
+  def story_names(cls) -> Sequence[str]:
     pass
 
   @classmethod
@@ -23,7 +23,7 @@
   def from_names(cls, names, separate=False):
     pass
 
-  def __init__(self, name: str, duration=15):
+  def __init__(self, name: str, duration: float = 15):
     assert name, "Invalid page name"
     self._name = name
     assert duration > 0, (
@@ -31,13 +31,13 @@
     self.duration = duration
 
   @property
-  def name(self):
+  def name(self) -> str:
     return self._name
 
   def details_json(self):
     return dict(name=self.name, duration=self.duration)
 
-  def is_done(self, _):
+  def is_done(self, _) -> bool:
     return True
 
   @abstractmethod
@@ -59,9 +59,12 @@
     return cls.SUBSTORIES
 
   @classmethod
-  def from_names(cls, names, separate=False, live=True):
-    if len(names) == 1:
-      first = names[0]
+  def from_names(cls, names:Union[Sequence[str], str], separate=False, live=True):
+    if len(names) == 1 or isinstance(names, str):
+      if isinstance(names, str):
+        first = names
+      else:
+        first = names[0]
       if first == "all":
         names = cls.SUBSTORIES
       elif first not in cls.SUBSTORIES:
@@ -97,13 +100,21 @@
     ]
 
   @classmethod
-  def get_substories(cls, separate, substories):
+  def get_substories(cls, separate:bool, substories:Sequence[str]):
     substories = substories or cls.SUBSTORIES
     if separate:
       return substories
     return [substories]
 
-  def __init__(self, *args, is_live=True, substories=None, **kwargs):
+
+  _substories : List[str]
+  is_live : bool
+
+  def __init__(self,
+               *args,
+               is_live: bool = True,
+               substories: Optional[Union[str, List[str]]] = None,
+               **kwargs):
     cls = self.__class__
     assert self.SUBSTORIES, f"{cls}.SUBSTORIES is not set."
     assert self.NAME is not None, f"{cls}.NAME is not set."
@@ -125,7 +136,7 @@
       self._url = self.URL_LOCAL
     assert self._url is not None, f"Invalid URL for {self.NAME}"
 
-  def _verify_url(self, url, property_name):
+  def _verify_url(self, url:str, property_name:str):
     cls = self.__class__
     assert url is not None, f"{cls}.{property_name} is not set."
 
diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py
index ec02634..ef34d3e 100644
--- a/tests/test_benchmarks.py
+++ b/tests/test_benchmarks.py
@@ -6,13 +6,16 @@
 import pathlib
 import pyfakefs.fake_filesystem_unittest
 
-from . import mockbenchmark
+try:
+  from . import mockbenchmark
+except ImportError:
+  # VSCode has issues discovering tests code
+  from tests import mockbenchmark
 
 import crossbench as cb
 import crossbench.benchmarks as bm
 
 
-
 class BaseRunnerTest(
     pyfakefs.fake_filesystem_unittest.TestCase, metaclass=abc.ABCMeta):
 
@@ -93,19 +96,110 @@
     self.assertTrue(self.browsers[1].did_run)
 
 
+class Speedometer2Test(BaseRunnerTest):
+  BENCHMARK = bm.speedometer.Speedometer20Benchmark
+
+  def test_story_filtering(self):
+    stories = bm.speedometer.Speedometer20Story.from_names([])
+    self.assertEqual(len(stories), 1)
+    stories = bm.speedometer.Speedometer20Story.from_names([], separate=True)
+    self.assertEqual(len(stories),
+                     len(bm.speedometer.Speedometer20Story.SUBSTORIES))
+    stories_b = bm.speedometer.Speedometer20Story.from_names(".*",
+                                                             separate=True)
+    self.assertListEqual(
+        [story.name for story in stories],
+        [story.name for story in stories_b],
+    )
+    stories_c = bm.speedometer.Speedometer20Story.from_names([".*"],
+                                                             separate=True)
+    self.assertListEqual(
+        [story.name for story in stories],
+        [story.name for story in stories_c],
+    )
+
+  def test_invalid_story_names(self):
+    with self.assertRaises(Exception):
+      # Only one regexp entry will work
+      bm.speedometer.Speedometer20Story.from_names([".*", 'a'], separate=True)
+
+
+  def test_run(self):
+    repetitions = 3
+    iterations = 2
+    stories = bm.speedometer.Speedometer20Story.from_names(['VanillaJS-TodoMVC'])
+    example_story_data = {
+        "tests": {
+            "Adding100Items": {
+                "tests": {
+                    "Sync": 74.6000000089407,
+                    "Async": 6.299999997019768
+                },
+                "total": 80.90000000596046
+            },
+            "CompletingAllItems": {
+                "tests": {
+                    "Sync": 22.600000008940697,
+                    "Async": 5.899999991059303
+                },
+                "total": 28.5
+            },
+            "DeletingItems": {
+                "tests": {
+                    "Sync": 11.800000011920929,
+                    "Async": 0.19999998807907104
+                },
+                "total": 12
+            }
+        },
+        "total": 121.40000000596046
+    }
+    speedometer_probe_results = [{
+        "tests": {story.name: example_story_data
+                  for story in stories},
+        "total": 1000,
+        "mean": 2000,
+        "geomean": 3000,
+        "score": 10
+    } for i in range(iterations)]
+
+    for browser in self.browsers:
+      browser.js_side_effect = [
+          True,  # Page is ready
+          None,  # filter benchmarks
+          None,  # Start running benchmark
+          False, # Wait,  ...
+          True,  # until done
+          speedometer_probe_results,
+      ]
+    benchmark = self.BENCHMARK(stories)
+    self.assertTrue(len(benchmark.describe()) > 0)
+    runner = cb.runner.Runner(
+        self.out_dir,
+        self.browsers,
+        benchmark,
+        use_checklist=False,
+        platform=self.platform,
+        repetitions=repetitions)
+    runner.run()
+    for browser in self.browsers:
+      self.assertEqual(len(browser.url_list), repetitions)
+      self.assertIn(bm.speedometer.Speedometer20Probe.JS, browser.js_list)
+
+
 class JetStream2Test(BaseRunnerTest):
   BENCHMARK = bm.jetstream.JetStream2Benchmark
 
   def test_run(self):
     stories = bm.jetstream.JetStream2Story.from_names(['WSL'])
-    example_story_data = {'firstItertaion': 1, 'average': 0.1, 'worst4': 1.1}
+    example_story_data = {'firstIteration': 1, 'average': 0.1, 'worst4': 1.1}
     jetstream_probe_results = {
         story.name: example_story_data for story in stories
     }
     for browser in self.browsers:
       browser.js_side_effect = [
           True,  # Page is ready
-          None,  # filter benchmnarks
+          None,  # filter benchmarks
           True,  # UI is updated and ready,
           None,  # Start running benchmark
           True,  # Wait until done
@@ -125,3 +219,6 @@
     for browser in self.browsers:
       self.assertEqual(len(browser.url_list), repetitions)
       self.assertIn(bm.jetstream.JetStream2Probe.JS, browser.js_list)
+
+class MotionMark2Test(BaseRunnerTest):
+  BENCHMARK = bm.motionmark.MotionMark12Benchmark
diff --git a/tests/test_cli.py b/tests/test_cli.py
index d4236e5..eb1d41f 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -10,7 +10,11 @@
 
 import pyfakefs.fake_filesystem_unittest
 
-from . import mockbenchmark
+try:
+  from . import mockbenchmark
+except ImportError:
+  # VSCode has issues discovering tests code
+  from tests import mockbenchmark
 
 import crossbench as cb
 from crossbench.cli import BrowserConfig, CrossBenchCLI, FlagGroupConfig