Refactor action runner - ActionRunner now contains a Run member variable - Refactor all actions and dispatch methods to use self.run Drive-by-fix: - Clean up tests a bit and move mocks to separate module TAG=agy CONV=db885144-5312-4d76-9b74-c4bd54119c6b Change-Id: I68185f262bc559cb1ca26e82d2b53942494aae8f Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/7885454 Commit-Queue: Camillo Bruni <cbruni@chromium.org> Reviewed-by: Patrick Thier <pthier@chromium.org>
diff --git a/crossbench/action_runner/action/action.py b/crossbench/action_runner/action/action.py index a8e0a46..270da24 100644 --- a/crossbench/action_runner/action/action.py +++ b/crossbench/action_runner/action/action.py
@@ -21,7 +21,6 @@ import urllib.parse as urlparse from crossbench.action_runner.base import ActionRunner - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -121,7 +120,7 @@ return self._timeout != dt.timedelta.max @abc.abstractmethod - def run_with(self, run: Run, action_runner: ActionRunner) -> None: + def run_with(self, action_runner: ActionRunner) -> None: pass @override
diff --git a/crossbench/action_runner/action/base_probe.py b/crossbench/action_runner/action/base_probe.py index 617c43f..fa3d070 100644 --- a/crossbench/action_runner/action/base_probe.py +++ b/crossbench/action_runner/action/base_probe.py
@@ -21,7 +21,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser from crossbench.probes.probe import Probe - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -52,8 +51,8 @@ return self._kwargs @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.invoke_probe(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.invoke_probe(self) def kwargs_to_json(self) -> JsonDict: return dict(self.kwargs)
diff --git a/crossbench/action_runner/action/clear_cache.py b/crossbench/action_runner/action/clear_cache.py index 59e2a36..a8f2b83 100644 --- a/crossbench/action_runner/action/clear_cache.py +++ b/crossbench/action_runner/action/clear_cache.py
@@ -13,12 +13,11 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner - from crossbench.runner.run import Run class ClearCacheAction(Action): TYPE: ClassVar[ActionType] = ActionType.CLEAR_CACHE @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.clear_cache(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.clear_cache(self)
diff --git a/crossbench/action_runner/action/click.py b/crossbench/action_runner/action/click.py index e9cab08..487de93 100644 --- a/crossbench/action_runner/action/click.py +++ b/crossbench/action_runner/action/click.py
@@ -21,7 +21,6 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -79,8 +78,8 @@ return self._verify @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.click(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.click(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/close_all_tabs.py b/crossbench/action_runner/action/close_all_tabs.py index 7af522b..63fd114 100644 --- a/crossbench/action_runner/action/close_all_tabs.py +++ b/crossbench/action_runner/action/close_all_tabs.py
@@ -13,12 +13,11 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner - from crossbench.runner.run import Run class CloseAllTabsAction(Action): TYPE: ClassVar[ActionType] = ActionType.CLOSE_ALL_TABS @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.close_all_tabs(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.close_all_tabs(self)
diff --git a/crossbench/action_runner/action/close_tab.py b/crossbench/action_runner/action/close_tab.py index 5fff8c9..a2f51f1 100644 --- a/crossbench/action_runner/action/close_tab.py +++ b/crossbench/action_runner/action/close_tab.py
@@ -16,7 +16,6 @@ from crossbench.action_runner.action.action import ActionT from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run class CloseTabAction(BaseTabAction): @@ -30,5 +29,5 @@ return parser @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.close_tab(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.close_tab(self)
diff --git a/crossbench/action_runner/action/get.py b/crossbench/action_runner/action/get.py index 2fb4e36..f4097be 100644 --- a/crossbench/action_runner/action/get.py +++ b/crossbench/action_runner/action/get.py
@@ -21,7 +21,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -96,8 +95,8 @@ return self._target @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.get(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.get(self) @override def to_json(self) -> JsonDict:
diff --git a/crossbench/action_runner/action/inject_new_document_script.py b/crossbench/action_runner/action/inject_new_document_script.py index 179592a..198deec 100644 --- a/crossbench/action_runner/action/inject_new_document_script.py +++ b/crossbench/action_runner/action/inject_new_document_script.py
@@ -13,12 +13,11 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner - from crossbench.runner.run import Run class InjectNewDocumentScriptAction(JsAction): TYPE: ClassVar[ActionType] = ActionType.INJECT_NEW_DOCUMENT_SCRIPT @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.inject_new_document_script(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.inject_new_document_script(self)
diff --git a/crossbench/action_runner/action/js.py b/crossbench/action_runner/action/js.py index 2936539..641fa4f 100644 --- a/crossbench/action_runner/action/js.py +++ b/crossbench/action_runner/action/js.py
@@ -22,7 +22,6 @@ from crossbench import path as pth from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -69,8 +68,8 @@ return self._final_script @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.js(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.js(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/meet_create.py b/crossbench/action_runner/action/meet_create.py index 219abaf..ffad768 100644 --- a/crossbench/action_runner/action/meet_create.py +++ b/crossbench/action_runner/action/meet_create.py
@@ -20,7 +20,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run class MeetCreateAction(BondAction): @@ -54,5 +53,5 @@ return self._target @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.bond.meet_create(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.bond.meet_create(self)
diff --git a/crossbench/action_runner/action/meet_script.py b/crossbench/action_runner/action/meet_script.py index 698a3da..1863b78 100644 --- a/crossbench/action_runner/action/meet_script.py +++ b/crossbench/action_runner/action/meet_script.py
@@ -19,7 +19,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run # This action is different from the `JsAction` in that it is not executed on the @@ -49,5 +48,6 @@ def script(self) -> str: return self._script - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.bond.meet_script(run, self) + @override + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.bond.meet_script(self)
diff --git a/crossbench/action_runner/action/open_devtools.py b/crossbench/action_runner/action/open_devtools.py index 622caa5..72bd8ea 100644 --- a/crossbench/action_runner/action/open_devtools.py +++ b/crossbench/action_runner/action/open_devtools.py
@@ -13,7 +13,6 @@ from crossbench.action_runner.action.action_type import ActionType from crossbench.action_runner.action.base_tab_action import BaseTabAction from crossbench.parse import ObjectParser -from crossbench.runner.run import Run if TYPE_CHECKING: import datetime as dt @@ -22,7 +21,6 @@ from crossbench.action_runner.action.action import ActionT from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run class OpenDevToolsAction(BaseTabAction): @@ -52,5 +50,5 @@ return parser @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.open_devtools(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.open_devtools(self)
diff --git a/crossbench/action_runner/action/scroll.py b/crossbench/action_runner/action/scroll.py index 40cf2b2..d475896 100644 --- a/crossbench/action_runner/action/scroll.py +++ b/crossbench/action_runner/action/scroll.py
@@ -19,7 +19,6 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -68,8 +67,8 @@ return self._required @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.scroll(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.scroll(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/swipe.py b/crossbench/action_runner/action/swipe.py index c134f4c..5d7abc5 100644 --- a/crossbench/action_runner/action/swipe.py +++ b/crossbench/action_runner/action/swipe.py
@@ -18,7 +18,6 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -77,8 +76,8 @@ return self._end_y @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.swipe(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.swipe(self) @override def to_json(self) -> JsonDict:
diff --git a/crossbench/action_runner/action/switch_tab.py b/crossbench/action_runner/action/switch_tab.py index 5b1aaac..04e5643 100644 --- a/crossbench/action_runner/action/switch_tab.py +++ b/crossbench/action_runner/action/switch_tab.py
@@ -16,7 +16,6 @@ from crossbench.action_runner.action.action import ActionT from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run class SwitchTabAction(BaseTabAction): @@ -30,8 +29,8 @@ return parser @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.switch_tab(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.switch_tab(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/text_input.py b/crossbench/action_runner/action/text_input.py index d6f0ed4..d61b402 100644 --- a/crossbench/action_runner/action/text_input.py +++ b/crossbench/action_runner/action/text_input.py
@@ -19,7 +19,6 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -64,8 +63,8 @@ return self._keyevent @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.text_input(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.text_input(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/wait.py b/crossbench/action_runner/action/wait.py index 62a6b76..73a4652 100644 --- a/crossbench/action_runner/action/wait.py +++ b/crossbench/action_runner/action/wait.py
@@ -13,12 +13,11 @@ if TYPE_CHECKING: from crossbench.action_runner.base import ActionRunner - from crossbench.runner.run import Run class WaitAction(DurationAction): TYPE: ClassVar[ActionType] = ActionType.WAIT @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.wait(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.wait(self)
diff --git a/crossbench/action_runner/action/wait_for_condition.py b/crossbench/action_runner/action/wait_for_condition.py index 15db72f..0aa09e1 100644 --- a/crossbench/action_runner/action/wait_for_condition.py +++ b/crossbench/action_runner/action/wait_for_condition.py
@@ -19,7 +19,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -47,8 +46,8 @@ return self._condition @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.wait_for_condition(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.wait_for_condition(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/wait_for_element.py b/crossbench/action_runner/action/wait_for_element.py index 95ad705..c38a7d3 100644 --- a/crossbench/action_runner/action/wait_for_element.py +++ b/crossbench/action_runner/action/wait_for_element.py
@@ -19,7 +19,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -72,8 +71,8 @@ return self._check_rect @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.wait_for_element(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.wait_for_element(self) @override def validate(self) -> None:
diff --git a/crossbench/action_runner/action/wait_for_ready_state.py b/crossbench/action_runner/action/wait_for_ready_state.py index 4c92fae..5049e6a 100644 --- a/crossbench/action_runner/action/wait_for_ready_state.py +++ b/crossbench/action_runner/action/wait_for_ready_state.py
@@ -19,7 +19,6 @@ from crossbench.action_runner.base import ActionRunner from crossbench.config import ConfigParser - from crossbench.runner.run import Run from crossbench.types import JsonDict @@ -47,8 +46,8 @@ return self._ready_state @override - def run_with(self, run: Run, action_runner: ActionRunner) -> None: - action_runner.wait_for_ready_state(run, self) + def run_with(self, action_runner: ActionRunner) -> None: + action_runner.wait_for_ready_state(self) @override def to_json(self) -> JsonDict:
diff --git a/crossbench/action_runner/android_input_action_runner.py b/crossbench/action_runner/android_input_action_runner.py index 1cac18f..f440864 100644 --- a/crossbench/action_runner/android_input_action_runner.py +++ b/crossbench/action_runner/android_input_action_runner.py
@@ -24,7 +24,6 @@ from crossbench.browsers.attributes import BrowserAttributes from crossbench.plt.android_adb import AndroidAdbPlatform from crossbench.runner.actions import Actions - from crossbench.runner.run import Run class ViewportInfo: @@ -108,10 +107,10 @@ rect.height ];""" - def scroll_touch(self, run: Run, action: i_action.ScrollAction) -> None: - with run.actions("ScrollAction", measure=False) as actions: + def scroll_touch(self, action: i_action.ScrollAction) -> None: + with self.actions("ScrollAction", measure=False) as actions: - viewport_info = self._get_viewport_info(run, actions, action.selector) + viewport_info = self._get_viewport_info(actions, action.selector) # The scroll distance is specified in terms of css pixels so adjust to the # native pixel density. @@ -152,37 +151,35 @@ y_start = scrollable_top y_end = scrollable_top + current_distance - self._swipe_impl(run, round(scroll_area.mid_x), round(y_start), - round(scroll_area.mid_x), round(y_end), - current_duration) + self._swipe_impl( + round(scroll_area.mid_x), round(y_start), round(scroll_area.mid_x), + round(y_end), current_duration) remaining_distance -= current_distance - def click_touch(self, run: Run, action: i_action.ClickAction) -> None: - self._click_impl(run, action, False) + def click_touch(self, action: i_action.ClickAction) -> None: + self._click_impl(action, False) - def click_mouse(self, run: Run, action: i_action.ClickAction) -> None: - self._click_impl(run, action, True) + def click_mouse(self, action: i_action.ClickAction) -> None: + self._click_impl(action, True) - def swipe(self, run: Run, action: i_action.SwipeAction) -> None: - with run.actions("SwipeAction", measure=False): - self._swipe_impl(run, action.start_x, action.start_y, action.end_x, + def swipe(self, action: i_action.SwipeAction) -> None: + with self.actions("SwipeAction", measure=False): + self._swipe_impl(action.start_x, action.start_y, action.end_x, action.end_y, action.duration) - def text_input_keyboard(self, run: Run, - action: i_action.TextInputAction) -> None: + def text_input_keyboard(self, action: i_action.TextInputAction) -> None: if action.text: - self._rate_limit_keystrokes(run, action, self._type_characters) + self._rate_limit_keystrokes(action, self._type_characters) elif keyevent := action.keyevent: - self._send_keyevent(run, keyevent) + self._send_keyevent(keyevent) - def _click_impl(self, run: Run, action: i_action.ClickAction, - use_mouse: bool) -> None: + def _click_impl(self, action: i_action.ClickAction, use_mouse: bool) -> None: if action.duration > dt.timedelta(): raise InputSourceNotImplementedError(self, action, action.input_source, "Non-zero duration not implemented") coordinates: Point | None = None - with run.actions("ClickAction", measure=False) as actions: + with self.actions("ClickAction", measure=False) as actions: if coordinates_config := action.position.coordinates: coordinates = coordinates_config.point() @@ -191,7 +188,7 @@ raise InputSourceNotImplementedError( self, action, action.input_source, "Mouse actions not implemented for UiSelectorConfig") - self._click_ui_selector(run, ui_selector, action.timeout) + self._click_ui_selector(ui_selector, action.timeout) elif selector_config := action.position.selector: if selector_config.wait: self.wait_for_element_impl( @@ -203,8 +200,7 @@ required=selector_config.required) viewport_info = self._get_viewport_info( - run, actions, selector_config.selector, - selector_config.scroll_into_view) + actions, selector_config.selector, selector_config.scroll_into_view) rect = viewport_info.element_rect() if not rect: @@ -231,7 +227,7 @@ ScreenshotPointAnnotation(label="click", point=coordinates)) cmd.extend(["tap", str(coordinates.x), str(coordinates.y)]) - run.browser_platform.sh(*cmd) + self.browser_platform.sh(*cmd) if action.verify: self.wait_for_element_impl( @@ -240,16 +236,15 @@ timeout=action.timeout, check_element_rect=True) - def _swipe_impl(self, run: Run, start_x: int, start_y: int, end_x: int, - end_y: int, duration: dt.timedelta) -> None: + def _swipe_impl(self, start_x: int, start_y: int, end_x: int, end_y: int, + duration: dt.timedelta) -> None: duration_millis = round(duration // dt.timedelta(milliseconds=1)) - run.browser_platform.sh("input", "swipe", str(start_x), str(start_y), - str(end_x), str(end_y), str(duration_millis)) + self.browser_platform.sh("input", "swipe", str(start_x), str(start_y), + str(end_x), str(end_y), str(duration_millis)) def _get_viewport_info(self, - run: Run, actions: Actions, selector: str | None = None, scroll_into_view: bool = False) -> ViewportInfo: @@ -265,8 +260,7 @@ height) = actions.js( script, arguments=[selector, scroll_into_view]) - raw_chrome_window_bounds: DisplayRectangle = self._find_chrome_window_size( - run) + raw_chrome_window_bounds: DisplayRectangle = self._find_chrome_window_size() element_rect: DisplayRectangle | None = None if found_element: @@ -284,7 +278,7 @@ raise RuntimeError("Unsupported browser for android action runner.") - def _find_chrome_window_size(self, run: Run) -> DisplayRectangle: + def _find_chrome_window_size(self) -> DisplayRectangle: # Find the chrome app window position by dumping the android app window # list. The list is sorted from highest to lowest z-order, so the first # Chrome window is the focused window. @@ -298,10 +292,10 @@ # # mAppBounds=Rect(0, 0 - 480, 800) browser_main_window_name = self._get_browser_window_name( - run.browser.attributes()) + self.browser.attributes()) - raw_window_config = run.browser_platform.sh_stdout("dumpsys", "window", - "windows") + raw_window_config = self.browser_platform.sh_stdout("dumpsys", "window", + "windows") raw_window_config = raw_window_config[raw_window_config .find(browser_main_window_name):] @@ -316,20 +310,20 @@ return DisplayRectangle( Point(int(match["left"]), int(match["top"])), width, height) - def _type_characters(self, run: Run, _: Actions, characters: str) -> None: + def _type_characters(self, _: Actions, characters: str) -> None: # TODO(kalutes) handle special characters and other whitespaces like '\t' # The 'input text' command cannot handle spaces directly. Replace space # characters with the encoding '%s'. characters = characters.replace(" ", "%s") - run.browser_platform.sh("input", "keyboard", "text", characters) + self.browser_platform.sh("input", "keyboard", "text", characters) - def _send_keyevent(self, run: Run, keyevent: str) -> None: - run.browser_platform.sh("input", "keyevent", keyevent) + def _send_keyevent(self, keyevent: str) -> None: + self.browser_platform.sh("input", "keyevent", keyevent) - def _click_ui_selector(self, run: Run, ui_selector: UiSelectorConfig, + def _click_ui_selector(self, ui_selector: UiSelectorConfig, timeout: dt.timedelta) -> None: - adb_platform = cast("AndroidAdbPlatform", run.browser_platform) + adb_platform = cast("AndroidAdbPlatform", self.browser_platform) with adb_platform.uiautomator_device() as ad: selector_dict = ui_selector.to_json() ui_object = ad.ui(**ui_selector.to_json())
diff --git a/crossbench/action_runner/base.py b/crossbench/action_runner/base.py index 7bc03b0..05be43c 100644 --- a/crossbench/action_runner/base.py +++ b/crossbench/action_runner/base.py
@@ -37,6 +37,8 @@ from crossbench.benchmarks.loading.page.combined import CombinedPage from crossbench.benchmarks.loading.page.interactive import InteractivePage from crossbench.benchmarks.loading.tab_controller import TabController + from crossbench.browsers.browser import Browser + from crossbench.plt.base import Platform from crossbench.runner.actions import Actions from crossbench.runner.run import Run @@ -131,15 +133,39 @@ _bond: BondActionRunner | None = None - def __init__(self) -> None: + def __init__(self, run: Run, step_by_step_mode: bool = False) -> None: + self._run = run self._listener = ActionRunnerListener() # TODO: Don't share state across runs self._info_stack: exception.TInfoStack | None = None - self._step_by_step_mode: bool = False + self._step_by_step_mode = step_by_step_mode self._failure_screenshot_annotations: list[ScreenshotAnnotation] = [] - def set_step_by_step_mode(self, step_by_step_mode: bool) -> None: - self._step_by_step_mode = step_by_step_mode + @property + def run(self) -> Run: + return self._run + + @run.setter + def run(self, value: Run) -> None: + self._run = value + + @property + def browser(self) -> Browser: + return self.run.browser + + @property + def host_platform(self) -> Platform: + return self.run.host_platform + + @property + def browser_platform(self) -> Platform: + return self.run.browser_platform + + def actions(self, + name: str, + verbose: bool = False, + measure: bool = True) -> Actions: + return self.run.actions(name, verbose=verbose, measure=measure) def set_listener(self, listener: ActionRunnerListener) -> None: self._listener = listener @@ -155,7 +181,7 @@ @property def bond(self) -> BondActionRunner: if not self._bond: - self._bond = BondActionRunner(self) + self._bond = BondActionRunner(self, self.run) return self._bond def teardown(self) -> None: @@ -225,17 +251,17 @@ message: str = action.TYPE.name with run.exceptions.annotate(message): sys.stdout.write(f" {message.ljust(30)}\r") - action.run_with(run, self) + action.run_with(self) - def wait(self, run: Run, action: i_action.WaitAction) -> None: - with run.actions("WaitAction", measure=False) as actions: + def wait(self, action: i_action.WaitAction) -> None: + with self.actions("WaitAction", measure=False) as actions: actions.wait(action.duration) - def js(self, run: Run, action: i_action.JsAction) -> None: - with run.actions("JS", measure=False) as actions: + def js(self, action: i_action.JsAction) -> None: + with self.actions("JS", measure=False) as actions: actions.js(action.script, action.timeout) - def click(self, run: Run, action: i_action.ClickAction) -> None: + def click(self, action: i_action.ClickAction) -> None: input_source = action.input_source if input_source is InputSource.JS: do_click = self.click_js @@ -248,7 +274,7 @@ for i in range(action.attempts): try: - do_click(run, action) + do_click(action) return except Exception as e: if i + 1 < action.attempts: @@ -257,38 +283,38 @@ continue raise e - def scroll(self, run: Run, action: i_action.ScrollAction) -> None: + def scroll(self, action: i_action.ScrollAction) -> None: input_source = action.input_source if input_source is InputSource.JS: - self.scroll_js(run, action) + self.scroll_js(action) elif input_source is InputSource.TOUCH: - self.scroll_touch(run, action) + self.scroll_touch(action) elif input_source is InputSource.MOUSE: - self.scroll_mouse(run, action) + self.scroll_mouse(action) else: raise RuntimeError(f"Unsupported input source: '{input_source}'") - def get(self, run: Run, action: i_action.GetAction) -> None: - with run.actions(f"Get {action.url}", measure=False) as actions: + def get(self, action: i_action.GetAction) -> None: + with self.actions(f"Get {action.url}", measure=False) as actions: with actions.wait_until(action.duration): actions.show_url(action.url, str(action.target), action.ready_state, action.timeout) - def clear_cache(self, run: Run, action: i_action.ClearCacheAction) -> None: + def clear_cache(self, action: i_action.ClearCacheAction) -> None: del action - with run.actions("ClearCacheAction", measure=False): - run.browser.clear_cache() + with self.actions("ClearCacheAction", measure=False): + self.browser.clear_cache() - def text_input(self, run: Run, action: i_action.TextInputAction) -> None: + def text_input(self, action: i_action.TextInputAction) -> None: input_source = action.input_source if input_source is InputSource.KEYBOARD: - self.text_input_keyboard(run, action) + self.text_input_keyboard(action) elif input_source is InputSource.JS and not action.keyevent: - self.text_input_js(run, action) + self.text_input_js(action) else: raise RuntimeError(f"Unsupported input source: '{input_source}'") - def click_js(self, run: Run, action: i_action.ClickAction) -> None: + def click_js(self, action: i_action.ClickAction) -> None: if action.duration > dt.timedelta(): raise InputSourceNotImplementedError( self, @@ -308,7 +334,7 @@ return_on_success=True, ) - with run.actions("ClickAction", measure=False) as actions: + with self.actions("ClickAction", measure=False) as actions: if selector_config.wait: self.wait_for_element_impl( actions, @@ -324,16 +350,14 @@ self.wait_for_element_impl( actions, selector=action.verify, timeout=action.timeout) - def click_touch(self, run: Run, action: i_action.ClickAction) -> None: - del run + def click_touch(self, action: i_action.ClickAction) -> None: raise InputSourceNotImplementedError(self, action, action.input_source) - def click_mouse(self, run: Run, action: i_action.ClickAction) -> None: - del run + def click_mouse(self, action: i_action.ClickAction) -> None: raise InputSourceNotImplementedError(self, action, action.input_source) - def scroll_js(self, run: Run, action: i_action.ScrollAction) -> None: - with run.actions("ScrollAction", measure=False) as actions: + def scroll_js(self, action: i_action.ScrollAction) -> None: + with self.actions("ScrollAction", measure=False) as actions: selector = "" selector_script = self.SELECT_WINDOW @@ -374,34 +398,28 @@ scroll_y = initial_scroll_y + distance actions.js(do_scroll_script, arguments=[selector, scroll_y]) - def scroll_touch(self, run: Run, action: i_action.ScrollAction) -> None: - del run + def scroll_touch(self, action: i_action.ScrollAction) -> None: raise InputSourceNotImplementedError(self, action, action.input_source) - def scroll_mouse(self, run: Run, action: i_action.ScrollAction) -> None: - del run + def scroll_mouse(self, action: i_action.ScrollAction) -> None: raise InputSourceNotImplementedError(self, action, action.input_source) - def text_input_js(self, run: Run, action: i_action.TextInputAction) -> None: - with run.actions("TextInput", measure=False) as actions: + def text_input_js(self, action: i_action.TextInputAction) -> None: + with self.actions("TextInput", measure=False) as actions: if text := action.text: actions.js( "document.activeElement.value = arguments[0]", arguments=[text]) else: raise InputSourceNotImplementedError(self, action, action.input_source) - def text_input_keyboard(self, run: Run, - action: i_action.TextInputAction) -> None: - del run + def text_input_keyboard(self, action: i_action.TextInputAction) -> None: raise InputSourceNotImplementedError(self, action, action.input_source) - def swipe(self, run: Run, action: i_action.SwipeAction) -> None: - del run + def swipe(self, action: i_action.SwipeAction) -> None: raise ActionNotImplementedError(self, action) - def wait_for_condition(self, run: Run, - action: i_action.WaitForConditionAction) -> None: - with run.actions("WaitForConditionAction", measure=False) as actions: + def wait_for_condition(self, action: i_action.WaitForConditionAction) -> None: + with self.actions("WaitForConditionAction", measure=False) as actions: actions.wait_js_condition( action.condition, min_interval=0.1, timeout=action.timeout) @@ -451,9 +469,8 @@ raise logging.debug("Element %s not found: %s", selector, e) - def wait_for_element(self, run: Run, - action: i_action.WaitForElementAction) -> None: - with run.actions("WaitForElementAction", measure=False) as actions: + def wait_for_element(self, action: i_action.WaitForElementAction) -> None: + with self.actions("WaitForElementAction", measure=False) as actions: self.wait_for_element_impl( actions=actions, selector=action.selector, @@ -463,29 +480,28 @@ check_element_rect=action.check_rect, ) - def wait_for_ready_state(self, run: Run, + def wait_for_ready_state(self, action: i_action.WaitForReadyStateAction) -> None: - with run.actions( + with self.actions( f"Wait for ready state {action.ready_state}", measure=False) as actions: actions.wait_for_ready_state(action.ready_state, action.timeout) def inject_new_document_script( - self, run: Run, action: i_action.InjectNewDocumentScriptAction) -> None: - run.browser.run_script_on_new_document(action.script) + self, action: i_action.InjectNewDocumentScriptAction) -> None: + self.browser.run_script_on_new_document(action.script) - def invoke_probe(self, run: Run, action: BaseProbeAction) -> None: - ctx = run.get_probe_context(action.probe_cls) + def invoke_probe(self, action: BaseProbeAction) -> None: + ctx = self.run.get_probe_context(action.probe_cls) if ctx is None: raise ProbeContextLookupError(action.probe_cls) - with run.actions(f"Invoke Probe ({action.probe_cls.NAME})", measure=False): + with self.actions(f"Invoke Probe ({action.probe_cls.NAME})", measure=False): ctx.invoke( info_stack=self.info_stack, timeout=action.timeout, **action.kwargs) - def open_devtools(self, run: Run, - action: i_action.OpenDevToolsAction) -> None: + def open_devtools(self, action: i_action.OpenDevToolsAction) -> None: logging.info("Opening DevTools panel '%s'...", action.panel_name) - DevToolsClient().open_frontend(run.browser, action.panel_name) + DevToolsClient().open_frontend(self.browser, action.panel_name) def screenshot_impl( self, @@ -601,9 +617,9 @@ page.create_failure_artifacts(run, "failure") raise - def switch_tab(self, run: Run, action: i_action.SwitchTabAction) -> None: - with run.actions("SwitchTabAction", measure=False): - run.browser.switch_tab( + def switch_tab(self, action: i_action.SwitchTabAction) -> None: + with self.actions("SwitchTabAction", measure=False): + self.browser.switch_tab( action.title, action.url, action.tab_index, @@ -611,9 +627,9 @@ action.timeout, ) - def close_tab(self, run: Run, action: i_action.CloseTabAction) -> None: - with run.actions("CloseTabAction", measure=False): - run.browser.close_tab( + def close_tab(self, action: i_action.CloseTabAction) -> None: + with self.actions("CloseTabAction", measure=False): + self.browser.close_tab( action.title, action.url, action.tab_index, @@ -621,11 +637,10 @@ action.timeout, ) - def close_all_tabs(self, run: Run, - action: i_action.CloseAllTabsAction) -> None: + def close_all_tabs(self, action: i_action.CloseAllTabsAction) -> None: del action - with run.actions("CloseAllTabsAction", measure=False): - run.browser.close_all_tabs() + with self.actions("CloseAllTabsAction", measure=False): + self.browser.close_all_tabs() def _get_scroll_field(self, has_selector: bool) -> str: if has_selector: @@ -634,19 +649,18 @@ def _rate_limit_keystrokes( self, - run: Run, action: i_action.TextInputAction, - do_type_function: Callable[[Run, Actions, str], Any], + do_type_function: Callable[[Actions, str], Any], ) -> None: action_text = cast(str, action.text) character_delay_s = (action.duration / len(action_text)).total_seconds() start_time = time.time() action_expected_end_time = start_time + action.duration.total_seconds() - with run.actions("TextInput", measure=False) as actions: + with self.actions("TextInput", measure=False) as actions: # When no duration is specified, input the entire text at once. if action.duration == dt.timedelta(): - do_type_function(run, actions, action_text) + do_type_function(actions, action_text) return character_expected_end_time = start_time @@ -654,7 +668,7 @@ for character in action_text: character_expected_end_time += character_delay_s - do_type_function(run, actions, character) + do_type_function(actions, character) expected_end_delta = character_expected_end_time - time.time()
diff --git a/crossbench/action_runner/bond_action_runner.py b/crossbench/action_runner/bond_action_runner.py index 46d85eb..501f166 100644 --- a/crossbench/action_runner/bond_action_runner.py +++ b/crossbench/action_runner/bond_action_runner.py
@@ -38,13 +38,18 @@ class BondActionRunner: - def __init__(self, action_runner: ActionRunner) -> None: + def __init__(self, action_runner: ActionRunner, run: Run) -> None: self._action_runner: ActionRunner = action_runner + self._run: Run = run self._bond_client: BondClient | None = None - def bond_client(self, run: Run) -> BondClient: + @property + def run(self) -> Run: + return self._run + + def bond_client(self) -> BondClient: if not self._bond_client: - secret = run.secrets.bond + secret = self.run.secrets.bond if not secret: raise RuntimeError("No bond service account secret provided") self._bond_client = BondClient(secret) @@ -70,9 +75,9 @@ raise TimeoutError("A previous request used up the timeout") return timeout - def meet_create(self, run: Run, action: i_action.MeetCreateAction) -> None: + def meet_create(self, action: i_action.MeetCreateAction) -> None: deadline = dt.datetime.now() + action.timeout - bond_client = self.bond_client(run) + bond_client = self.bond_client() conference_code = bond_client.create_meeting(timeout=action.timeout) if action.bots: bond_client.add_bots( @@ -81,14 +86,13 @@ timeout=self._timeout_from_deadline(deadline)) url = f"https://meet.google.com/{conference_code}" self._action_runner.get( - run, GetAction( url, ready_state=ReadyState.COMPLETE, target=action.target, timeout=self._timeout_from_deadline(deadline))) - def meet_script(self, run: Run, action: i_action.MeetScriptAction) -> None: - conference_code = self.get_current_conference_code(run.browser) - bond_client = self.bond_client(run) + def meet_script(self, action: i_action.MeetScriptAction) -> None: + conference_code = self.get_current_conference_code(self.run.browser) + bond_client = self.bond_client() bond_client.run_script(conference_code, action.script, action.timeout)
diff --git a/crossbench/action_runner/chromeos_input_action_runner.py b/crossbench/action_runner/chromeos_input_action_runner.py index 1062600..0110299 100644 --- a/crossbench/action_runner/chromeos_input_action_runner.py +++ b/crossbench/action_runner/chromeos_input_action_runner.py
@@ -260,8 +260,8 @@ class ChromeOSInputActionRunner(ActionRunner): """Custom ActionRunner for chromeOS devices.""" - def __init__(self) -> None: - super().__init__() + def __init__(self, run: Run, step_by_step_mode: bool = False) -> None: + super().__init__(run, step_by_step_mode) self._touch_device: TouchDevice | None = None self._mouse_process: subprocess.Popen | None = None @@ -272,11 +272,11 @@ self._mouse_process.kill() self._mouse_process.wait() - def click_touch(self, run: Run, action: i_action.ClickAction) -> None: + def click_touch(self, action: i_action.ClickAction) -> None: if not self._touch_device: - self._touch_device = self._setup_touch_device(run) + self._touch_device = self._setup_touch_device() - with run.actions("ClickAction", measure=False) as actions: + with self.actions("ClickAction", measure=False) as actions: click_location, viewport = self._get_click_location(actions, action) @@ -284,7 +284,6 @@ return self._execute_touch_playback( - run, ChromeOSTouchEvent( self._touch_device, viewport.native_screen, @@ -299,14 +298,14 @@ timeout=action.timeout, check_element_rect=True) - def click_mouse(self, run: Run, action: i_action.ClickAction) -> None: - with run.actions("ClickAction", measure=False) as actions: + def click_mouse(self, action: i_action.ClickAction) -> None: + with self.actions("ClickAction", measure=False) as actions: click_location, viewport = self._get_click_location(actions, action) if not self._mouse_process: self._mouse_process = self._setup_mouse_process( - run, viewport.native_screen.width, viewport.native_screen.height) + viewport.native_screen.width, viewport.native_screen.height) if not click_location: return @@ -332,11 +331,11 @@ timeout=action.timeout, check_element_rect=True) - def scroll_touch(self, run: Run, action: i_action.ScrollAction) -> None: + def scroll_touch(self, action: i_action.ScrollAction) -> None: if not self._touch_device: - self._touch_device = self._setup_touch_device(run) + self._touch_device = self._setup_touch_device() - with run.actions("ScrollAction", measure=False) as actions: + with self.actions("ScrollAction", measure=False) as actions: viewport_info: ChromeOSViewportInfo = self._get_viewport_info( actions, action.selector, False) @@ -373,7 +372,6 @@ y_end = round(scrollable_top + swipe_distance) self._execute_touch_playback( - run, ChromeOSTouchEvent( self._touch_device, viewport_info.native_screen, @@ -381,12 +379,11 @@ end_position=Point(scroll_area.middle.x, y_end), duration=swipe_duration)) - def text_input_keyboard(self, run: Run, - action: i_action.TextInputAction) -> None: + def text_input_keyboard(self, action: i_action.TextInputAction) -> None: if action.keyevent: raise ValueError("Keyevents are currently not supported on ChromeOS") - browser_platform = run.browser_platform + browser_platform = self.browser_platform script = (SCRIPTS_DIR / "text_input.py").read_text() @@ -400,8 +397,8 @@ assert typing_stdin, "Got no stdin" self._rate_limit_keystrokes( - run, action, - lambda run, actions, text: typing_stdin.write(text.encode("utf-8"))) + action, + lambda actions, text: typing_stdin.write(text.encode("utf-8"))) finally: if typing_stdin: typing_stdin.close() @@ -481,21 +478,21 @@ return viewport_info - def _query_touch_device(self, run: Run) -> str: + def _query_touch_device(self) -> str: try: with (SCRIPTS_DIR / "query_touch_device.py").open() as file: - return run.browser_platform.sh_stdout("python3", "-", stdin=file) + return self.browser_platform.sh_stdout("python3", "-", stdin=file) except Exception as e: raise RuntimeError( "Failed to query touchscreen information from device.") from e - def _setup_touch_device(self, run: Run) -> TouchDevice: - touch_device_output = self._query_touch_device(run) + def _setup_touch_device(self) -> TouchDevice: + touch_device_output = self._query_touch_device() return TouchDevice.parse_str(touch_device_output) - def _setup_mouse_process(self, run: Run, screen_width: int, + def _setup_mouse_process(self, screen_width: int, screen_height: int) -> subprocess.Popen: - browser_platform = run.browser_platform + browser_platform = self.browser_platform script = (SCRIPTS_DIR / "mouse.py").read_text() with browser_platform.NamedTemporaryFile() as script_file: @@ -520,8 +517,7 @@ return mouse_process - def _execute_touch_playback(self, run: Run, - touch_event: ChromeOSTouchEvent) -> None: + def _execute_touch_playback(self, touch_event: ChromeOSTouchEvent) -> None: # Ideally the touch event data could just be sent to |input| of evemu-play, # but after a lot of testing, evemu-play *only* behaves when input is # redirected from a file such as with: @@ -535,12 +531,12 @@ touch_event_cmds = str(touch_event) - browser_platform = run.browser_platform + browser_platform = self.browser_platform with browser_platform.NamedTemporaryFile() as playback_file: browser_platform.write_text(playback_file, touch_event_cmds) # Then run evemu-play with the input redirected from the temp file. - run.browser_platform.sh( # noqa: S604 + self.browser_platform.sh( # noqa: S604 f"evemu-play --insert-slot0 " f"{shlex.quote(self._touch_device.device_path)} < " f"{playback_file}",
diff --git a/crossbench/action_runner/config.py b/crossbench/action_runner/config.py index fe98aad..62d419e 100644 --- a/crossbench/action_runner/config.py +++ b/crossbench/action_runner/config.py
@@ -16,6 +16,7 @@ if TYPE_CHECKING: from crossbench.plt.base import Platform + from crossbench.runner.run import Run class ActionRunnerType(ConfigEnum): @@ -43,23 +44,29 @@ "type", type=ActionRunnerType, default=ActionRunnerType.AUTO) return parser - def instantiate(self, platform: Platform) -> ActionRunner: + def instantiate(self, + platform: Platform, + run: Run, + step_by_step_mode: bool = False) -> ActionRunner: match self.type: case ActionRunnerType.ANDROID: - return AndroidInputActionRunner() + return AndroidInputActionRunner(run, step_by_step_mode) case ActionRunnerType.CHROMEOS: - return ChromeOSInputActionRunner() + return ChromeOSInputActionRunner(run, step_by_step_mode) case ActionRunnerType.BASIC: # TODO: rename - return ActionRunner() + return ActionRunner(run, step_by_step_mode) case ActionRunnerType.AUTO: - return self.instantiate_default(platform) + return self.instantiate_default(platform, run, step_by_step_mode) case _: raise ValueError(f"Unsupported action runner type: {self.type}") - def instantiate_default(self, platform: Platform) -> ActionRunner: + def instantiate_default(self, + platform: Platform, + run: Run, + step_by_step_mode: bool = False) -> ActionRunner: if platform.is_android: - return AndroidInputActionRunner() + return AndroidInputActionRunner(run, step_by_step_mode) if platform.is_chromeos: - return ChromeOSInputActionRunner() - return ActionRunner() + return ChromeOSInputActionRunner(run, step_by_step_mode) + return ActionRunner(run, step_by_step_mode)
diff --git a/crossbench/benchmarks/base.py b/crossbench/benchmarks/base.py index 434cb99..088fb6e 100644 --- a/crossbench/benchmarks/base.py +++ b/crossbench/benchmarks/base.py
@@ -30,6 +30,7 @@ from crossbench.cli.parser import CBArgumentParser from crossbench.cli.types import Subparsers from crossbench.plt.base import Platform + from crossbench.runner.run import Run from crossbench.runner.runner import Runner VersionParts: TypeAlias = tuple[str] | tuple[int, ...] @@ -155,8 +156,12 @@ f"class as {self.DEFAULT_STORY_CLS}") return list(stories) - def new_action_runner(self, platform: Platform) -> ActionRunner: - return self._action_runner_config.instantiate(platform) + def new_action_runner(self, + platform: Platform, + run: Run, + step_by_step_mode: bool = False) -> ActionRunner: + return self._action_runner_config.instantiate(platform, run, + step_by_step_mode) def setup(self, runner: Runner) -> None: del runner
diff --git a/crossbench/benchmarks/devtools_frontend/devtools_frontend_benchmark.py b/crossbench/benchmarks/devtools_frontend/devtools_frontend_benchmark.py index 9f6a1da..ce9cd81 100644 --- a/crossbench/benchmarks/devtools_frontend/devtools_frontend_benchmark.py +++ b/crossbench/benchmarks/devtools_frontend/devtools_frontend_benchmark.py
@@ -113,7 +113,7 @@ with run.actions("Show URL") as actions: actions.show_url(DevToolsFrontendBenchmark.STORY_URLS[site]) actions.wait(1.0) # Wait for page load. - action_runner.open_devtools(run, OpenDevToolsAction(panel_name=panel)) + action_runner.open_devtools(OpenDevToolsAction(panel_name=panel)) actions.wait(1.5) # Let DevTools settle. logging.info("Stopping benchmark...")
diff --git a/crossbench/benchmarks/loading/config/login/google.py b/crossbench/benchmarks/loading/config/login/google.py index 7f838fa..3fad6b9 100644 --- a/crossbench/benchmarks/loading/config/login/google.py +++ b/crossbench/benchmarks/loading/config/login/google.py
@@ -144,31 +144,31 @@ if current_url.startswith(ADD_PASSKEY_REDIRECT): logging.info("Dismissing passkey enrollment page.") - self._dismiss_login_page(action, runner, run, SKIP_PASSKEY_ACTION, + self._dismiss_login_page(action, runner, SKIP_PASSKEY_ACTION, ADD_PASSKEY_REDIRECT, time_left) if current_url.startswith(ADD_RECOVERY_PHONE_REDIRECT): logging.info("Dismissing account recovery page.") - self._dismiss_login_page(action, runner, run, SKIP_RECOVERY_PHONE, + self._dismiss_login_page(action, runner, SKIP_RECOVERY_PHONE, ADD_RECOVERY_PHONE_REDIRECT, time_left) if current_url.startswith(ADD_HOME_ADDRESS_REDIRECT): logging.info("Dismissing add home address page.") - self._dismiss_login_page(action, runner, run, SKIP_HOME_ADDRESS, + self._dismiss_login_page(action, runner, SKIP_HOME_ADDRESS, ADD_HOME_ADDRESS_REDIRECT, time_left) - self._clear_suspicious_activity(action, runner, run) + self._clear_suspicious_activity(action, runner) - def _dismiss_login_page(self, action: Actions, runner: ActionRunner, run: Run, + def _dismiss_login_page(self, action: Actions, runner: ActionRunner, click_action: ClickAction, current_url: str, timeout: dt.timedelta) -> None: - runner.click(run, click_action) + runner.click(click_action) action.wait_js_condition( f"return !document.URL.startsWith('{current_url}');", 0.2, timeout) action.wait_for_ready_state(ReadyState.COMPLETE, timeout) - def _clear_suspicious_activity(self, action: Actions, runner: ActionRunner, - run: Run) -> None: + def _clear_suspicious_activity(self, action: Actions, + runner: ActionRunner) -> None: has_suspicious_activity = action.js( "return document.querySelector(" "\"[aria-label='Check activity']\") != null;") @@ -176,5 +176,5 @@ if not has_suspicious_activity: return - runner.click(run, CHECK_SUSPICIOUS_ACTIVITY) - runner.click(run, CLICK_YES_IT_WAS_ME) + runner.click(CHECK_SUSPICIOUS_ACTIVITY) + runner.click(CLICK_YES_IT_WAS_ME)
diff --git a/crossbench/benchmarks/loading/page/interactive.py b/crossbench/benchmarks/loading/page/interactive.py index 4101224..2bc77fc 100644 --- a/crossbench/benchmarks/loading/page/interactive.py +++ b/crossbench/benchmarks/loading/page/interactive.py
@@ -100,7 +100,7 @@ logging.error("Failed to take a failure screenshot: %s", e) try: - action_runner.invoke_probe(run, DumpHtmlAction(suffix=message)) + action_runner.invoke_probe(DumpHtmlAction(suffix=message)) except ProbeContextLookupError: pass except Exception as e: # noqa: BLE001
diff --git a/crossbench/benchmarks/speedometer/speedometer.py b/crossbench/benchmarks/speedometer/speedometer.py index c728e48..4d44b77 100644 --- a/crossbench/benchmarks/speedometer/speedometer.py +++ b/crossbench/benchmarks/speedometer/speedometer.py
@@ -289,7 +289,7 @@ action = ClickAction(InputSource.TOUCH, PositionConfig(selector=selector_config)) try: - run.action_runner.click_touch(run, action) + run.action_runner.click_touch(action) return True except ElementNotFoundError: return False
diff --git a/crossbench/benchmarks/web_power/page_load.py b/crossbench/benchmarks/web_power/page_load.py index 4387425..578842e 100644 --- a/crossbench/benchmarks/web_power/page_load.py +++ b/crossbench/benchmarks/web_power/page_load.py
@@ -80,7 +80,7 @@ with run.actions("Run", verbose=True): for i in playback: with run.actions(f"Cache_Clear_{i}"): - run.action_runner.clear_cache(run, ClearCacheAction()) + run.action_runner.clear_cache(ClearCacheAction()) with run.actions(f"Close_Tab_{i}"): run.browser.close_tab(tab_index=0, timeout=dt.timedelta(seconds=1)) with run.actions(f"Page_Load_{i}") as actions:
diff --git a/crossbench/runner/run.py b/crossbench/runner/run.py index b9fc476..9409c7f 100644 --- a/crossbench/runner/run.py +++ b/crossbench/runner/run.py
@@ -63,7 +63,6 @@ runner: Runner, browser_session: BrowserSessionRunGroup, story: Story, - action_runner: ActionRunner, repetition: int, is_warmup: bool, temperature: str, @@ -81,7 +80,7 @@ self._env = RunEnv(self, self._browser.settings.env_config, env_validation_mode) self._story = story - self._action_runner = action_runner + self._action_runner = runner.new_action_runner(self._browser.platform, self) self._repetition = NumberParser.positive_zero_int(repetition, "repetition") self._is_warmup = is_warmup assert temperature, "Missing cache-temperature value."
diff --git a/crossbench/runner/runner.py b/crossbench/runner/runner.py index 8666868..08ef614 100644 --- a/crossbench/runner/runner.py +++ b/crossbench/runner/runner.py
@@ -500,6 +500,10 @@ def has_browser_group(self) -> bool: return self._browser_group is not None + def new_action_runner(self, platform: plt.Platform, run: Run) -> ActionRunner: + return self._benchmark.new_action_runner(platform, run, + self._step_by_step_mode) + def wait_range(self, min_interval: AnyTimeUnit, timeout: AnyTimeUnit, @@ -650,12 +654,9 @@ if len(self.cache_temperatures) > 1: name_parts.append(f"temperature={temperature_icon(temperature)}") name_parts.append(f"index={index}") - action_runner = self.benchmark.new_action_runner(browser.platform) - action_runner.set_step_by_step_mode(self._step_by_step_mode) yield self.create_run( browser_session, story, - action_runner, repetition, is_warmup, f"{t_index}_{temperature}", @@ -667,13 +668,21 @@ index += 1 browser_session.set_ready() - def create_run(self, browser_session: BrowserSessionRunGroup, story: Story, - action_runner: ActionRunner, repetition: int, is_warmup: bool, - temperature: str, index: int, name: str, timeout: dt.timedelta, - throw: bool, env_validation_mode: ValidationMode) -> Run: - return Run(self, browser_session, story, action_runner, repetition, - is_warmup, temperature, index, name, timeout, throw, - env_validation_mode) + def create_run( + self, + browser_session: BrowserSessionRunGroup, + story: Story, + repetition: int, + is_warmup: bool, + temperature: str, + index: int, + name: str, + timeout: dt.timedelta, + throw: bool, + env_validation_mode: ValidationMode, + ) -> Run: + return Run(self, browser_session, story, repetition, is_warmup, temperature, + index, name, timeout, throw, env_validation_mode) def assert_successful_sessions_and_runs(self) -> None: if self._exceptions.is_success:
diff --git a/tests/crossbench/base.py b/tests/crossbench/base.py index 5bac730..3ee15e6 100644 --- a/tests/crossbench/base.py +++ b/tests/crossbench/base.py
@@ -12,7 +12,7 @@ import io import logging import pathlib -from typing import TYPE_CHECKING, Callable, Final, Iterator, Sequence +from typing import TYPE_CHECKING, Any, Callable, Final, Iterator, Sequence from unittest import mock from pyfakefs import fake_filesystem_unittest @@ -35,10 +35,13 @@ from crossbench.cli.config.secrets import Secrets from crossbench.cli.subcommand.benchmark import BenchmarkSubcommand from crossbench.config import config_dir +from crossbench.flags.base import Flags from crossbench.probes.cb_perfetto.perfetto import TraceConfig +from crossbench.runner.groups.session import BrowserSessionRunGroup from crossbench.runner.runner import Runner from tests.crossbench import mock_browser from tests.crossbench.mock_helper import MockCLI, MockPlatform +from tests.crossbench.runner.mocks import MockRun, MockRunner if TYPE_CHECKING: from pyfakefs import fake_filesystem @@ -211,6 +214,26 @@ self.assertListEqual(self.platform.sh_results, []) super().tearDown() + def mock_run( + self, + runner: Any | None = None, + browser: Any | None = None, + story: str = "story", + ) -> MockRun: + runner = runner or MockRunner() + browser = browser or self.browsers[0] + session = BrowserSessionRunGroup( + runner.env, + runner.probes, + browser, + Flags(), + 1, + pathlib.Path(), + True, + True, + ) + return MockRun(runner, session, story) + class SysExitTestException(Exception):
diff --git a/tests/crossbench/benchmarks/loading/action_runner/test_action_runner.py b/tests/crossbench/benchmarks/loading/action_runner/test_action_runner.py index dd4a851..7f5d6f6 100644 --- a/tests/crossbench/benchmarks/loading/action_runner/test_action_runner.py +++ b/tests/crossbench/benchmarks/loading/action_runner/test_action_runner.py
@@ -36,8 +36,8 @@ class MockActionRunner(ActionRunner): - def __init__(self): - super().__init__() + def __init__(self, run: Run) -> None: + super().__init__(run) self.click_js = MagicMock(name="Mock click_js") @@ -45,20 +45,20 @@ def test_click_attempts_first_success(self): mock_run = MagicMock(name="Mock Run") - mock_action_runner = MockActionRunner() + mock_action_runner = MockActionRunner(mock_run) config_dict = {"action": "click", "selector": "#button", "attempts": 3} action = ClickAction.config_parser().parse(config_dict) mock_action_runner.click_js.side_effect = [None] - mock_action_runner.click(mock_run, action) + mock_action_runner.click(action) - mock_action_runner.click_js.assert_called_once_with(mock_run, action) + mock_action_runner.click_js.assert_called_once_with(action) def test_click_attempts_last_success(self): mock_run = MagicMock(name="Mock Run") - mock_action_runner = MockActionRunner() + mock_action_runner = MockActionRunner(mock_run) config_dict = {"action": "click", "selector": "#button", "attempts": 3} action = ClickAction.config_parser().parse(config_dict) @@ -69,17 +69,17 @@ None, ] - mock_action_runner.click(mock_run, action) + mock_action_runner.click(action) mock_action_runner.click_js.assert_has_calls([ - call(mock_run, action), - call(mock_run, action), - call(mock_run, action), + call(action), + call(action), + call(action), ]) def test_click_attempts_fail(self): mock_run = MagicMock(name="Mock Run") - mock_action_runner = MockActionRunner() + mock_action_runner = MockActionRunner(mock_run) config_dict = {"action": "click", "selector": "#button", "attempts": 3} action = ClickAction.config_parser().parse(config_dict) @@ -94,12 +94,12 @@ ] with self.assertRaises(TestException): - mock_action_runner.click(mock_run, action) + mock_action_runner.click(action) mock_action_runner.click_js.assert_has_calls([ - call(mock_run, action), - call(mock_run, action), - call(mock_run, action), + call(action), + call(action), + call(action), ]) @@ -131,14 +131,15 @@ True, True, ) - self.action_runner = ActionRunner() self.mock_run: Any = MockRun( self.runner, self.session, "run 1", - self.action_runner, probe=self.probe, ) + self.action_runner = ActionRunner(self.mock_run) + self.mock_run.action_runner = self.action_runner + if not probe_context_cls: self.probe_context = self.probe.create_context(cast(Run, self.mock_run))
diff --git a/tests/crossbench/benchmarks/loading/action_runner/test_action_runner_config.py b/tests/crossbench/benchmarks/loading/action_runner/test_action_runner_config.py index 7f230de..9ec93c8 100644 --- a/tests/crossbench/benchmarks/loading/action_runner/test_action_runner_config.py +++ b/tests/crossbench/benchmarks/loading/action_runner/test_action_runner_config.py
@@ -6,6 +6,7 @@ import argparse import unittest +from unittest import mock from crossbench import plt from crossbench.action_runner.android_input_action_runner import \ @@ -15,11 +16,15 @@ ChromeOSInputActionRunner from crossbench.action_runner.config import ActionRunnerConfig, \ ActionRunnerType +from crossbench.runner.run import Run from tests import test_helper class ActionRunnerConfigTest(unittest.TestCase): + def setUp(self) -> None: + self.mock_run = mock.MagicMock(spec=Run) + def test_parse_invalid(self): for invalid in ["bas", "adnroid", "chroms"]: with self.subTest(pattern=invalid): @@ -30,42 +35,48 @@ action_runner = ActionRunnerConfig.parse("basic") self.assertIsInstance(action_runner, ActionRunnerConfig) self.assertEqual(action_runner.type, ActionRunnerType.BASIC) - self.assertIsInstance(action_runner.instantiate(plt.PLATFORM), ActionRunner) + self.assertIsInstance( + action_runner.instantiate(plt.PLATFORM, self.mock_run), ActionRunner) def test_parse_auto(self): action_runner = ActionRunnerConfig.parse("auto") self.assertIsInstance(action_runner, ActionRunnerConfig) self.assertEqual(action_runner.type, ActionRunnerType.AUTO) - self.assertIsInstance(action_runner.instantiate(plt.PLATFORM), ActionRunner) + self.assertIsInstance( + action_runner.instantiate(plt.PLATFORM, self.mock_run), ActionRunner) def test_parse_auto_android(self): action_runner = ActionRunnerConfig.parse("auto") - mock_platform = unittest.mock.MagicMock() + mock_platform = mock.MagicMock() mock_platform.is_android = True self.assertIsInstance( - action_runner.instantiate(mock_platform), AndroidInputActionRunner) + action_runner.instantiate(mock_platform, self.mock_run), + AndroidInputActionRunner) def test_parse_auto_chromeos(self): action_runner = ActionRunnerConfig.parse("auto") - mock_platform = unittest.mock.MagicMock() + mock_platform = mock.MagicMock() mock_platform.is_android = False mock_platform.is_chromeos = True self.assertIsInstance( - action_runner.instantiate(mock_platform), ChromeOSInputActionRunner) + action_runner.instantiate(mock_platform, self.mock_run), + ChromeOSInputActionRunner) def test_parse_android(self): action_runner = ActionRunnerConfig.parse("android") self.assertIsInstance(action_runner, ActionRunnerConfig) self.assertEqual(action_runner.type, ActionRunnerType.ANDROID) self.assertIsInstance( - action_runner.instantiate(plt.PLATFORM), AndroidInputActionRunner) + action_runner.instantiate(plt.PLATFORM, self.mock_run), + AndroidInputActionRunner) def test_parse_chromeos(self): action_runner = ActionRunnerConfig.parse("chromeos") self.assertIsInstance(action_runner, ActionRunnerConfig) self.assertEqual(action_runner.type, ActionRunnerType.CHROMEOS) self.assertIsInstance( - action_runner.instantiate(plt.PLATFORM), ChromeOSInputActionRunner) + action_runner.instantiate(plt.PLATFORM, self.mock_run), + ChromeOSInputActionRunner) if __name__ == "__main__":
diff --git a/tests/crossbench/benchmarks/loading/action_runner/test_android_input_action_runner.py b/tests/crossbench/benchmarks/loading/action_runner/test_android_input_action_runner.py index 09cfa17..4c6e200 100644 --- a/tests/crossbench/benchmarks/loading/action_runner/test_android_input_action_runner.py +++ b/tests/crossbench/benchmarks/loading/action_runner/test_android_input_action_runner.py
@@ -139,12 +139,13 @@ self.session = BrowserSessionRunGroup(self.runner.env, self.runner.probes, self.browser, Flags(), 1, self.root_dir, True, True) - self.action_runner = AndroidInputActionRunner() - self.mock_run = MockRun(self.runner, self.session, "run 1", - self.action_runner) + self.mock_run = MockRun(self.runner, self.session, "run 1") + self.action_runner = AndroidInputActionRunner(self.mock_run) + self.mock_run.action_runner = self.action_runner + def run_action(self, action: Action) -> None: - action.run_with(self.mock_run, self.action_runner) + action.run_with(self.action_runner) def expect_action_setup( self,
diff --git a/tests/crossbench/benchmarks/loading/action_runner/test_bond_action_runner.py b/tests/crossbench/benchmarks/loading/action_runner/test_bond_action_runner.py index 9f96cac..7c6b7d5 100644 --- a/tests/crossbench/benchmarks/loading/action_runner/test_bond_action_runner.py +++ b/tests/crossbench/benchmarks/loading/action_runner/test_bond_action_runner.py
@@ -57,9 +57,8 @@ mock_action_runner = MagicMock(name="Mock ActionRunner") mock_action_runner.get.side_effect = [None] - bond_action_runner = BondActionRunner(mock_action_runner) - mock_run = self._make_mock_run() + bond_action_runner = BondActionRunner(mock_action_runner, mock_run) mock_bond_client = mock_bond_client_cls.return_value mock_bond_client.create_meeting.side_effect = ["mock-conference-code"] @@ -78,16 +77,18 @@ action) def test_get_current_conference_code(self): - action_runner = ActionRunner() - bond_action_runner = BondActionRunner(action_runner) + mock_run = self._make_mock_run() + action_runner = ActionRunner(mock_run) + bond_action_runner = BondActionRunner(action_runner, mock_run) for browser in self.browsers: browser.set_current_url("https://meet.google.com/abc-def-ghi") code = bond_action_runner.get_current_conference_code(browser=browser) self.assertEqual(code, "abc-def-ghi") def test_get_current_conference_code_invalid(self): - action_runner = ActionRunner() - bond_action_runner = BondActionRunner(action_runner) + mock_run = self._make_mock_run() + action_runner = ActionRunner(mock_run) + bond_action_runner = BondActionRunner(action_runner, mock_run) for browser in self.browsers: browser.set_current_url("https://www.google.com") with self.assertRaisesRegex(RuntimeError, @@ -101,14 +102,13 @@ (mock_run, mock_bond_client, mock_action_runner, bond_action_runner, action) = self._make_meet_create_mocks(mock_datetime, mock_bond_client_cls) - bond_action_runner.meet_create(mock_run, action) + bond_action_runner.meet_create(action) mock_bond_client.create_meeting.assert_called_once_with( timeout=dt.timedelta(seconds=30)) mock_bond_client.add_bots.assert_called_once_with( "mock-conference-code", action.bots, timeout=dt.timedelta(seconds=29)) mock_action_runner.get.assert_called_once_with( - mock_run, GetAction( "https://meet.google.com/mock-conference-code", ready_state=ReadyState.COMPLETE, @@ -127,7 +127,7 @@ create_meeting_duration=dt.timedelta(seconds=30)) with self.assertRaises(TimeoutError): - bond_action_runner.meet_create(mock_run, action) + bond_action_runner.meet_create(action) mock_bond_client.create_meeting.assert_called_once_with( timeout=dt.timedelta(seconds=30)) @@ -147,7 +147,7 @@ add_bots_duration=dt.timedelta(seconds=29)) with self.assertRaises(TimeoutError): - bond_action_runner.meet_create(mock_run, action) + bond_action_runner.meet_create(action) mock_bond_client.create_meeting.assert_called_once_with( timeout=dt.timedelta(seconds=30)) @@ -158,9 +158,10 @@ @patch( "crossbench.action_runner.bond_action_runner.BondClient", autospec=True) def test_meet_script(self, mock_bond_client_cls): - bond_action_runner = BondActionRunner(MagicMock(name="Mock ActionRunner")) - mock_run = self._make_mock_run() + bond_action_runner = BondActionRunner( + MagicMock(name="Mock ActionRunner"), mock_run) + mock_run.browser.current_url = "https://meet.google.com/abc-def-ghi" action = MeetScriptAction.parse_dict({ @@ -172,7 +173,7 @@ mock_bond_client = mock_bond_client_cls.return_value mock_bond_client.run_script.side_effect = [None] - bond_action_runner.meet_script(mock_run, action) + bond_action_runner.meet_script(action) mock_bond_client.run_script.assert_called_once_with( "abc-def-ghi", "test script", dt.timedelta(seconds=17))
diff --git a/tests/crossbench/benchmarks/loading/action_runner/test_chromeos_input_action_runner.py b/tests/crossbench/benchmarks/loading/action_runner/test_chromeos_input_action_runner.py index 9271c3e..fc84e2d 100644 --- a/tests/crossbench/benchmarks/loading/action_runner/test_chromeos_input_action_runner.py +++ b/tests/crossbench/benchmarks/loading/action_runner/test_chromeos_input_action_runner.py
@@ -358,11 +358,13 @@ self.session = BrowserSessionRunGroup(self.runner.env, self.runner.probes, self.browser, Flags(), 1, self.root_dir, True, True) - self.action_runner = ChromeOSInputActionRunner() - self.run = MockRun(self.runner, self.session, "run 1", self.action_runner) + self.run = MockRun(self.runner, self.session, "run 1") + self.action_runner = ChromeOSInputActionRunner(self.run) + self.run.action_runner = self.action_runner + def run_action(self, action: Action) -> None: - action.run_with(self.run, self.action_runner) + action.run_with(self.action_runner) def expect_touch_setup(self, expected_js: JsInvocation, touch_count: int = 1):
diff --git a/tests/crossbench/benchmarks/loading/config/test_login.py b/tests/crossbench/benchmarks/loading/config/test_login.py index e1b88ac..45f7193 100644 --- a/tests/crossbench/benchmarks/loading/config/test_login.py +++ b/tests/crossbench/benchmarks/loading/config/test_login.py
@@ -5,7 +5,6 @@ import pathlib -from crossbench.action_runner.base import ActionRunner from crossbench.benchmarks.loading.config.pages import PagesConfig from crossbench.benchmarks.loading.loading_benchmark import LoadingPageFilter from crossbench.browsers.settings import Settings @@ -61,9 +60,8 @@ self.session = BrowserSessionRunGroup(self.runner.env, self.runner.probes, self.browser, Flags(), 1, self.root_dir, True, True) - self.action_runner = ActionRunner() - self.mock_run = MockRun(self.runner, self.session, "run 1", - self.action_runner) + self.mock_run = MockRun(self.runner, self.session, "run 1") + self.action_runner = self.mock_run.action_runner def expect_successful_google_login(self): # Wait for readystate interactive
diff --git a/tests/crossbench/benchmarks/loading/test_loading.py b/tests/crossbench/benchmarks/loading/test_loading.py index 4974e79..9ed8a27 100644 --- a/tests/crossbench/benchmarks/loading/test_loading.py +++ b/tests/crossbench/benchmarks/loading/test_loading.py
@@ -56,7 +56,8 @@ run_login: bool = True, run_setup: bool = True) -> LoadingPageFilter: if action_runner is None: - action_runner = ActionRunner() + action_runner = ActionRunner(self.mock_run()) + args = argparse.Namespace( about_blank_duration=about_blank_duration, playback=playback,
diff --git a/tests/crossbench/benchmarks/loadline/test_loadline.py b/tests/crossbench/benchmarks/loadline/test_loadline.py index 90dcfd8..818c433 100644 --- a/tests/crossbench/benchmarks/loadline/test_loadline.py +++ b/tests/crossbench/benchmarks/loadline/test_loadline.py
@@ -42,7 +42,7 @@ about_blank_duration=dt.timedelta(), playback=PlaybackController.default(), tabs=TabController.default(), - action_runner=ActionRunner(), + action_runner=ActionRunner(self.mock_run()), run_login=True, run_setup=True, )
diff --git a/tests/crossbench/probes/test_probe_results.py b/tests/crossbench/probes/test_probe_results.py index f318cd6..e8a1c6c 100644 --- a/tests/crossbench/probes/test_probe_results.py +++ b/tests/crossbench/probes/test_probe_results.py
@@ -301,7 +301,7 @@ self.browser = browser self.browser_platform = browser_platform self.is_remote = False - self.action_runner: ActionRunner = action_runner or ActionRunner() + self.action_runner: ActionRunner = action_runner or ActionRunner(self) class BrowserProbeResultTestCase(BaseCrossbenchTestCase):
diff --git a/tests/crossbench/runner/helper.py b/tests/crossbench/runner/helper.py index 7b90614..178b7fb 100644 --- a/tests/crossbench/runner/helper.py +++ b/tests/crossbench/runner/helper.py
@@ -5,30 +5,15 @@ from __future__ import annotations import abc -import datetime as dt -import json import pathlib -from typing import TYPE_CHECKING, Any, Iterable, NamedTuple +from typing import TYPE_CHECKING, Iterable from unittest import mock from typing_extensions import override from crossbench import path as pth -from crossbench.action_runner.base import ActionRunner from crossbench.browsers.settings import Settings -from crossbench.cli.config.secrets import Secrets -from crossbench.env.runner_env import RunnerEnv -from crossbench.exception import Annotator -from crossbench.helper.durations import Durations -from crossbench.helper.wait import WaitRange -from crossbench.path import AnyPath, safe_filename -from crossbench.probes.probe import Probe -from crossbench.probes.probe_context import ProbeContext -from crossbench.probes.results import LocalProbeResult, ProbeResult -from crossbench.runner.actions import Actions -from crossbench.runner.result_origin import ResultOrigin from crossbench.runner.runner import Runner -from crossbench.runner.timing import Timing from tests.crossbench.base import BaseCrossbenchTestCase from tests.crossbench.mock_browser import MockChromeDev, MockFirefox from tests.crossbench.mock_helper import MockBenchmark, MockStory @@ -37,257 +22,23 @@ from crossbench import plt from crossbench.benchmarks.base import Benchmark from crossbench.browsers.browser import Browser - from crossbench.probes.probe import ProbeT - from crossbench.runner.run import Run - from crossbench.runner.timing import AnyTimeUnit + from crossbench.path import AnyPath + from crossbench.probes.probe import Probe +from tests.crossbench.runner.mocks import MockBrowser, MockNetwork, \ + MockPlatform, MockProbe, MockProbeContext, MockRun, MockRunner, MockWait -class MockBrowser: - - def __init__(self, unique_name: str, platform) -> None: - self.unique_name = unique_name - self.platform = platform - self.network = MockNetwork() - - def __str__(self): - return self.unique_name - - -class MockRun(ResultOrigin): - - def __init__(self, - runner, - browser_session, - story="story", - action_runner: ActionRunner | None = None, - repetition=0, - is_warmup=False, - temperature="default", - index=0, - name="run 0", - probe=None, - probe_context=None) -> None: - self._runner = runner - self.browser_session = browser_session - self._browser = browser_session.browser - self._exceptions = Annotator(False) - self._durations = Durations() - self._browser_tmp_dir = pth.AnyPath("/browser_tmp") - self.repetition = repetition - self.is_warmup = is_warmup - self.temperature = temperature - self.name = name - self._probes: list[Probe] = [probe] - self.probe_context: ProbeContext | None = probe_context - self.timing = Timing() - self.is_success = True - self.index = index - self.story = story - self.action_runner: ActionRunner = action_runner or ActionRunner() - self.story_secrets = Secrets() - self._out_dir = ( - browser_session.root_dir / safe_filename(self._browser.unique_name) / - "stories" / name / f"repetition={self.repetition}" / self.temperature) - self.group_dir = self._out_dir.parent - self.did_setup = False - self.did_run = False - self.did_teardown = False - self.did_teardown_browser = False - self.is_dry_run: bool | None = None - - def validate_env(self, env: RunnerEnv): - pass - - def setup(self, is_dry_run: bool) -> None: - assert self.is_dry_run is None - self.is_dry_run = is_dry_run - assert not self.did_setup - self.did_setup = True - - def actions(self, - name: str, - verbose: bool = False, - measure: bool = True) -> Actions: - return Actions(name, self, verbose=verbose, measure=measure) - - def set_probe_context(self, probe_context: ProbeContext) -> None: - self.probe_context = probe_context - - @property - def runner(self) -> Runner: - return self._runner - - @runner.setter - def runner(self, value: Runner) -> None: - self._runner = value - - @property - def probes(self) -> list[Probe]: - return self._probes - - @probes.setter - def probes(self, value: list[Probe]) -> None: - self._probes = value - - @property - def browser(self) -> Browser: - return self._browser - - @browser.setter - def browser(self, value: Browser) -> None: - self._browser = value - - @property - def out_dir(self) -> pth.LocalPath: - return self._out_dir - - @out_dir.setter - def out_dir(self, value: pth.LocalPath) -> None: - self._out_dir = value - - @property - def exceptions(self) -> Annotator: - return self._exceptions - - @property - def durations(self) -> Durations: - return self._durations - - @property - def browser_tmp_dir(self) -> pth.AnyPath: - return self._browser_tmp_dir - - @property - def secrets(self) -> Secrets: - return self.story_secrets.merge(fallback=self.browser.secrets) - - @property - def is_remote(self) -> bool: - return self.browser_platform.is_remote - - def max_end_datetime(self) -> dt.datetime: - return dt.datetime.max - - def run(self, is_dry_run: bool) -> None: - assert self.is_dry_run is is_dry_run - assert not self.did_run - self.did_run = True - - def teardown(self, is_dry_run: bool) -> None: - assert self.is_dry_run is is_dry_run - assert not self.did_teardown - self.did_teardown = True - - def wait_range(self, - min_interval: AnyTimeUnit, - timeout: AnyTimeUnit, - delay: AnyTimeUnit = 0) -> WaitRange: - timing = self.timing - return WaitRange( - min=timing.timedelta(min_interval), - timeout=timing.timeout_timedelta(timeout), - delay=timing.timedelta(delay)) - - def get_probe_context(self, - probe_cls: type[ProbeT]) -> ProbeContext[ProbeT] | None: - del probe_cls - return self.probe_context - - def get_default_probe_result_path(self, probe: Probe) -> AnyPath: - del probe - return AnyPath("/") - - def get_local_probe_result_path(self, probe: Probe) -> pth.LocalPath: - del probe - return pth.LocalPath("/") - - def _teardown_browser(self, is_dry_run: bool) -> None: - assert self.is_dry_run is is_dry_run - assert not self.did_teardown_browser - self.did_teardown_browser = True - self.browser.quit() - - def __repr__(self): - return f"MockRun({self.name}, id={hex(id(self))})" - - def __str__(self): - return self.name - - -class MockPlatform: - - def __init__(self, name) -> None: - self.name = name - - def __str__(self): - return self.name - - @property - def key(self) -> str: - return self.name - - -class MockWait(NamedTuple): - time: AnyTimeUnit - absolute_time: bool - - -class MockRunner: - - def __init__(self, probes: list[Probe] | None = None) -> None: - self.benchmark = MockBenchmark(stories=[MockStory("mock_story")]) - self.runs: tuple[Run, ...] = () - self.platform = MockPlatform("test-platform") - self.repetitions = 1 - self.create_symlinks = True - self.probes: list[Probe] = probes if probes else [] - self.browsers: list[Browser] = [] - self.out_dir = pathlib.Path("results/out") - self.timing = Timing() - self.env = RunnerEnv(self.platform, self.out_dir, self.browsers, - self.probes, self.repetitions) - self.mock_waits: list[MockWait] = [] - - def wait(self, time: AnyTimeUnit, absolute_time: bool = False) -> None: - self.mock_waits.append(MockWait(time, absolute_time)) - - -class MockNetwork: - pass - - -class MockProbe(Probe): - NAME = "test-probe" - - def __init__(self, - test_data: Any = (), - context_cls: type[MockProbeContext] | None = None) -> None: - super().__init__() - self.test_data = test_data - self.context_cls = context_cls or MockProbeContext - - @property - @override - def result_path_name(self) -> str: - return f"{self.name}.json" - - @override - def get_context_cls(self): - return self.context_cls - - -class MockProbeContext(ProbeContext): - - def start(self) -> None: - pass - - def stop(self) -> None: - pass - - def teardown(self) -> ProbeResult: - with pathlib.Path(self.result_path).open("w", encoding="utf-8") as f: - json.dump(self.probe.test_data, f) - return LocalProbeResult(json=(self.result_path,)) +__all__ = [ + "BaseRunnerTestCase", + "MockBrowser", + "MockRun", + "MockPlatform", + "MockWait", + "MockRunner", + "MockNetwork", + "MockProbe", + "MockProbeContext", +] class CrossbenchMagicMockMixin:
diff --git a/tests/crossbench/runner/mocks.py b/tests/crossbench/runner/mocks.py new file mode 100644 index 0000000..8bab448 --- /dev/null +++ b/tests/crossbench/runner/mocks.py
@@ -0,0 +1,285 @@ +# Copyright 2026 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +from typing import TYPE_CHECKING, Any, NamedTuple + +from typing_extensions import override + +from crossbench import path as pth +from crossbench.action_runner.base import ActionRunner +from crossbench.cli.config.secrets import Secrets +from crossbench.env.runner_env import RunnerEnv +from crossbench.exception import Annotator +from crossbench.helper.durations import Durations +from crossbench.helper.wait import WaitRange +from crossbench.path import AnyPath, safe_filename +from crossbench.probes.probe import Probe +from crossbench.probes.probe_context import ProbeContext +from crossbench.probes.results import LocalProbeResult, ProbeResult +from crossbench.runner.actions import Actions +from crossbench.runner.result_origin import ResultOrigin +from crossbench.runner.timing import Timing +from tests.crossbench.mock_helper import MockBenchmark, MockStory + +if TYPE_CHECKING: + from crossbench.browsers.browser import Browser + from crossbench.probes.probe import ProbeT + from crossbench.runner.run import Run + from crossbench.runner.runner import Runner + from crossbench.runner.timing import AnyTimeUnit + + +class MockBrowser: + + def __init__(self, unique_name: str, platform) -> None: + self.unique_name = unique_name + self.platform = platform + self.network = MockNetwork() + + def __str__(self): + return self.unique_name + + +class MockRun(ResultOrigin): + + def __init__(self, + runner, + browser_session, + story="story", + action_runner: ActionRunner | None = None, + repetition=0, + is_warmup=False, + temperature="default", + index=0, + name="run 0", + probe=None, + probe_context=None) -> None: + self._runner = runner + self.browser_session = browser_session + self._browser = browser_session.browser + self._exceptions = Annotator(False) + self._durations = Durations() + self._browser_tmp_dir = pth.AnyPath("/browser_tmp") + self.repetition = repetition + self.is_warmup = is_warmup + self.temperature = temperature + self.name = name + self._probes: list[Probe] = [probe] + self.probe_context: ProbeContext | None = probe_context + self.timing = Timing() + self.is_success = True + self.index = index + self.story = story + self.action_runner: ActionRunner = action_runner or ActionRunner(self) + self.action_runner.run = self + + self.story_secrets = Secrets() + self._out_dir = ( + browser_session.root_dir / safe_filename(self._browser.unique_name) / + "stories" / name / f"repetition={self.repetition}" / self.temperature) + self.group_dir = self._out_dir.parent + self.did_setup = False + self.did_run = False + self.did_teardown = False + self.did_teardown_browser = False + self.is_dry_run: bool | None = None + + def validate_env(self, env: RunnerEnv): + pass + + def setup(self, is_dry_run: bool) -> None: + assert self.is_dry_run is None + self.is_dry_run = is_dry_run + assert not self.did_setup + self.did_setup = True + + def actions(self, + name: str, + verbose: bool = False, + measure: bool = True) -> Actions: + return Actions(name, self, verbose=verbose, measure=measure) + + def set_probe_context(self, probe_context: ProbeContext) -> None: + self.probe_context = probe_context + + @property + def runner(self) -> Runner: + return self._runner + + @runner.setter + def runner(self, value: Runner) -> None: + self._runner = value + + @property + def probes(self) -> list[Probe]: + return self._probes + + @probes.setter + def probes(self, value: list[Probe]) -> None: + self._probes = value + + @property + def browser(self) -> Browser: + return self._browser + + @browser.setter + def browser(self, value: Browser) -> None: + self._browser = value + + @property + def out_dir(self) -> pth.LocalPath: + return self._out_dir + + @out_dir.setter + def out_dir(self, value: pth.LocalPath) -> None: + self._out_dir = value + + @property + def exceptions(self) -> Annotator: + return self._exceptions + + @property + def durations(self) -> Durations: + return self._durations + + @property + def browser_tmp_dir(self) -> pth.AnyPath: + return self._browser_tmp_dir + + @property + def secrets(self) -> Secrets: + return self.story_secrets.merge(fallback=self.browser.secrets) + + @property + def is_remote(self) -> bool: + return self.browser_platform.is_remote + + def max_end_datetime(self) -> dt.datetime: + return dt.datetime.max + + def run(self, is_dry_run: bool) -> None: + assert self.is_dry_run is is_dry_run + assert not self.did_run + self.did_run = True + + def teardown(self, is_dry_run: bool) -> None: + assert self.is_dry_run is is_dry_run + assert not self.did_teardown + self.did_teardown = True + + def wait_range(self, + min_interval: AnyTimeUnit, + timeout: AnyTimeUnit, + delay: AnyTimeUnit = 0) -> WaitRange: + timing = self.timing + return WaitRange( + min=timing.timedelta(min_interval), + timeout=timing.timeout_timedelta(timeout), + delay=timing.timedelta(delay)) + + def get_probe_context(self, + probe_cls: type[ProbeT]) -> ProbeContext[ProbeT] | None: + del probe_cls + return self.probe_context + + def get_default_probe_result_path(self, probe: Probe) -> AnyPath: + del probe + return AnyPath("/") + + def get_local_probe_result_path(self, probe: Probe) -> pth.LocalPath: + del probe + return pth.LocalPath("/") + + def _teardown_browser(self, is_dry_run: bool) -> None: + assert self.is_dry_run is is_dry_run + assert not self.did_teardown_browser + self.did_teardown_browser = True + self.browser.quit() + + def __repr__(self): + return f"MockRun({self.name}, id={hex(id(self))})" + + def __str__(self): + return self.name + + +class MockPlatform: + + def __init__(self, name) -> None: + self.name = name + + def __str__(self): + return self.name + + @property + def key(self) -> str: + return self.name + + +class MockWait(NamedTuple): + time: AnyTimeUnit + absolute_time: bool + + +class MockRunner: + + def __init__(self, probes: list[Probe] | None = None) -> None: + self.benchmark = MockBenchmark(stories=[MockStory("mock_story")]) + self.runs: tuple[Run, ...] = () + self.platform = MockPlatform("test-platform") + self.repetitions = 1 + self.create_symlinks = True + self.probes: list[Probe] = probes if probes else [] + self.browsers: list[Browser] = [] + self.out_dir = pathlib.Path("results/out") + self.timing = Timing() + self.env = RunnerEnv(self.platform, self.out_dir, self.browsers, + self.probes, self.repetitions) + self.mock_waits: list[MockWait] = [] + + def wait(self, time: AnyTimeUnit, absolute_time: bool = False) -> None: + self.mock_waits.append(MockWait(time, absolute_time)) + + +class MockNetwork: + pass + + +class MockProbe(Probe): + NAME = "test-probe" + + def __init__(self, + test_data: Any = (), + context_cls: type[MockProbeContext] | None = None) -> None: + super().__init__() + self.test_data = test_data + self.context_cls = context_cls or MockProbeContext + + @property + @override + def result_path_name(self) -> str: + return f"{self.name}.json" + + @override + def get_context_cls(self): + return self.context_cls + + +class MockProbeContext(ProbeContext): + + def start(self) -> None: + pass + + def stop(self) -> None: + pass + + def teardown(self) -> ProbeResult: + with pathlib.Path(self.result_path).open("w", encoding="utf-8") as f: + json.dump(self.probe.test_data, f) + return LocalProbeResult(json=(self.result_path,))
diff --git a/tests/crossbench/runner/test_run.py b/tests/crossbench/runner/test_run.py index 5fab6db..347c5c7 100644 --- a/tests/crossbench/runner/test_run.py +++ b/tests/crossbench/runner/test_run.py
@@ -29,7 +29,7 @@ def test_find_probe_context(self): self.runner.attach_probe(MockProbe()) session = self.default_session() - run = Run(self.runner, session, MockStory("mock story"), None, 1, False, + run = Run(self.runner, session, MockStory("mock story"), 1, False, "1_default", 1, "test run", dt.timedelta(minutes=1), True) session.set_ready() with session.open(): @@ -38,7 +38,7 @@ def test_annotate(self): session = self.default_session() - run = Run(self.runner, session, MockStory("mock story"), None, 1, False, + run = Run(self.runner, session, MockStory("mock story"), 1, False, "1_default", 1, "test run", dt.timedelta(minutes=1), True) self.assertFalse(list(run.annotations)) annotation = RunAnnotation.warning("Some warning")