| # Copyright 2022 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import io |
| import json |
| import pathlib |
| import sys |
| import unittest |
| from typing import Dict, List, Tuple, Type |
| from unittest import mock |
| |
| import hjson |
| import pyfakefs.fake_filesystem_unittest |
| import pytest |
| |
| import crossbench |
| from crossbench import helper |
| from crossbench.browsers.chrome import Chrome, ChromeWebDriver |
| from crossbench.browsers.safari import Safari |
| from crossbench.cli import (BrowserConfig, ConfigFileError, CrossBenchCLI, |
| FlagGroupConfig, ProbeConfig) |
| from crossbench.probes.power_sampler import PowerSamplerProbe |
| from crossbench.probes.v8.log import V8LogProbe |
| from crossbench.runner import Runner |
| from tests import mock_browser |
| from tests.mock_helper import BaseCrossbenchTestCase, MockCLI |
| |
| |
| class SysExitException(Exception): |
| |
| def __init__(self): |
| super().__init__("sys.exit") |
| |
| |
| class TestCLI(BaseCrossbenchTestCase): |
| |
| def run_cli(self, *args, raises=None): |
| with mock.patch("sys.stdout", new_callable=io.StringIO) as mock_stdout: |
| cli = MockCLI() |
| if raises: |
| with self.assertRaises(raises): |
| cli.run(args) |
| else: |
| cli.run(args) |
| return cli, mock_stdout.getvalue() |
| |
| def test_invalid(self): |
| with mock.patch("sys.exit", side_effect=SysExitException): |
| self.run_cli( |
| "unknown subcommand", "--invalid flag", raises=SysExitException) |
| |
| def test_describe_invalid(self): |
| with mock.patch("sys.exit", side_effect=SysExitException): |
| self.run_cli("describe", "", raises=SysExitException) |
| with mock.patch("sys.exit", side_effect=SysExitException): |
| self.run_cli("describe", "--unknown", raises=SysExitException) |
| |
| def test_describe(self): |
| # Non-json output shouldn't fail |
| self.run_cli("describe") |
| self.run_cli("describe", "all") |
| _, stdout = self.run_cli("describe", "--json") |
| data = json.loads(stdout) |
| self.assertIn("benchmarks", data) |
| self.assertIn("probes", data) |
| self.assertIsInstance(data["benchmarks"], dict) |
| self.assertIsInstance(data["probes"], dict) |
| |
| def test_describe_benchmarks(self): |
| # Non-json output shouldn't fail |
| self.run_cli("describe", "benchmarks") |
| _, stdout = self.run_cli("describe", "--json", "benchmarks") |
| data = json.loads(stdout) |
| self.assertNotIn("benchmarks", data) |
| self.assertNotIn("probes", data) |
| self.assertIsInstance(data, dict) |
| self.assertIn("loading", data) |
| |
| def test_describe_probes(self): |
| # Non-json output shouldn't fail |
| self.run_cli("describe", "probes") |
| _, stdout = self.run_cli("describe", "--json", "probes") |
| data = json.loads(stdout) |
| self.assertNotIn("benchmarks", data) |
| self.assertNotIn("probes", data) |
| self.assertIsInstance(data, dict) |
| self.assertIn("v8.log", data) |
| |
| def test_help(self): |
| with mock.patch("sys.exit", side_effect=SysExitException) as exit_mock: |
| _, stdout = self.run_cli("--help", raises=SysExitException) |
| self.assertTrue(exit_mock.called) |
| exit_mock.assert_called_with(0) |
| self.assertGreater(len(stdout), 0) |
| |
| def test_help_subcommand(self): |
| for benchmark_cls, aliases in CrossBenchCLI.BENCHMARKS: |
| subcommands = (benchmark_cls.NAME,) + aliases |
| for subcommand in subcommands: |
| with mock.patch( |
| "sys.exit", side_effect=SysExitException()) as exit_mock: |
| stdout = self.run_cli(subcommand, "--help", raises=SysExitException) |
| self.assertTrue(exit_mock.called) |
| exit_mock.assert_called_with(0) |
| self.assertGreater(len(stdout), 0) |
| |
| def test_invalid_probe(self): |
| with self.assertRaises(ValueError), mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", "--probe=invalid_probe_name", "--throw") |
| |
| def test_basic_probe_setting(self): |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| self.run_cli("loading", "--probe=v8.log", f"--urls={url}", |
| "--env-validation=skip", "--throw") |
| for browser in self.browsers: |
| self.assertListEqual([url], browser.url_list[1:]) |
| self.assertIn("--log-all", browser.js_flags) |
| |
| def test_invalid_empty_probe_config_file(self): |
| config_file = pathlib.Path("/config.hjson") |
| config_file.touch() |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| with self.assertRaises(ValueError): |
| self.run_cli("loading", f"--probe-config={config_file}", |
| f"--urls={url}", "--env-validation=skip", "--throw") |
| for browser in self.browsers: |
| self.assertListEqual([], browser.url_list[1:]) |
| self.assertNotIn("--log", browser.js_flags) |
| |
| def test_empty_probe_config_file(self): |
| config_file = pathlib.Path("/config.hjson") |
| config_data = {"probes": {}} |
| with config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| self.run_cli("loading", f"--probe-config={config_file}", f"--urls={url}", |
| "--env-validation=skip") |
| for browser in self.browsers: |
| self.assertListEqual([url], browser.url_list[1:]) |
| self.assertNotIn("--log", browser.js_flags) |
| |
| def test_invalid_probe_config_file(self): |
| config_file = pathlib.Path("/config.hjson") |
| config_data = {"probes": {"invalid probe name": {}}} |
| with config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| with self.assertRaises(ValueError): |
| self.run_cli("loading", f"--probe-config={config_file}", |
| f"--urls={url}", "--env-validation=skip", "--throw") |
| for browser in self.browsers: |
| self.assertListEqual([], browser.url_list) |
| self.assertEqual(len(browser.js_flags), 0) |
| |
| def test_probe_config_file(self): |
| config_file = pathlib.Path("/config.hjson") |
| js_flags = ["--log-foo", "--log-bar"] |
| config_data = {"probes": {"v8.log": {"js_flags": js_flags}}} |
| with config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| self.run_cli("loading", f"--probe-config={config_file}", f"--urls={url}", |
| "--env-validation=skip") |
| for browser in self.browsers: |
| self.assertListEqual([url], browser.url_list[1:]) |
| for flag in js_flags: |
| self.assertIn(flag, browser.js_flags) |
| |
| def test_probe_config_file_invalid_probe(self): |
| config_file = pathlib.Path("/config.hjson") |
| config_data = {"probes": {"invalid probe name": {}}} |
| with config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", |
| return_value=self.browsers), self.assertRaises(ValueError): |
| self.run_cli("loading", f"--probe-config={config_file}", |
| "--urls=http://test.com", "--env-validation=skip", "--throw") |
| |
| def test_invalid_browser_identifier(self): |
| with self.assertRaises(ValueError): |
| self.run_cli( |
| "loading", |
| "--browser=unknown_browser_identifier", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| "--throw", |
| raises=SysExitException) |
| |
| def test_unknown_browser_binary(self): |
| browser_bin = pathlib.Path("/foo/custom/browser.bin") |
| browser_bin.parent.mkdir(parents=True) |
| browser_bin.touch() |
| with self.assertRaises(ValueError): |
| self.run_cli( |
| "loading", |
| f"--browser={browser_bin}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| "--throw", |
| raises=SysExitException) |
| |
| def test_custom_chrome_browser_binary(self): |
| browser_cls = mock_browser.MockChromeStable |
| # TODO: migrate to with_stem once python 3.9 is available everywhere |
| suffix = browser_cls.APP_PATH.suffix |
| browser_bin = browser_cls.APP_PATH.with_name( |
| f"Custom Google Chrome{suffix}") |
| browser_cls.setup_bin(self.fs, browser_bin, "Chrome") |
| |
| with mock.patch.object( |
| BrowserConfig, "_get_browser_cls_from_path", |
| return_value=browser_cls) as get_browser_cls: |
| self.run_cli("loading", f"--browser={browser_bin}", |
| "--urls=http://test.com", "--env-validation=skip") |
| get_browser_cls.assert_called_once_with(browser_bin) |
| |
| def test_custom_chrome_browser_binary_custom_flags(self): |
| browser_cls = mock_browser.MockChromeStable |
| # TODO: migrate to with_stem once python 3.9 is available everywhere |
| suffix = browser_cls.APP_PATH.suffix |
| browser_bin = browser_cls.APP_PATH.with_name( |
| f"Custom Google Chrome{suffix}") |
| browser_cls.setup_bin(self.fs, browser_bin, "Chrome") |
| |
| with mock.patch.object( |
| BrowserConfig, "_get_browser_cls_from_path", |
| return_value=browser_cls), mock.patch.object( |
| CrossBenchCLI, "_run_benchmark") as run_benchmark: |
| self.run_cli("loading", f"--browser={browser_bin}", |
| "--urls=http://test.com", "--env-validation=skip", "--", |
| "--chrome-flag1=value1", "--chrome-flag2") |
| run_benchmark.assert_called_once() |
| runner = run_benchmark.call_args[0][1] |
| self.assertIsInstance(runner, Runner) |
| self.assertEqual(len(runner.browsers), 1) |
| browser = runner.browsers[0] |
| self.assertListEqual(["--chrome-flag1=value1", "--chrome-flag2"], |
| list(browser.flags.get_list())) |
| |
| def test_browser_identifiers(self): |
| browsers: Dict[str, Type[mock_browser.MockBrowser]] = { |
| "chrome": mock_browser.MockChromeStable, |
| "chrome-stable": mock_browser.MockChromeStable, |
| "chr-stable": mock_browser.MockChromeStable, |
| "chrome-beta": mock_browser.MockChromeBeta, |
| "chr-beta": mock_browser.MockChromeBeta, |
| "chrome-dev": mock_browser.MockChromeDev, |
| "edge": mock_browser.MockEdgeStable, |
| "edge-stable": mock_browser.MockEdgeStable, |
| "edge-beta": mock_browser.MockEdgeBeta, |
| "edge-dev": mock_browser.MockEdgeDev, |
| "ff": mock_browser.MockFirefox, |
| "firefox": mock_browser.MockFirefox, |
| "firefox-dev": mock_browser.MockFirefoxDeveloperEdition, |
| "firefox-developer-edition": mock_browser.MockFirefoxDeveloperEdition, |
| "ff-dev": mock_browser.MockFirefoxDeveloperEdition, |
| "firefox-nightly": mock_browser.MockFirefoxNightly, |
| "ff-nightly": mock_browser.MockFirefoxNightly, |
| "ff-trunk": mock_browser.MockFirefoxNightly, |
| } |
| if not self.platform.is_linux: |
| browsers["chr-canary"] = mock_browser.MockChromeCanary |
| browsers["chrome-canary"] = mock_browser.MockChromeCanary |
| browsers["edge-canary"] = mock_browser.MockEdgeCanary |
| if self.platform.is_macos: |
| browsers.update({ |
| "safari": mock_browser.MockSafari, |
| "sf": mock_browser.MockSafari, |
| "safari-technology-preview": mock_browser.MockSafariTechnologyPreview, |
| "sf-tp": mock_browser.MockSafariTechnologyPreview, |
| "tp": mock_browser.MockSafariTechnologyPreview, |
| }) |
| |
| for identifier, browser_cls in browsers.items(): |
| out_dir = self.out_dir / identifier |
| self.assertFalse(out_dir.exists()) |
| with mock.patch.object( |
| BrowserConfig, "_get_browser_cls_from_path", |
| return_value=browser_cls) as get_browser_cls: |
| url = "http://test.com" |
| self.run_cli("loading", f"--browser={identifier}", f"--urls={url}", |
| "--env-validation=skip", f"--out-dir={out_dir}") |
| self.assertTrue(out_dir.exists()) |
| get_browser_cls.assert_called_once() |
| result_file = list(out_dir.glob("**/results.json"))[0] |
| with result_file.open(encoding="utf-8") as f: |
| results = json.load(f) |
| self.assertEqual(results["browser"]["version"], browser_cls.VERSION) |
| self.assertIn("test.com", results["stories"]) |
| |
| def test_browser_identifiers_duplicate(self): |
| with self.assertRaises(ValueError): |
| self.run_cli("loading", "--browser=chrome", "--browser=chrome", |
| "--urls=http://test.com", "--env-validation=skip", "--throw") |
| |
| def test_browser_identifiers_multiple(self): |
| mock_browsers: List[Type[mock_browser.MockBrowser]] = [ |
| mock_browser.MockChromeStable, |
| mock_browser.MockChromeBeta, |
| mock_browser.MockChromeDev, |
| ] |
| |
| def mock_get_browser_cls_from_path(path): |
| for mock_browser_cls in mock_browsers: |
| if mock_browser_cls.APP_PATH == path: |
| return mock_browser_cls |
| raise ValueError("Unknown browser path") |
| |
| with mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path) as get_browser_cls: |
| url = "http://test.com" |
| self.run_cli("loading", "--browser=chrome-beta", |
| "--browser=chrome-stable", "--browser=chrome-dev", |
| f"--urls={url}", "--env-validation=skip", |
| f"--out-dir={self.out_dir}") |
| self.assertTrue(self.out_dir.exists()) |
| get_browser_cls.assert_called() |
| result_files = list(self.out_dir.glob("*/results.json")) |
| self.assertEqual(len(result_files), 3) |
| versions = [] |
| for result_file in result_files: |
| with result_file.open(encoding="utf-8") as f: |
| results = json.load(f) |
| versions.append(results["browser"]["version"]) |
| self.assertIn("test.com", results["stories"]) |
| self.assertTrue(len(set(versions)), 3) |
| for mock_browser_cls in mock_browsers: |
| self.assertIn(mock_browser_cls.VERSION, versions) |
| |
| def test_browser_identifiers_multiple_same_major_version(self): |
| |
| class MockChromeBeta2(mock_browser.MockChromeBeta): |
| VERSION = "100.22.33.100" |
| |
| class MockChromeDev2(mock_browser.MockChromeDev): |
| VERSION = "100.22.33.200" |
| |
| mock_browsers: List[Type[mock_browser.MockBrowser]] = [ |
| MockChromeBeta2, |
| MockChromeDev2, |
| ] |
| |
| def mock_get_browser_cls_from_path(path): |
| for mock_browser_cls in mock_browsers: |
| if mock_browser_cls.APP_PATH == path: |
| return mock_browser_cls |
| raise ValueError("Unknown browser path") |
| |
| with mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path) as get_browser_cls: |
| url = "http://test.com" |
| self.run_cli("loading", "--browser=chrome-dev", "--browser=chrome-beta", |
| f"--urls={url}", "--env-validation=skip", |
| f"--out-dir={self.out_dir}") |
| self.assertTrue(self.out_dir.exists()) |
| get_browser_cls.assert_called() |
| result_files = list(self.out_dir.glob("*/results.json")) |
| self.assertEqual(len(result_files), 2) |
| versions = [] |
| for result_file in result_files: |
| with result_file.open(encoding="utf-8") as f: |
| results = json.load(f) |
| versions.append(results["browser"]["version"]) |
| self.assertIn("test.com", results["stories"]) |
| self.assertTrue(len(set(versions)), 2) |
| for mock_browser_cls in mock_browsers: |
| self.assertIn(mock_browser_cls.VERSION, versions) |
| |
| def test_browser_identifiers_multiple_same_version(self): |
| |
| class MockChromeBeta2(mock_browser.MockChromeBeta): |
| VERSION = "100.22.33.999" |
| |
| class MockChromeDev2(mock_browser.MockChromeDev): |
| VERSION = "100.22.33.999" |
| |
| mock_browsers: List[Type[mock_browser.MockBrowser]] = [ |
| MockChromeBeta2, |
| MockChromeDev2, |
| ] |
| |
| def mock_get_browser_cls_from_path(path): |
| for mock_browser_cls in mock_browsers: |
| if mock_browser_cls.APP_PATH == path: |
| return mock_browser_cls |
| raise ValueError("Unknown browser path") |
| |
| with mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path) as get_browser_cls: |
| url = "http://test.com" |
| self.run_cli("loading", "--browser=chrome-dev", "--browser=chrome-beta", |
| f"--urls={url}", "--env-validation=skip", |
| f"--out-dir={self.out_dir}") |
| self.assertTrue(self.out_dir.exists()) |
| get_browser_cls.assert_called() |
| result_files = list(self.out_dir.glob("*/results.json")) |
| self.assertEqual(len(result_files), 2) |
| versions = [] |
| for result_file in result_files: |
| with result_file.open(encoding="utf-8") as f: |
| results = json.load(f) |
| versions.append(results["browser"]["version"]) |
| self.assertIn("test.com", results["stories"]) |
| self.assertTrue(len(set(versions)), 1) |
| for mock_browser_cls in mock_browsers: |
| self.assertIn(mock_browser_cls.VERSION, versions) |
| |
| def test_probe_invalid_inline_json_config(self): |
| with self.assertRaises(ValueError), mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", "--probe=v8.log{invalid json: d a t a}", |
| "--urls=cnn", "--env-validation=skip", "--throw") |
| |
| def test_probe_empty_inline_json_config(self): |
| js_flags = ["--log-foo", "--log-bar"] |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| self.run_cli("loading", "--probe=v8.log{}", f"--urls={url}", |
| "--env-validation=skip") |
| for browser in self.browsers: |
| self.assertListEqual([url], browser.url_list[1:]) |
| for flag in js_flags: |
| self.assertNotIn(flag, browser.js_flags) |
| |
| def test_probe_inline_json_config(self): |
| js_flags = ["--log-foo", "--log-bar"] |
| json_config = json.dumps({"js_flags": js_flags}) |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| url = "http://test.com" |
| self.run_cli("loading", f"--probe=v8.log{json_config}", f"--urls={url}", |
| "--env-validation=skip") |
| for browser in self.browsers: |
| self.assertListEqual([url], browser.url_list[1:]) |
| for flag in js_flags: |
| self.assertIn(flag, browser.js_flags) |
| |
| def test_env_config_name(self): |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", "--env=strict", "--urls=http://test.com", |
| "--env-validation=skip") |
| |
| def test_env_config_inline_hjson(self): |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", "--env={\"power_use_battery\":false}", |
| "--urls=http://test.com", "--env-validation=skip") |
| |
| def test_env_config_inline_invalid(self): |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| "--env=not a valid name", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| "--env={not valid hjson}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| "--env={unknown_property:1}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| |
| def test_env_config_invalid_file(self): |
| config = pathlib.Path("/test.config.hjson") |
| # No "env" property |
| with config.open("w", encoding="utf-8") as f: |
| hjson.dump({}, f) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| f"--env-config={config}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| # "env" not a dict |
| with config.open("w", encoding="utf-8") as f: |
| hjson.dump({"env": []}, f) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| f"--env-config={config}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| with config.open("w", encoding="utf-8") as f: |
| hjson.dump({"env": {"unknown_property_name": 1}}, f) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| f"--env-config={config}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| |
| def test_multiple_browser_compatible_flags(self): |
| mock_browsers: List[Type[mock_browser.MockBrowser]] = [ |
| mock_browser.MockChromeStable, |
| mock_browser.MockFirefox, |
| mock_browser.MockChromeDev, |
| ] |
| |
| def mock_get_browser_cls_from_path(path): |
| for mock_browser_cls in mock_browsers: |
| if mock_browser_cls.APP_PATH == path: |
| return mock_browser_cls |
| raise ValueError("Unknown browser path") |
| |
| for chrome_flag in ("--js-flags=--no-opt", "--enable-features=Foo", |
| "--disable-features=bar"): |
| # Fail for chrome flags for non-chrome browser |
| with self.assertRaises(ValueError), mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path): |
| self.run_cli("loading", "--urls=http://test.com", |
| "--env-validation=skip", "--throw", "--browser=firefox", |
| chrome_flag) |
| # Fail for mixed browsers and chrome flags |
| with self.assertRaises(ValueError), mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path): |
| self.run_cli("loading", "--urls=http://test.com", |
| "--env-validation=skip", "--throw", "--browser=chrome", |
| "--browser=firefox", chrome_flag) |
| with self.assertRaises(ValueError), mock.patch.object( |
| BrowserConfig, |
| "_get_browser_cls_from_path", |
| side_effect=mock_get_browser_cls_from_path): |
| self.run_cli("loading", "--urls=http://test.com", |
| "--env-validation=skip", "--throw", "--browser=chrome", |
| "--browser=firefox", "--", chrome_flag) |
| # Flags for the same type are allowed. |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", "--urls=http://test.com", "--env-validation=skip", |
| "--throw", "--browser=chrome", "--browser=chrome-dev", "--", |
| "--js-flags=--no-opt") |
| |
| def test_env_config_file(self): |
| config = pathlib.Path("/test.config.hjson") |
| with config.open("w", encoding="utf-8") as f: |
| hjson.dump({"env": {}}, f) |
| with mock.patch.object( |
| CrossBenchCLI, "_get_browsers", return_value=self.browsers): |
| self.run_cli("loading", f"--env-config={config}", |
| "--urls=http://test.com", "--env-validation=skip") |
| |
| def test_env_invalid_inline_and_file(self): |
| config = pathlib.Path("/test.config.hjson") |
| with config.open("w", encoding="utf-8") as f: |
| hjson.dump({"env": {}}, f) |
| with mock.patch("sys.exit", side_effect=SysExitException()): |
| self.run_cli( |
| "loading", |
| "--env=strict", |
| f"--env-config={config}", |
| "--urls=http://test.com", |
| "--env-validation=skip", |
| raises=SysExitException) |
| |
| |
| class TestProbeConfig(pyfakefs.fake_filesystem_unittest.TestCase): |
| # pylint: disable=expression-not-assigned |
| |
| def setUp(self): |
| # TODO: Move to separate common helper class |
| self.setUpPyfakefs(modules_to_reload=[crossbench, mock_browser]) |
| |
| def parse_config(self, config_data) -> ProbeConfig: |
| probe_config_file = pathlib.Path("/probe.config.hjson") |
| with probe_config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| with probe_config_file.open(encoding="utf-8") as f: |
| return ProbeConfig.load(f) |
| |
| def test_invalid_empty(self): |
| with self.assertRaises(ValueError): |
| self.parse_config({}).probes |
| with self.assertRaises(ValueError): |
| self.parse_config({"foo": {}}).probes |
| |
| def test_invalid_names(self): |
| with self.assertRaises(ValueError): |
| self.parse_config({"probes": {"invalid probe name": {}}}).probes |
| |
| def test_empty(self): |
| config = self.parse_config({"probes": {}}) |
| self.assertListEqual(config.probes, []) |
| |
| def test_single_v8_log(self): |
| js_flags = ["--log-maps", "--log-function-events"] |
| config = self.parse_config( |
| {"probes": { |
| "v8.log": { |
| "prof": True, |
| "js_flags": js_flags, |
| } |
| }}) |
| self.assertTrue(len(config.probes), 1) |
| probe = config.probes[0] |
| assert isinstance(probe, V8LogProbe) |
| for flag in js_flags + ["--prof"]: |
| self.assertIn(flag, probe.js_flags) |
| |
| def test_from_cli_args(self): |
| file = pathlib.Path("probe.config.hjson") |
| js_flags = ["--log-maps", "--log-function-events"] |
| config_data = { |
| "probes": { |
| "v8.log": { |
| "prof": True, |
| "js_flags": js_flags, |
| } |
| } |
| } |
| with file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| args = mock.Mock(probe_config=file) |
| config = ProbeConfig.from_cli_args(args) |
| self.assertTrue(len(config.probes), 1) |
| probe = config.probes[0] |
| self.assertTrue(isinstance(probe, V8LogProbe)) |
| for flag in js_flags + ["--prof"]: |
| self.assertIn(flag, probe.js_flags) |
| |
| def test_inline_config(self): |
| mock_d8_file = pathlib.Path("out/d8") |
| self.fs.create_file(mock_d8_file) |
| config_data = {"d8_binary": str(mock_d8_file)} |
| args = mock.Mock( |
| probe=[f"v8.log{hjson.dumps(config_data)}"], |
| probe_config=None, |
| throw=True, |
| wraps=False) |
| config = ProbeConfig.from_cli_args(args) |
| self.assertTrue(len(config.probes), 1) |
| probe = config.probes[0] |
| self.assertTrue(isinstance(probe, V8LogProbe)) |
| |
| def test_inline_config_dir_instead_of_file(self): |
| mock_dir = pathlib.Path("some/dir") |
| mock_dir.mkdir(parents=True) |
| config_data = {"d8_binary": str(mock_dir)} |
| args = mock.Mock( |
| probe=[f"v8.log{hjson.dumps(config_data)}"], |
| probe_config=None, |
| throw=True, |
| wraps=False) |
| with self.assertRaises(ValueError) as cm: |
| ProbeConfig.from_cli_args(args) |
| self.assertIn(str(mock_dir), str(cm.exception)) |
| |
| def test_inline_config_non_existent_file(self): |
| config_data = {"d8_binary": "does/not/exist/d8"} |
| args = mock.Mock( |
| probe=[f"v8.log{hjson.dumps(config_data)}"], |
| probe_config=None, |
| throw=True, |
| wraps=False) |
| with self.assertRaises(ValueError) as cm: |
| ProbeConfig.from_cli_args(args) |
| self.assertIn("does/not/exist/d8", str(cm.exception)) |
| |
| def test_multiple_probes(self): |
| powersampler_bin = pathlib.Path("/powersampler.bin") |
| powersampler_bin.touch() |
| config = self.parse_config({ |
| "probes": { |
| "v8.log": { |
| "log_all": True, |
| }, |
| "powersampler": { |
| "bin_path": str(powersampler_bin) |
| } |
| } |
| }) |
| self.assertTrue(len(config.probes), 2) |
| log_probe = config.probes[0] |
| assert isinstance(log_probe, V8LogProbe) |
| powersampler_probe = config.probes[1] |
| assert isinstance(powersampler_probe, PowerSamplerProbe) |
| self.assertEqual(powersampler_probe.bin_path, powersampler_bin) |
| |
| |
| class TestBrowserConfig(BaseCrossbenchTestCase): |
| # pylint: disable=expression-not-assigned |
| |
| EXAMPLE_CONFIG_PATH = pathlib.Path( |
| __file__).parent.parent / "config" / "browser.config.example.hjson" |
| |
| def setUp(self): |
| super().setUp() |
| self.browser_lookup: Dict[ |
| str, Tuple[Type[mock_browser.MockBrowser], pathlib.Path]] = { |
| "chr-stable": (mock_browser.MockChromeStable, |
| mock_browser.MockChromeStable.APP_PATH), |
| "chr-dev": (mock_browser.MockChromeDev, |
| mock_browser.MockChromeDev.APP_PATH), |
| "chrome-stable": (mock_browser.MockChromeStable, |
| mock_browser.MockChromeStable.APP_PATH), |
| "chrome-dev": (mock_browser.MockChromeDev, |
| mock_browser.MockChromeDev.APP_PATH), |
| } |
| for _, (_, browser_path) in self.browser_lookup.items(): |
| self.assertTrue(browser_path.exists()) |
| |
| @unittest.skipIf(hjson.__name__ != "hjson", "hjson not available") |
| def test_load_browser_config_template(self): |
| if not self.EXAMPLE_CONFIG_PATH.exists(): |
| raise unittest.SkipTest( |
| f"Test file {self.EXAMPLE_CONFIG_PATH} does not exist") |
| self.fs.add_real_file(self.EXAMPLE_CONFIG_PATH) |
| with self.EXAMPLE_CONFIG_PATH.open(encoding="utf-8") as f: |
| config = BrowserConfig(browser_lookup_override=self.browser_lookup) |
| config.load(f) |
| self.assertIn("flag-group-1", config.flag_groups) |
| self.assertGreaterEqual(len(config.flag_groups), 1) |
| self.assertGreaterEqual(len(config.variants), 1) |
| |
| def test_flag_combination_invalid(self): |
| with self.assertRaises(ConfigFileError) as cm: |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "invalid-flag-name": [None, "", "v1"], |
| }, |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1",] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| self.assertIn("group1", str(cm.exception)) |
| self.assertIn("invalid-flag-name", str(cm.exception)) |
| |
| def test_flag_combination_none(self): |
| with self.assertRaises(ConfigFileError) as cm: |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--foo": ["None,", "", "v1"], |
| }, |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| self.assertIn("None", str(cm.exception)) |
| |
| def test_flag_combination_duplicate(self): |
| with self.assertRaises(ValueError) as cm: |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--duplicate-flag": [None, "", "v1"], |
| }, |
| "group2": { |
| "--duplicate-flag": [None, "", "v1"], |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1", "group2"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| self.assertIn("--duplicate-flag", str(cm.exception)) |
| |
| def test_empty(self): |
| with self.assertRaises(ValueError): |
| BrowserConfig({"other": {}}).variants |
| with self.assertRaises(ValueError): |
| BrowserConfig({"browsers": {}}).variants |
| |
| def test_unknown_group(self): |
| with self.assertRaises(ValueError) as cm: |
| BrowserConfig({ |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["unknown-flag-group"] |
| } |
| } |
| }).variants |
| self.assertIn("unknown-flag-group", str(cm.exception)) |
| |
| def test_duplicate_group(self): |
| with self.assertRaises(ConfigFileError): |
| BrowserConfig({ |
| "flags": { |
| "group1": {} |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1", "group1"] |
| } |
| } |
| }).variants |
| |
| def test_non_list_group(self): |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": {} |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": "group1" |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| with self.assertRaises(ConfigFileError) as cm: |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": {} |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": 1 |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| self.assertIn("chrome-stable", str(cm.exception)) |
| self.assertIn("flags", str(cm.exception)) |
| |
| with self.assertRaises(ConfigFileError) as cm: |
| BrowserConfig( |
| { |
| "flags": { |
| "group1": {} |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": { |
| "group1": True |
| } |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup).variants |
| self.assertIn("chrome-stable", str(cm.exception)) |
| self.assertIn("flags", str(cm.exception)) |
| |
| def test_duplicate_flag_variant_value(self): |
| with self.assertRaises(ConfigFileError) as cm: |
| BrowserConfig({ |
| "flags": { |
| "group1": { |
| "--flag": ["repeated", "repeated"] |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": "group1", |
| } |
| } |
| }).variants |
| self.assertIn("group1", str(cm.exception)) |
| self.assertIn("--flag", str(cm.exception)) |
| |
| def test_unknown_path(self): |
| with self.assertRaises(Exception): |
| BrowserConfig({ |
| "browsers": { |
| "chrome-stable": { |
| "path": "path/does/not/exist", |
| } |
| } |
| }).variants |
| with self.assertRaises(Exception): |
| BrowserConfig({ |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-unknown", |
| } |
| } |
| }).variants |
| |
| def test_flag_combination_simple(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--foo": [None, "", "v1"], |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| browsers = config.variants |
| self.assertEqual(len(browsers), 3) |
| for browser in browsers: |
| assert isinstance(browser, mock_browser.MockChromeStable) |
| self.assertDictEqual(dict(browser.js_flags), {}) |
| self.assertDictEqual(dict(browsers[0].flags), {}) |
| self.assertDictEqual(dict(browsers[1].flags), {"--foo": None}) |
| self.assertDictEqual(dict(browsers[2].flags), {"--foo": "v1"}) |
| |
| def test_flag_combination(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--foo": [None, "", "v1"], |
| "--bar": [None, "", "v1"], |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| self.assertEqual(len(config.variants), 3 * 3) |
| |
| def test_flag_combination_mixed_inline(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "compile-hints-experiment": { |
| "--enable-features": [None, "ConsumeCompileHints"] |
| } |
| }, |
| "browsers": { |
| "chrome-release": { |
| "path": "chrome-stable", |
| "flags": ["--no-sandbox", "compile-hints-experiment"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| browsers = config.variants |
| self.assertEqual(len(browsers), 2) |
| self.assertListEqual(["--no-sandbox"], list(browsers[0].flags.get_list())) |
| self.assertListEqual( |
| ["--no-sandbox", "--enable-features=ConsumeCompileHints"], |
| list(browsers[1].flags.get_list())) |
| |
| def test_flag_single_inline(self): |
| config = BrowserConfig( |
| { |
| "browsers": { |
| "chrome-release": { |
| "path": "chrome-stable", |
| "flags": "--no-sandbox", |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| browsers = config.variants |
| self.assertEqual(len(browsers), 1) |
| self.assertListEqual(["--no-sandbox"], list(browsers[0].flags.get_list())) |
| |
| def test_flag_combination_mixed_fixed(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "compile-hints-experiment": { |
| "--no-sandbox": "", |
| "--enable-features": [None, "ConsumeCompileHints"] |
| } |
| }, |
| "browsers": { |
| "chrome-release": { |
| "path": "chrome-stable", |
| "flags": "compile-hints-experiment" |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| browsers = config.variants |
| self.assertEqual(len(browsers), 2) |
| self.assertListEqual(["--no-sandbox"], list(browsers[0].flags.get_list())) |
| self.assertListEqual( |
| ["--no-sandbox", "--enable-features=ConsumeCompileHints"], |
| list(browsers[1].flags.get_list())) |
| |
| def test_no_flags(self): |
| config = BrowserConfig( |
| { |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| }, |
| "chrome-dev": { |
| "path": "chrome-dev", |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| self.assertEqual(len(config.variants), 2) |
| browser_0 = config.variants[0] |
| assert isinstance(browser_0, mock_browser.MockChromeStable) |
| self.assertEqual(browser_0.app_path, mock_browser.MockChromeStable.APP_PATH) |
| browser_1 = config.variants[1] |
| assert isinstance(browser_1, mock_browser.MockChromeDev) |
| self.assertEqual(browser_1.app_path, mock_browser.MockChromeDev.APP_PATH) |
| |
| def test_inline_flags(self): |
| with mock.patch.object( |
| ChromeWebDriver, "_extract_version", |
| return_value="101.22.333.44"), mock.patch.object( |
| Chrome, |
| "stable_path", |
| return_value=mock_browser.MockChromeStable.APP_PATH): |
| |
| config = BrowserConfig({ |
| "browsers": { |
| "stable": { |
| "path": "chrome-stable", |
| "flags": ["--foo=bar"] |
| } |
| } |
| }) |
| self.assertEqual(len(config.variants), 1) |
| browser = config.variants[0] |
| # TODO: Fix once app lookup is cleaned up |
| self.assertEqual(browser.app_path, mock_browser.MockChromeStable.APP_PATH) |
| self.assertEqual(browser.version, "101.22.333.44") |
| |
| def test_inline_load_safari(self): |
| if not helper.platform.is_macos: |
| return |
| with mock.patch.object(Safari, "_extract_version", return_value="16.0"): |
| config = BrowserConfig({"browsers": {"safari": {"path": "safari",}}}) |
| self.assertEqual(len(config.variants), 1) |
| |
| def test_flag_combination_with_fixed(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--foo": [None, "", "v1"], |
| "--bar": [None, "", "v1"], |
| "--always_1": "true", |
| "--always_2": "true", |
| "--always_3": "true", |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| self.assertEqual(len(config.variants), 3 * 3) |
| for browser in config.variants: |
| assert isinstance(browser, mock_browser.MockChromeStable) |
| self.assertEqual(browser.app_path, mock_browser.MockChromeStable.APP_PATH) |
| |
| def test_flag_group_combination(self): |
| config = BrowserConfig( |
| { |
| "flags": { |
| "group1": { |
| "--foo": [None, "", "v1"], |
| }, |
| "group2": { |
| "--bar": [None, "", "v1"], |
| }, |
| "group3": { |
| "--other": ["v1", "v2"], |
| } |
| }, |
| "browsers": { |
| "chrome-stable": { |
| "path": "chrome-stable", |
| "flags": ["group1", "group2", "group3"] |
| } |
| } |
| }, |
| browser_lookup_override=self.browser_lookup) |
| self.assertEqual(len(config.variants), 3 * 3 * 2) |
| |
| def test_from_cli_args_browser_config(self): |
| browser_cls = mock_browser.MockChromeStable |
| # TODO: migrate to with_stem once python 3.9 is available everywhere |
| suffix = browser_cls.APP_PATH.suffix |
| browser_bin = browser_cls.APP_PATH.with_name( |
| f"Custom Google Chrome{suffix}") |
| browser_cls.setup_bin(self.fs, browser_bin, "Chrome") |
| config_data = {"browsers": {"chrome-stable": {"path": str(browser_bin),}}} |
| config_file = pathlib.Path("config.hjson") |
| with config_file.open("w", encoding="utf-8") as f: |
| hjson.dump(config_data, f) |
| |
| args = mock.Mock(browser=None, browser_config=config_file) |
| with mock.patch.object( |
| BrowserConfig, "_get_browser_cls_from_path", return_value=browser_cls): |
| config = BrowserConfig.from_cli_args(args) |
| self.assertEqual(len(config.variants), 1) |
| browser = config.variants[0] |
| self.assertIsInstance(browser, browser_cls) |
| self.assertEqual(browser.app_path, browser_bin) |
| |
| def test_from_cli_args_browser(self): |
| browser_cls = mock_browser.MockChromeStable |
| # TODO: migrate to with_stem once python 3.9 is available everywhere |
| suffix = browser_cls.APP_PATH.suffix |
| browser_bin = browser_cls.APP_PATH.with_name( |
| f"Custom Google Chrome{suffix}") |
| browser_cls.setup_bin(self.fs, browser_bin, "Chrome") |
| args = mock.Mock( |
| browser=[ |
| str(browser_bin), |
| ], |
| browser_config=None, |
| enable_features=None, |
| disable_features=None, |
| js_flags=None, |
| other_browser_args=[]) |
| with mock.patch.object( |
| BrowserConfig, "_get_browser_cls_from_path", return_value=browser_cls): |
| config = BrowserConfig.from_cli_args(args) |
| self.assertEqual(len(config.variants), 1) |
| browser = config.variants[0] |
| self.assertIsInstance(browser, browser_cls) |
| self.assertEqual(browser.app_path, browser_bin) |
| |
| |
| class TestFlagGroupConfig(unittest.TestCase): |
| |
| def parse(self, config_dict): |
| config = FlagGroupConfig("test", config_dict) |
| variants = list(config.get_variant_items()) |
| return variants |
| |
| def test_empty(self): |
| config = FlagGroupConfig("empty_name", {}) |
| self.assertEqual(config.name, "empty_name") |
| variants = list(config.get_variant_items()) |
| self.assertEqual(len(variants), 0) |
| |
| def test_single_flag(self): |
| variants = self.parse({"--foo": set()}) |
| self.assertListEqual(variants, [ |
| (), |
| ]) |
| |
| variants = self.parse({"--foo": []}) |
| self.assertListEqual(variants, [ |
| (), |
| ]) |
| |
| variants = self.parse({"--foo": (None,)}) |
| self.assertListEqual(variants, [ |
| (None,), |
| ]) |
| |
| variants = self.parse({"--foo": ("",)}) |
| self.assertEqual(len(variants), 1) |
| self.assertTupleEqual( |
| variants[0], |
| (("--foo", None),), |
| ) |
| |
| variants = self.parse({"--foo": ( |
| "", |
| None, |
| )}) |
| self.assertEqual(len(variants), 1) |
| self.assertTupleEqual(variants[0], (("--foo", None), None)) |
| |
| variants = self.parse({"--foo": ( |
| "v1", |
| "v2", |
| "", |
| None, |
| )}) |
| self.assertEqual(len(variants), 1) |
| self.assertTupleEqual(variants[0], (("--foo", "v1"), ("--foo", "v2"), |
| ("--foo", None), None)) |
| |
| def test_two_flags(self): |
| variants = self.parse({"--foo": [], "--bar": []}) |
| self.assertListEqual(variants, [(), ()]) |
| |
| variants = self.parse({"--foo": "a", "--bar": "b"}) |
| self.assertEqual(len(variants), 2) |
| self.assertTupleEqual(variants[0], (("--foo", "a"),)) |
| self.assertTupleEqual(variants[1], (("--bar", "b"),)) |
| |
| variants = self.parse({"--foo": ["a1", "a2"], "--bar": "b"}) |
| self.assertEqual(len(variants), 2) |
| self.assertTupleEqual(variants[0], ( |
| ("--foo", "a1"), |
| ("--foo", "a2"), |
| )) |
| self.assertTupleEqual(variants[1], (("--bar", "b"),)) |
| |
| variants = self.parse({"--foo": ["a1", "a2"], "--bar": ["b1", "b2"]}) |
| self.assertEqual(len(variants), 2) |
| self.assertTupleEqual(variants[0], ( |
| ("--foo", "a1"), |
| ("--foo", "a2"), |
| )) |
| self.assertTupleEqual(variants[1], ( |
| ("--bar", "b1"), |
| ("--bar", "b2"), |
| )) |
| |
| |
| if __name__ == "__main__": |
| sys.exit(pytest.main([__file__])) |
