| # Copyright 2016 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # It's reasonable for unittests to be messing with protected members. |
| # Additionally, there is a lot to test, so line count is expected to be large. |
| # pylint: disable=protected-access,too-many-lines |
| |
| |
| import copy |
| import dataclasses |
| import json |
| import os |
| import tempfile |
| import typing |
| from typing import Any, Type |
| import unittest |
| from unittest import mock |
| |
| # Needed in order for the tools/perf and Telemetry imports to work. |
| # pylint: disable=unused-import,wrong-import-order |
| from gpu_path_util import setup_telemetry_paths |
| from gpu_path_util import setup_tools_perf_paths |
| # pylint: enable=unused-import,wrong-import-order |
| from chrome_telemetry_build import chromium_config |
| from py_utils import tempfile_ext |
| from telemetry.internal.browser import browser_options as bo |
| from telemetry.internal.util import binary_manager |
| from telemetry.internal.platform import system_info |
| from telemetry.testing import browser_test_runner |
| from telemetry.testing import fakes |
| from telemetry.testing import run_browser_tests |
| |
| import gpu_path_util |
| import gpu_project_config |
| from gpu_tests import common_typing as ct |
| from gpu_tests import context_lost_integration_test |
| from gpu_tests import gpu_helper |
| from gpu_tests import gpu_integration_test |
| from gpu_tests import trace_integration_test as trace_it |
| from gpu_tests import webgl1_conformance_integration_test as webgl1_cit |
| from gpu_tests import webgl2_conformance_integration_test as webgl2_cit |
| from gpu_tests import webgpu_compat_cts_integration_test as webgpu_compat_cit |
| import run_gpu_integration_test |
| |
| # Unittest test cases are defined as public methods, so ignore complaints about |
| # having too many. |
| # pylint: disable=too-many-public-methods |
| |
| VENDOR_NVIDIA = 0x10DE |
| VENDOR_AMD = 0x1002 |
| VENDOR_INTEL = 0x8086 |
| |
| VENDOR_STRING_IMAGINATION = 'Imagination Technologies' |
| DEVICE_STRING_SGX = 'PowerVR SGX 554' |
| |
| GpuTestClassType = Type[gpu_integration_test.GpuIntegrationTest] |
| |
| |
| def _GetSystemInfo( # pylint: disable=too-many-arguments |
| gpu: int = 0, |
| device: int = 0, |
| vendor_string: str = '', |
| device_string: str = '', |
| passthrough: bool = False, |
| gl_renderer: str = '', |
| is_asan: bool = False, |
| is_clang_coverage: bool = False, |
| target_cpu_bits: int = 64) -> system_info.SystemInfo: |
| sys_info = { |
| 'model_name': '', |
| 'gpu': { |
| 'devices': [ |
| { |
| 'vendor_id': gpu, |
| 'device_id': device, |
| 'vendor_string': vendor_string, |
| 'device_string': device_string |
| }, |
| ], |
| 'aux_attributes': { |
| 'passthrough_cmd_decoder': passthrough, |
| 'is_asan': is_asan, |
| 'is_clang_coverage': is_clang_coverage, |
| 'target_cpu_bits': target_cpu_bits |
| }, |
| 'feature_status': { |
| 'gpu_compositing': 'enabled', |
| 'opengl': 'enabled_on' |
| } |
| } |
| } |
| if gl_renderer: |
| sys_info['gpu']['aux_attributes']['gl_renderer'] = gl_renderer |
| return system_info.SystemInfo.FromDict(sys_info) |
| |
| |
| def _GetTagsToTest(browser: fakes.FakeBrowser, |
| test_class: GpuTestClassType | None = None) -> set[str]: |
| browser = typing.cast(ct.Browser, browser) |
| test_class = test_class or gpu_integration_test.GpuIntegrationTest |
| tags = None |
| with mock.patch.object( |
| test_class, 'ExpectationsFiles', return_value=['exp.txt']): |
| tags = set(test_class.GetPlatformTags(browser)) |
| return tags |
| |
| |
| def _GenerateNvidiaExampleTagsForTestClassAndArgs( |
| test_class: GpuTestClassType, |
| args: mock.MagicMock, |
| is_asan: bool = False, |
| is_clang_coverage: bool = False, |
| target_cpu_bits: int = 64, |
| ) -> set[str]: |
| tags = None |
| with mock.patch.object( |
| test_class, 'ExpectationsFiles', return_value=['exp.txt']): |
| _ = list(test_class.GenerateTestCases__RunGpuTest(args)) |
| platform = fakes.FakePlatform('win', 'win10') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, |
| device=0x1cb3, |
| gl_renderer='ANGLE Direct3D9', |
| is_asan=is_asan, |
| is_clang_coverage=is_clang_coverage, |
| target_cpu_bits=target_cpu_bits) |
| tags = _GetTagsToTest(browser, test_class) |
| return tags |
| |
| |
| @dataclasses.dataclass |
| class _IntegrationTestArgs(): |
| """Struct-like object for defining an integration test.""" |
| test_name: str |
| failures: list[str] = ct.EmptyList() |
| successes: list[str] = ct.EmptyList() |
| skips: list[str] = ct.EmptyList() |
| additional_args: list[str] = ct.EmptyList() |
| |
| |
| class GpuIntegrationTestUnittest(unittest.TestCase): |
| def setUp(self) -> None: |
| self._test_state = {} |
| self._test_result = {} |
| |
| def _RunGpuIntegrationTests(self, |
| test_name: str, |
| extra_args: list[str] | None = None) -> None: |
| extra_args = extra_args or [] |
| unittest_config = chromium_config.ChromiumConfig( |
| top_level_dir=gpu_path_util.GPU_DIR, |
| benchmark_dirs=[os.path.join(gpu_path_util.GPU_DIR, 'unittest_data')]) |
| with binary_manager.TemporarilyReplaceBinaryManager(None), \ |
| mock.patch.object(gpu_project_config, 'CONFIG', unittest_config): |
| # TODO(crbug.com/40139419): Using NamedTemporaryFile() as a generator is |
| # causing windows bots to fail. When the issue is fixed with |
| # tempfile_ext.NamedTemporaryFile(), put it in the list of generators |
| # starting this with block. Also remove the try finally statement |
| # below. |
| with tempfile.NamedTemporaryFile(delete=False) as temp_file: |
| temp_file_name = temp_file.name |
| try: |
| test_argv = [ |
| test_name, |
| f'--write-full-results-to={temp_file_name}', |
| # We don't want the underlying typ-based tests to report their |
| # results to ResultDB. |
| '--disable-resultsink', |
| # These tests currently rely on some information sticking around |
| # between tests, so we need to use the older global process pool |
| # approach instead of having different pools scoped for |
| # parallel/serial execution. |
| '--use-global-pool', |
| ] + extra_args |
| processed_args = run_gpu_integration_test.ProcessArgs(test_argv) |
| telemetry_args = browser_test_runner.ProcessConfig( |
| unittest_config, processed_args) |
| run_browser_tests.RunTests(telemetry_args) |
| with open(temp_file_name, encoding='utf-8') as f: |
| self._test_result = json.load(f) |
| finally: |
| os.remove(temp_file_name) |
| |
| def testOverrideDefaultRetryArgumentsinRunGpuIntegrationTests(self) -> None: |
| self._RunGpuIntegrationTests('run_tests_with_expectations_files', |
| ['--retry-limit=1']) |
| self.assertEqual( |
| self._test_result['tests']['a']['b']['unexpected-fail.html']['actual'], |
| 'FAIL FAIL') |
| |
| def testDefaultRetryArgumentsinRunGpuIntegrationTests(self) -> None: |
| self._RunGpuIntegrationTests('run_tests_with_expectations_files') |
| self.assertEqual( |
| self._test_result['tests']['a']['b']['expected-flaky.html']['actual'], |
| 'FAIL FAIL FAIL') |
| |
| def testTestNamePrefixGenerationInRunGpuIntegrationTests(self) -> None: |
| self._RunGpuIntegrationTests('simple_integration_unittest') |
| self.assertIn('expected_failure', self._test_result['tests']) |
| |
| # pylint: disable=too-many-arguments |
| def _TestTagGenerationForMockPlatform(self, |
| test_class: GpuTestClassType, |
| args: mock.MagicMock, |
| is_asan: bool = False, |
| is_clang_coverage: bool = False, |
| target_cpu_bits: int = 64) -> set[str]: |
| tag_set = _GenerateNvidiaExampleTagsForTestClassAndArgs( |
| test_class, args, is_asan, is_clang_coverage, target_cpu_bits) |
| self.assertTrue( |
| set([ |
| 'win', 'win10', 'angle-d3d9', 'release', 'nvidia', 'nvidia-0x1cb3', |
| 'no-passthrough' |
| ]).issubset(tag_set)) |
| return tag_set |
| # pylint: enable=too-many-arguments |
| |
| def testGenerateContextLostExampleTagsForAsan(self) -> None: |
| args = gpu_helper.GetMockArgs() |
| tag_set = self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| is_asan=True) |
| self.assertIn('asan', tag_set) |
| self.assertNotIn('no-asan', tag_set) |
| |
| def testGenerateContextLostExampleTagsForNoAsan(self) -> None: |
| args = gpu_helper.GetMockArgs() |
| tag_set = self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| is_asan=False) |
| self.assertIn('no-asan', tag_set) |
| self.assertNotIn('asan', tag_set) |
| |
| def testGenerateContextLostExampleTagsForClangCoverage(self) -> None: |
| args = gpu_helper.GetMockArgs() |
| tag_set = self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| is_clang_coverage=True) |
| self.assertIn('clang-coverage', tag_set) |
| self.assertNotIn('no-clang-coverage', tag_set) |
| |
| def testGenerateContextLostExampleTagsForNoClangCoverage(self) -> None: |
| args = gpu_helper.GetMockArgs() |
| tag_set = self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| is_clang_coverage=False) |
| self.assertIn('no-clang-coverage', tag_set) |
| self.assertNotIn('clang-coverage', tag_set) |
| |
| def testGenerateContextLostExampleTagsForTargetCpu(self) -> None: |
| args = gpu_helper.GetMockArgs() |
| self.assertIn( |
| 'target-cpu-64', |
| self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| target_cpu_bits=64)) |
| self.assertIn( |
| 'target-cpu-32', |
| self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| target_cpu_bits=32)) |
| self.assertIn( |
| 'target-cpu-31', |
| self._TestTagGenerationForMockPlatform( |
| context_lost_integration_test.ContextLostIntegrationTest, |
| args, |
| target_cpu_bits=31)) |
| |
| def testGenerateWebglConformanceExampleTagsForAsan(self) -> None: |
| args = gpu_helper.GetMockArgs(webgl_version='1.0.0') |
| tag_set = self._TestTagGenerationForMockPlatform( |
| webgl1_cit.WebGL1ConformanceIntegrationTest, args, is_asan=True) |
| self.assertTrue(set(['asan']).issubset(tag_set)) |
| self.assertFalse(set(['no-asan']) & tag_set) |
| |
| def testGenerateWebglConformanceExampleTagsForNoAsan(self) -> None: |
| args = gpu_helper.GetMockArgs(webgl_version='2.0.0') |
| tag_set = self._TestTagGenerationForMockPlatform( |
| webgl2_cit.WebGL2ConformanceIntegrationTest, args) |
| self.assertTrue(set(['no-asan']).issubset(tag_set)) |
| self.assertFalse(set(['asan']) & tag_set) |
| |
| def testWebGlConformanceTimeoutNoAsan(self) -> None: |
| instance = webgl1_cit.WebGL1ConformanceIntegrationTest( |
| '_RunConformanceTest') |
| instance._is_asan = False |
| self.assertEqual(instance._GetTestTimeout(), 300) |
| |
| def testWebGlConformanceTimeoutAsan(self) -> None: |
| instance = webgl1_cit.WebGL1ConformanceIntegrationTest( |
| '_RunConformanceTest') |
| instance._is_asan = True |
| self.assertEqual(instance._GetTestTimeout(), 600) |
| |
| def testAsanClassMemberSetCorrectly(self): |
| test_class = gpu_integration_test.GpuIntegrationTest |
| platform = fakes.FakePlatform('win', 'win10') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser = typing.cast(ct.Browser, browser) |
| |
| browser._returned_system_info = _GetSystemInfo(is_asan=True) |
| with mock.patch.object(test_class, |
| 'ExpectationsFiles', |
| return_value=['exp.txt']): |
| test_class.GetPlatformTags(browser) |
| self.assertTrue(test_class._is_asan) |
| |
| browser._returned_system_info = _GetSystemInfo(is_asan=False) |
| with mock.patch.object(test_class, |
| 'ExpectationsFiles', |
| return_value=['exp.txt']): |
| test_class.GetPlatformTags(browser) |
| self.assertFalse(test_class._is_asan) |
| |
| @mock.patch('gpu_tests.util.host_information.IsLinux', return_value=False) |
| @mock.patch('gpu_tests.util.host_information.GetSystemMemoryBytes', |
| return_value=32_000_000_000) |
| @mock.patch('gpu_tests.util.host_information.IsArmCpu', |
| return_value=False) |
| def testGenerateNvidiaExampleTags(self, _, __, ___) -> None: |
| platform = fakes.FakePlatform('win', 'win10') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, device=0x1cb3, gl_renderer='ANGLE Direct3D9') |
| self.assertEqual( |
| _GetTagsToTest(browser), |
| set([ |
| 'win', |
| 'win10', |
| 'release', |
| 'nvidia', |
| 'nvidia-0x1cb3', |
| 'angle-d3d9', |
| 'no-passthrough', |
| 'renderer-skia-gl', |
| 'no-asan', |
| 'arch-x86_64', |
| 'target-cpu-64', |
| 'no-clang-coverage', |
| 'graphite-disabled', |
| 'memory_ge_16gb', |
| ])) |
| |
| @mock.patch('gpu_tests.util.host_information.IsLinux', return_value=False) |
| @mock.patch('gpu_tests.util.host_information.GetSystemMemoryBytes', |
| return_value=32_000_000_000) |
| @mock.patch('gpu_tests.util.host_information.IsArmCpu', |
| return_value=False) |
| def testGenerateVendorTagUsingVendorString(self, _, __, ___) -> None: |
| platform = fakes.FakePlatform('mac', 'mojave') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| vendor_string=VENDOR_STRING_IMAGINATION, |
| device_string=DEVICE_STRING_SGX, |
| passthrough=True, |
| gl_renderer='ANGLE OpenGL ES') |
| self.assertEqual( |
| _GetTagsToTest(browser), |
| set([ |
| 'mac', |
| 'mojave', |
| 'release', |
| 'imagination', |
| 'no-asan', |
| 'arch-x86_64', |
| 'target-cpu-64', |
| 'imagination-PowerVR-SGX-554', |
| 'angle-opengles', |
| 'passthrough', |
| 'renderer-skia-gl', |
| 'no-clang-coverage', |
| 'graphite-disabled', |
| 'memory_ge_16gb', |
| ])) |
| |
| @mock.patch('gpu_tests.util.host_information.IsLinux', return_value=False) |
| @mock.patch('gpu_tests.util.host_information.GetSystemMemoryBytes', |
| return_value=32_000_000_000) |
| @mock.patch('gpu_tests.util.host_information.IsArmCpu', |
| return_value=False) |
| def testGenerateVendorTagUsingDeviceString(self, _, __, ___) -> None: |
| platform = fakes.FakePlatform('mac', 'mojave') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| vendor_string='illegal vendor string', |
| device_string='ANGLE (Imagination, Triangle Monster 3000, 1.0)') |
| self.assertEqual( |
| _GetTagsToTest(browser), |
| set([ |
| 'mac', |
| 'mojave', |
| 'release', |
| 'imagination', |
| 'no-asan', |
| 'arch-x86_64', |
| 'target-cpu-64', |
| 'imagination-Triangle-Monster-3000', |
| 'angle-disabled', |
| 'no-passthrough', |
| 'renderer-skia-gl', |
| 'no-clang-coverage', |
| 'graphite-disabled', |
| 'memory_ge_16gb', |
| ])) |
| |
| @mock.patch('gpu_tests.util.host_information.IsLinux', return_value=False) |
| def testGenerateMemoryTags(self, _): |
| platform = fakes.FakePlatform('win', 'win10') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, device=0x1cb3, gl_renderer='ANGLE Direct3D9') |
| |
| with mock.patch('gpu_tests.util.host_information.GetSystemMemoryBytes', |
| return_value=8_000_000_000): |
| tags = _GetTagsToTest(browser) |
| self.assertIn('memory_lt_16gb', tags) |
| self.assertNotIn('memory_ge_16gb', tags) |
| |
| with mock.patch('gpu_tests.util.host_information.GetSystemMemoryBytes', |
| return_value=16_000_000_000): |
| tags = _GetTagsToTest(browser) |
| self.assertNotIn('memory_lt_16gb', tags) |
| self.assertIn('memory_ge_16gb', tags) |
| |
| # Ensure that tags aren't generated on remote platforms. |
| platform = fakes.FakePlatform('android', 'android-15') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, device=0x0000, gl_renderer='ANGLE OpenGL ES') |
| tags = _GetTagsToTest(browser) |
| for t in tags: |
| self.assertFalse(t.startswith('memory_')) |
| |
| @mock.patch('gpu_tests.util.host_information.IsLinux', return_value=False) |
| def testGenerateArchitectureTags(self, _): |
| platform = fakes.FakePlatform('win', 'win10') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, device=0x1cb3, gl_renderer='ANGLE Direct3D9') |
| |
| with mock.patch('gpu_tests.util.host_information.IsArmCpu', |
| return_value=True): |
| tags = _GetTagsToTest(browser) |
| self.assertIn('arch-arm64', tags) |
| self.assertNotIn('arch-x86_64', tags) |
| |
| with mock.patch('gpu_tests.util.host_information.IsArmCpu', |
| return_value=False): |
| tags = _GetTagsToTest(browser) |
| self.assertNotIn('arch-arm64', tags) |
| self.assertIn('arch-x86_64', tags) |
| |
| # Ensure that tags aren't generated on remote platforms. |
| platform = fakes.FakePlatform('android', 'android-15') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser._returned_system_info = _GetSystemInfo( |
| gpu=VENDOR_NVIDIA, device=0x0000, gl_renderer='ANGLE OpenGL ES') |
| tags = _GetTagsToTest(browser) |
| for t in tags: |
| self.assertFalse(t.startswith('arch-')) |
| |
| @mock.patch.dict(os.environ, clear=True) |
| def testGenerateDisplayServer(self) -> None: |
| platform = fakes.FakePlatform('mac', 'mojave') |
| browser = fakes.FakeBrowser(platform, 'release') |
| browser = typing.cast(ct.Browser, browser) |
| |
| with mock.patch('gpu_tests.util.host_information.IsLinux', |
| return_value=False): |
| tags = gpu_integration_test.GpuIntegrationTest.GetPlatformTags(browser) |
| for t in tags: |
| self.assertFalse(t.startswith('display-server')) |
| |
| with mock.patch('gpu_tests.util.host_information.IsLinux', |
| return_value=True): |
| tags = gpu_integration_test.GpuIntegrationTest.GetPlatformTags(browser) |
| self.assertIn('display-server-x', tags) |
| |
| os.environ['WAYLAND_DISPLAY'] = 'wayland-0' |
| tags = gpu_integration_test.GpuIntegrationTest.GetPlatformTags(browser) |
| self.assertIn('display-server-wayland', tags) |
| |
| def testTraceTestPrefixesInSync(self): |
| """Verifies that the trace test known prefix list is in sync.""" |
| test_cases = list( |
| trace_it.TraceIntegrationTest.GenerateTestCases__RunGpuTest( |
| mock.MagicMock())) |
| valid_prefixes = tuple(trace_it.TraceIntegrationTest.known_test_prefixes) |
| for test_name, _ in test_cases: |
| self.assertTrue(test_name.startswith(valid_prefixes)) |
| |
| def testSimpleIntegrationTest(self) -> None: |
| test_args = _IntegrationTestArgs('simple_integration_unittest') |
| test_args.failures = [ |
| 'unexpected_error', |
| 'unexpected_failure', |
| ] |
| test_args.successes = [ |
| 'expected_flaky', |
| 'expected_failure', |
| ] |
| test_args.skips = ['expected_skip'] |
| test_args.additional_args = [ |
| '--retry-only-retry-on-failure', |
| '--retry-limit=3', |
| '--test-name-prefix=unittest_data.integration_tests.SimpleTest.', |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| # The number of browser starts include the one call to StartBrowser at the |
| # beginning of the run of the test suite and for each RestartBrowser call |
| # which happens after every failure |
| self.assertEqual(self._test_state['num_browser_starts'], 6) |
| |
| def testIntegrationTesttWithBrowserFailure(self) -> None: |
| test_args = _IntegrationTestArgs( |
| 'browser_start_failure_integration_unittest') |
| test_args.successes = [ |
| 'unittest_data.integration_tests.BrowserStartFailureTest.restart' |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| self.assertEqual(self._test_state['num_browser_crashes'], 2) |
| self.assertEqual(self._test_state['num_browser_starts'], 3) |
| |
| def testIntegrationTestWithBrowserCrashUponStart(self) -> None: |
| test_args = _IntegrationTestArgs( |
| 'browser_crash_after_start_integration_unittest') |
| test_args.successes = [ |
| 'unittest_data.integration_tests.BrowserCrashAfterStartTest.restart' |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| self.assertEqual(self._test_state['num_browser_crashes'], 2) |
| self.assertEqual(self._test_state['num_browser_starts'], 3) |
| |
| def testRetryLimit(self) -> None: |
| test_args = _IntegrationTestArgs('test_retry_limit') |
| test_args.failures = [ |
| 'unittest_data.integration_tests.TestRetryLimit.unexpected_failure' |
| ] |
| test_args.additional_args = ['--retry-limit=2'] |
| |
| self._RunIntegrationTest(test_args) |
| # The number of attempted runs is 1 + the retry limit. |
| self.assertEqual(self._test_state['num_test_runs'], 3) |
| |
| def _RunTestsWithExpectationsFiles(self) -> None: |
| test_args = _IntegrationTestArgs('run_tests_with_expectations_files') |
| test_args.failures = ['a/b/unexpected-fail.html'] |
| test_args.successes = [ |
| 'a/b/expected-fail.html', |
| 'a/b/expected-flaky.html', |
| ] |
| test_args.skips = ['should_skip'] |
| test_args.additional_args = [ |
| '--retry-limit=3', |
| '--retry-only-retry-on-failure-tests', |
| ('--test-name-prefix=unittest_data.integration_tests.' |
| 'RunTestsWithExpectationsFiles.'), |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| |
| def testTestFilterCommandLineArg(self) -> None: |
| test_args = _IntegrationTestArgs('run_tests_with_expectations_files') |
| test_args.failures = ['a/b/unexpected-fail.html'] |
| test_args.successes = ['a/b/expected-fail.html'] |
| test_args.skips = ['should_skip'] |
| test_args.additional_args = [ |
| '--retry-limit=3', |
| '--retry-only-retry-on-failure-tests', |
| ('--test-filter=a/b/unexpected-fail.html::a/b/expected-fail.html::' |
| 'should_skip'), |
| ('--test-name-prefix=unittest_data.integration_tests.' |
| 'RunTestsWithExpectationsFiles.'), |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| |
| def testUseTestExpectationsFileToHandleExpectedSkip(self) -> None: |
| self._RunTestsWithExpectationsFiles() |
| results = self._test_result['tests']['should_skip'] |
| self.assertEqual(results['expected'], 'SKIP') |
| self.assertEqual(results['actual'], 'SKIP') |
| self.assertNotIn('is_regression', results) |
| |
| def testUseTestExpectationsFileToHandleUnexpectedTestFailure(self) -> None: |
| self._RunTestsWithExpectationsFiles() |
| results = self._test_result['tests']['a']['b']['unexpected-fail.html'] |
| self.assertEqual(results['expected'], 'PASS') |
| self.assertEqual(results['actual'], 'FAIL') |
| self.assertIn('is_regression', results) |
| |
| def testUseTestExpectationsFileToHandleExpectedFailure(self) -> None: |
| self._RunTestsWithExpectationsFiles() |
| results = self._test_result['tests']['a']['b']['expected-fail.html'] |
| self.assertEqual(results['expected'], 'FAIL') |
| self.assertEqual(results['actual'], 'FAIL') |
| self.assertNotIn('is_regression', results) |
| |
| def testUseTestExpectationsFileToHandleExpectedFlakyTest(self) -> None: |
| self._RunTestsWithExpectationsFiles() |
| results = self._test_result['tests']['a']['b']['expected-flaky.html'] |
| self.assertEqual(results['expected'], 'PASS') |
| self.assertEqual(results['actual'], 'FAIL FAIL FAIL PASS') |
| self.assertNotIn('is_regression', results) |
| |
| def testRepeat(self) -> None: |
| test_args = _IntegrationTestArgs('test_repeat') |
| test_args.successes = ['unittest_data.integration_tests.TestRepeat.success'] |
| test_args.additional_args = ['--repeat=3'] |
| |
| self._RunIntegrationTest(test_args) |
| self.assertEqual(self._test_state['num_test_runs'], 3) |
| |
| def testAlsoRunDisabledTests(self) -> None: |
| test_args = _IntegrationTestArgs('test_also_run_disabled_tests') |
| test_args.failures = [ |
| 'skip', |
| 'flaky', |
| ] |
| # Tests that are expected to fail and do fail are treated as test passes |
| test_args.successes = ['expected_failure'] |
| test_args.additional_args = [ |
| '--all', |
| '--test-name-prefix', |
| 'unittest_data.integration_tests.TestAlsoRunDisabledTests.', |
| '--retry-limit=3', |
| '--retry-only-retry-on-failure', |
| ] |
| |
| self._RunIntegrationTest(test_args) |
| self.assertEqual(self._test_state['num_flaky_test_runs'], 4) |
| self.assertEqual(self._test_state['num_test_runs'], 6) |
| |
| def testStartBrowser_Retries(self) -> None: |
| class TestException(Exception): |
| pass |
| |
| def SetBrowserAndRaiseTestException() -> None: |
| gpu_integration_test.GpuIntegrationTest.browser = mock.MagicMock() |
| raise TestException |
| |
| gpu_integration_test.GpuIntegrationTest.browser = None |
| gpu_integration_test.GpuIntegrationTest.platform = None |
| gpu_integration_test.GpuIntegrationTest._finder_options = mock.MagicMock() |
| with mock.patch.object( |
| gpu_integration_test.serially_executed_browser_test_case.\ |
| SeriallyExecutedBrowserTestCase, |
| 'StartBrowser', |
| side_effect=SetBrowserAndRaiseTestException) as mock_start_browser: |
| with mock.patch.object(gpu_integration_test.GpuIntegrationTest, |
| 'StopBrowser') as mock_stop_browser: |
| with self.assertRaises(TestException): |
| gpu_integration_test.GpuIntegrationTest.StartBrowser() |
| self.assertEqual(mock_start_browser.call_count, |
| gpu_integration_test._START_BROWSER_RETRIES) |
| self.assertEqual(mock_stop_browser.call_count, |
| gpu_integration_test._START_BROWSER_RETRIES) |
| |
| def _RunIntegrationTest(self, test_args: _IntegrationTestArgs) -> None: |
| """Runs an integration and asserts fail/success/skip expectations. |
| |
| Args: |
| test_args: A _IntegrationTestArgs instance to use. |
| """ |
| config = chromium_config.ChromiumConfig( |
| top_level_dir=gpu_path_util.GPU_DIR, |
| benchmark_dirs=[os.path.join(gpu_path_util.GPU_DIR, 'unittest_data')]) |
| |
| with binary_manager.TemporarilyReplaceBinaryManager(None), \ |
| tempfile_ext.NamedTemporaryDirectory() as temp_dir: |
| test_results_path = os.path.join(temp_dir, 'test_results.json') |
| test_state_path = os.path.join(temp_dir, 'test_state.json') |
| # We are processing ChromiumConfig instance and getting the argument |
| # list. Then we pass it directly to run_browser_tests.RunTests. If |
| # we called browser_test_runner.Run, then it would spawn another |
| # subprocess which is less efficient. |
| args = browser_test_runner.ProcessConfig( |
| config, |
| [ |
| test_args.test_name, |
| f'--write-full-results-to={test_results_path}', |
| f'--test-state-json-path={test_state_path}', |
| # We don't want the underlying typ-based tests to report their |
| # results to ResultDB. |
| '--disable-resultsink', |
| # These tests currently rely on some information sticking around |
| # between tests, so we need to use the older global process pool |
| # approach instead of having different pools scoped for |
| # parallel/serial execution. |
| '--use-global-pool', |
| ] + test_args.additional_args) |
| run_browser_tests.RunTests(args) |
| with open(test_results_path, encoding='utf-8') as f: |
| self._test_result = json.load(f) |
| with open(test_state_path, encoding='utf-8') as f: |
| self._test_state = json.load(f) |
| actual_successes, actual_failures, actual_skips = (_ExtractTestResults( |
| self._test_result)) |
| self.assertEqual(set(actual_failures), set(test_args.failures)) |
| self.assertEqual(set(actual_successes), set(test_args.successes)) |
| self.assertEqual(set(actual_skips), set(test_args.skips)) |
| |
| |
| def RunFakeBrowserStartWithArgsAndGpuInfo( |
| additional_args: list[str], |
| gpu_info: Any, |
| gpu_test_class: GpuTestClassType = gpu_integration_test.GpuIntegrationTest |
| ) -> None: |
| cls = gpu_test_class |
| |
| def FakeStartBrowser(): |
| cls.browser = mock.Mock() |
| cls.browser.tabs = [mock.Mock()] |
| mock_system_info = mock.Mock() |
| mock_system_info.gpu = gpu_info |
| cls.browser.GetSystemInfo = mock.Mock(return_value=mock_system_info) |
| |
| with mock.patch( |
| 'telemetry.testing.serially_executed_browser_test_case.' |
| 'SeriallyExecutedBrowserTestCase.StartBrowser', |
| side_effect=FakeStartBrowser): |
| options = fakes.CreateBrowserFinderOptions() |
| cls._finder_options = options |
| cls._original_finder_options = options |
| cls.platform = None |
| cls.CustomizeBrowserArgs(additional_args) |
| cls.StartBrowser() |
| |
| |
| def CreateGpuInfo(aux_attributes: dict | None = None, |
| feature_statuses: dict | None = None) -> mock.Mock: |
| aux_attributes = aux_attributes or {} |
| feature_statuses = feature_statuses or {} |
| |
| gpu_info = mock.Mock() |
| gpu_info.aux_attributes = aux_attributes |
| gpu_info.feature_status = feature_statuses |
| device = mock.Mock() |
| device.device_string = 'device_string' |
| gpu_info.devices = [device] |
| return gpu_info |
| |
| |
| # TODO(crbug.com/372740546): Find a way to properly unittest the cases |
| # where --gpu-disabled is passed in as well. Currently, we run into |
| # problems due to cls.platform being None, which causes problems with |
| # GPU code in _GenerateAndSanitizeBrowserArgs. Setting cls.platform to |
| # non-None values causes Telemetry code to fail due to the platform |
| # changing when it expects it to stay constant throughout the entire |
| # suite. |
| class FeatureVerificationUnittest(unittest.TestCase): |
| |
| # pylint: disable=no-self-use |
| def testVerifyGLBackendSuccessUnspecified(self): |
| """Tests GL backend verification that passes w/o a backend specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo([], gpu_info) |
| |
| def testVerifyGLBackendSuccessSpecified(self): |
| """Tests GL backend verification that passes w/ a backend specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-gl=angle'], gpu_info) |
| |
| def testVerifyGLBackendFailure(self): |
| """Tests GL backend verification that fails.""" |
| gpu_info = CreateGpuInfo(aux_attributes={}) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested GL backend \\(angle\\) had no effect on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-gl=angle'], gpu_info) |
| |
| def testVerifyANGLEBackendSuccessUnspecified(self): |
| """Tests ANGLE backend verification that passes w/o a backend specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo([], gpu_info) |
| |
| def testVerifyANGLEBackendSuccessSpecified(self): |
| """Tests ANGLE backend verification that passes w/ a backend specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-angle=gl'], gpu_info) |
| |
| def testVerifyANGLEBackendFailureUnknownBackend(self): |
| """Tests ANGLE backend verification failure due to an unknown backend.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE foo', |
| }) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested ANGLE backend \\(foo\\) had no effect on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-angle=foo'], gpu_info) |
| |
| def testVerifyANGLEBackendFailureMismatchedBackend(self): |
| """Tests ANGLE backend verification failure due to mismatched backends.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE Vulkan', |
| }) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested ANGLE backend \\(gl\\) had no effect on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-angle=gl'], gpu_info) |
| |
| def testVerifyCommandDecoderSuccessUnspecified(self): |
| """Tests cmd decoder verification that passes w/o a decoder specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'passthrough_cmd_decoder': True, |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo([], gpu_info) |
| |
| def testVerifyCommandDecoderSuccessSpecified(self): |
| """Tests cmd decoder verification that passes w/ a decoder specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'passthrough_cmd_decoder': True, |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-cmd-decoder=passthrough'], |
| gpu_info) |
| |
| def testVerifyCommandDecoderFailureUnknownDecoder(self): |
| """Tests cmd decoder verification that fails due to an unknown decoder.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'passthrough_cmd_decoder': True, |
| }) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested command decoder \\(foo\\) had no effect on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-cmd-decoder=foo'], gpu_info) |
| |
| def testVerifyCommandDecoderFailureMismatchedDecoder(self): |
| """Tests cmd decoder verification that fails due to a mismatched decoder.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'passthrough_cmd_decoder': False, |
| }) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested command decoder \\(passthrough\\) had no effect on the ' |
| 'browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--use-cmd-decoder=passthrough'], |
| gpu_info) |
| |
| def testVerifySkiaGraphiteSuccessUnspecified(self): |
| """Tests Skia Graphite verification that passes w/o specification.""" |
| gpu_info = CreateGpuInfo(feature_statuses={ |
| 'skia_graphite': 'enabled_on', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo([], gpu_info) |
| |
| def testVerifySkiaGraphiteSuccessSpecified(self): |
| """Tests Skia Graphite verification that passes w/ specification.""" |
| gpu_info = CreateGpuInfo(feature_statuses={ |
| 'skia_graphite': 'enabled_on', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--enable-features=SkiaGraphite'], |
| gpu_info) |
| |
| gpu_info = CreateGpuInfo(feature_statuses={}) |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--disable-features=SkiaGraphite'], |
| gpu_info) |
| |
| def testVerifySkiaGraphiteFailureMismatchedStatus(self): |
| """Tests Skia Graphite verification that fails due to mismatched status.""" |
| gpu_info = CreateGpuInfo(feature_statuses={}) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested Skia Graphite status \\(graphite-enabled\\) had no effect ' |
| 'on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--enable-features=SkiaGraphite'], |
| gpu_info) |
| |
| gpu_info = CreateGpuInfo(feature_statuses={ |
| 'skia_graphite': 'enabled_on', |
| }) |
| with self.assertRaisesRegex( |
| RuntimeError, |
| 'Requested Skia Graphite status \\(graphite-disabled\\) had no effect ' |
| 'on the browser:.*'): |
| RunFakeBrowserStartWithArgsAndGpuInfo(['--disable-features=SkiaGraphite'], |
| gpu_info) |
| |
| # pylint: enable=no-self-use |
| |
| |
| class FeatureVerificationWebGPUCompatUnittest(unittest.TestCase): |
| |
| # pylint: disable=no-self-use |
| def testVerifyCompatContextSuccessUnspecified(self): |
| """Tests WebGPU compat verification that passes w/o the es31 flag.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL ES 3.1', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo( |
| [], gpu_info, webgpu_compat_cit.WebGpuCompatCtsIntegrationTest) |
| |
| def testVerifyCompatContextSuccessSpecified(self): |
| """Tests WebGPU compat verification that passes w/ the es31 flag.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'ANGLE OpenGL ES 3.1', |
| }) |
| RunFakeBrowserStartWithArgsAndGpuInfo( |
| ['--use-dawn-features=gl_force_es_31_and_no_extensions'], gpu_info, |
| webgpu_compat_cit.WebGpuCompatCtsIntegrationTest) |
| |
| # TODO(crbug.com/388318201): Uncomment the following test |
| # def testVerifyCompatContextFailure(self): |
| # """Tests WebGPU compat verification that fails.""" |
| # gpu_info = CreateGpuInfo(aux_attributes={ |
| # 'gl_renderer': 'ANGLE OpenGL ES 3.2', |
| # }) |
| # with self.assertRaisesRegex(RuntimeError, |
| # 'Requested WebGPU compat context min ES31 .*'): |
| # RunFakeBrowserStartWithArgsAndGpuInfo( |
| # ['--use-dawn-features=gl_force_es_31_and_no_extensions'], gpu_info, |
| # webgpu_compat_cit.WebGpuCompatCtsIntegrationTest) |
| |
| class PreemptArgsUnittest(unittest.TestCase): |
| |
| def testNoConflictIsNoOp(self): |
| """Tests that no conflict arguments results in a no-op.""" |
| options = bo.BrowserOptions() |
| options.AppendExtraBrowserArgs(['--use-angle=gl', '--another-arg']) |
| expected_browser_args = copy.deepcopy(options.extra_browser_args) |
| gpu_integration_test._PreemptArguments(options, |
| ['--use-webgpu-adapter=swiftshader']) |
| self.assertEqual(options.extra_browser_args, expected_browser_args) |
| |
| def testConflictingArgsRemoved(self): |
| for arg in gpu_integration_test._ARGS_TO_PREEMPT: |
| options = bo.BrowserOptions() |
| options.AppendExtraBrowserArgs([f'{arg}=a', '--another-arg']) |
| gpu_integration_test._PreemptArguments(options, |
| [f'{arg}=b', '--yet-another-arg']) |
| self.assertEqual(options.extra_browser_args, set(['--another-arg'])) |
| |
| |
| class GetGPUInfoErrorStringUnittest(unittest.TestCase): |
| |
| def testMinimalInformation(self): |
| """Tests error string generation w/ the minimum possible information.""" |
| gpu_info = CreateGpuInfo() |
| expected_error = 'primary gpu=device_string' |
| self.assertEqual(gpu_integration_test._GetGPUInfoErrorString(gpu_info), |
| expected_error) |
| |
| def testGLRenderer(self): |
| """Tests error string generation w/ the GL renderer specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'foo', |
| }) |
| expected_error = 'primary gpu=device_string, gl_renderer=foo' |
| self.assertEqual(gpu_integration_test._GetGPUInfoErrorString(gpu_info), |
| expected_error) |
| |
| def testFeatureStatuses(self): |
| """Tests error string generation w/ feature statuses specified.""" |
| gpu_info = CreateGpuInfo(feature_statuses={ |
| 'featureA': 'on', |
| 'featureB': 'off', |
| }) |
| expected_error = ('primary gpu=device_string, ' |
| 'feature_statuses=featureA=on,featureB=off') |
| self.assertEqual(gpu_integration_test._GetGPUInfoErrorString(gpu_info), |
| expected_error) |
| |
| def testGLRendererAndFeatureStatuses(self): |
| """Tests error string generation w/ GL renderer/feature status specified.""" |
| gpu_info = CreateGpuInfo(aux_attributes={ |
| 'gl_renderer': 'foo', |
| }, |
| feature_statuses={ |
| 'featureA': 'on', |
| 'featureB': 'off' |
| }) |
| expected_error = ('primary gpu=device_string, ' |
| 'gl_renderer=foo, ' |
| 'feature_statuses=featureA=on,featureB=off') |
| self.assertEqual(gpu_integration_test._GetGPUInfoErrorString(gpu_info), |
| expected_error) |
| |
| |
| def _ExtractTestResults( |
| test_result: dict[str, dict]) -> tuple[list[str], list[str], list[str]]: |
| delimiter = test_result['path_delimiter'] |
| failures = [] |
| successes = [] |
| skips = [] |
| |
| def _IsLeafNode(node): |
| test_dict = node[1] |
| return 'expected' in test_dict and isinstance(test_dict['expected'], str) |
| |
| node_queues = [] |
| for t in test_result['tests']: |
| node_queues.append((t, test_result['tests'][t])) |
| while node_queues: |
| node = node_queues.pop() |
| full_test_name, test_dict = node |
| if _IsLeafNode(node): |
| if all(res not in test_dict['expected'].split() |
| for res in test_dict['actual'].split()): |
| failures.append(full_test_name) |
| elif test_dict['expected'] == test_dict['actual'] == 'SKIP': |
| skips.append(full_test_name) |
| else: |
| successes.append(full_test_name) |
| else: |
| for k in test_dict: |
| node_queues.append((f'{full_test_name}{delimiter}{k}', test_dict[k])) |
| return successes, failures, skips |
| |
| |
| if __name__ == '__main__': |
| unittest.main(verbosity=2) |
