| # Copyright 2019 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import inspect |
| import os |
| import re |
| from typing import Type |
| import unittest |
| from unittest import mock |
| |
| from py_utils import discover |
| from py_utils import tempfile_ext |
| from typ import expectations_parser |
| from typ import json_results |
| |
| import gpu_project_config |
| from gpu_tests import common_typing as ct |
| from gpu_tests import gpu_helper |
| from gpu_tests import gpu_integration_test |
| from gpu_tests import pixel_integration_test |
| from gpu_tests import pixel_test_pages |
| from gpu_tests import trace_integration_test as trace_it |
| from gpu_tests import trace_test_pages |
| from gpu_tests import webgl1_conformance_integration_test as webgl1_cit |
| from gpu_tests import webgl2_conformance_integration_test as webgl2_cit |
| from gpu_tests import webgl_test_util |
| |
| |
| VALID_BUG_REGEXES = [ |
| re.compile(r'crbug\.com\/\d+'), |
| re.compile(r'crbug\.com\/angleproject\/\d+'), |
| re.compile(r'crbug\.com\/dawn\/\d+'), |
| re.compile(r'crbug\.com\/swiftshader\/\d+'), |
| re.compile(r'crbug\.com\/tint\/\d+'), |
| re.compile(r'skbug\.com\/\d+'), |
| ] |
| |
| ResultType = json_results.ResultType |
| |
| INTEL_DRIVER_VERSION_SCHEMA = """ |
| The version format of Intel graphics driver is AA.BB.CC.DDDD (legacy schema) |
| and AA.BB.CCC.DDDD (new schema). |
| |
| AA.BB: You are free to specify the real number here, but they are meaningless |
| when comparing two version numbers. Usually it's okay to leave it to "0.0". |
| |
| CC or CCC: It's meaningful to indicate different branches. Different CC means |
| different branch, while all CCCs share the same branch. |
| |
| DDDD: It's always meaningful. |
| """ |
| |
| |
| def check_intel_driver_version(version: str) -> bool: |
| ver_list = version.split('.') |
| if len(ver_list) != 4: |
| return False |
| for ver in ver_list: |
| if not ver.isdigit(): |
| return False |
| return True |
| |
| |
| def _ExtractUnitTestTestExpectations(file_name: str) -> list[str]: |
| file_name = os.path.join( |
| os.path.dirname(os.path.abspath(__file__)), '..', 'unittest_data', |
| 'test_expectations', file_name) |
| test_expectations_list = [] |
| with open(file_name, 'r', encoding='utf-8') as test_data: |
| test_expectations = '' |
| reach_end = False |
| while not reach_end: |
| line = test_data.readline() |
| if line: |
| line = line.strip() |
| if line: |
| test_expectations += line + '\n' |
| continue |
| else: |
| reach_end = True |
| |
| if test_expectations: |
| test_expectations_list.append(test_expectations) |
| test_expectations = '' |
| |
| assert test_expectations_list |
| return test_expectations_list |
| |
| |
| def CheckTestExpectationsAreForExistingTests( |
| unittest_testcase: unittest.TestCase, |
| test_class: Type[gpu_integration_test.GpuIntegrationTest], |
| mock_options: mock.MagicMock, |
| test_names: list[str] | None = None) -> None: |
| test_names = test_names or [ |
| args[0] for args in test_class.GenerateTestCases__RunGpuTest(mock_options) |
| ] |
| expectations_file = test_class.ExpectationsFiles()[0] |
| with open(expectations_file, 'r', encoding='utf-8') as f: |
| test_expectations = expectations_parser.TestExpectations() |
| test_expectations.parse_tagged_list(f.read(), f.name) |
| broke_expectations = '\n'.join([ |
| f"\t- {f.name}:{exp.lineno}: Expectation with pattern '{exp.test}' " |
| f'does not match any tests in the {test_class.Name()} test suite' |
| for exp in test_expectations.check_for_broken_expectations(test_names) |
| ]) |
| unittest_testcase.assertEqual( |
| broke_expectations, '', |
| f'The following expectations were found to not apply to any tests in ' |
| f'the {test_class.Name()} test suite:\n{broke_expectations}') |
| |
| |
| def CheckTestExpectationPatternsForConflicts( |
| expectations: str, file_name: str, |
| tag_conflict_checker: ct.TagConflictChecker) -> str: |
| test_expectations = expectations_parser.TestExpectations() |
| _, errors = test_expectations.parse_tagged_list( |
| expectations, file_name=file_name, tags_conflict=tag_conflict_checker) |
| return errors |
| |
| |
| def _FindTestCases() -> list[Type[gpu_integration_test.GpuIntegrationTest]]: |
| test_cases = [] |
| for start_dir in gpu_project_config.CONFIG.start_dirs: |
| # Note we deliberately only scan the integration tests as a |
| # workaround for http://crbug.com/1195465 . |
| modules_to_classes = discover.DiscoverClasses( |
| start_dir, |
| gpu_project_config.CONFIG.top_level_dir, |
| base_class=gpu_integration_test.GpuIntegrationTest, |
| pattern='*_integration_test.py') |
| test_cases.extend(modules_to_classes.values()) |
| return test_cases |
| |
| |
| class GpuTestExpectationsValidation(unittest.TestCase): |
| maxDiff = None |
| |
| def testNoConflictsInGpuTestExpectations(self) -> None: |
| errors = '' |
| for test_case in _FindTestCases(): |
| if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: |
| webgl_version = 1 |
| if issubclass(test_case, webgl2_cit.WebGL2ConformanceIntegrationTest): |
| webgl_version = 2 |
| _ = list( |
| test_case.GenerateTestCases__RunGpuTest( |
| gpu_helper.GetMockArgs(webgl_version=f'{webgl_version}.0.0'))) |
| if test_case.ExpectationsFiles(): |
| with open(test_case.ExpectationsFiles()[0], encoding='utf-8') as f: |
| errors += CheckTestExpectationPatternsForConflicts( |
| f.read(), os.path.basename(f.name), |
| test_case.GetTagConflictChecker()) |
| self.assertEqual(errors, '') |
| |
| def testExpectationsFilesCanBeParsed(self) -> None: |
| for test_case in _FindTestCases(): |
| if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: |
| webgl_version = 1 |
| if issubclass(test_case, webgl2_cit.WebGL2ConformanceIntegrationTest): |
| webgl_version = 2 |
| _ = list( |
| test_case.GenerateTestCases__RunGpuTest( |
| gpu_helper.GetMockArgs(webgl_version=f'{webgl_version}.0.0'))) |
| if test_case.ExpectationsFiles(): |
| with open(test_case.ExpectationsFiles()[0], encoding='utf-8') as f: |
| test_expectations = expectations_parser.TestExpectations() |
| ret, err = test_expectations.parse_tagged_list(f.read(), f.name) |
| self.assertEqual( |
| ret, 0, f'Error parsing {os.path.basename(f.name)}:\n\t{err}') |
| |
| def testWebglTestPathsExist(self) -> None: |
| def _CheckWebglConformanceTestPathIsValid(pattern: str) -> None: |
| if not 'WebglExtension_' in pattern: |
| full_path = os.path.normpath( |
| os.path.join(webgl_test_util.conformance_path, pattern)) |
| self.assertTrue(os.path.exists(full_path), |
| f'{full_path} does not exist') |
| |
| webgl_test_classes = ( |
| webgl1_cit.WebGL1ConformanceIntegrationTest, |
| webgl2_cit.WebGL2ConformanceIntegrationTest, |
| ) |
| for webgl_version in range(1, 3): |
| webgl_test_class = webgl_test_classes[webgl_version - 1] |
| _ = list( |
| webgl_test_class.GenerateTestCases__RunGpuTest( |
| gpu_helper.GetMockArgs(webgl_version=f'{webgl_version}.0.0'))) |
| with open(webgl_test_class.ExpectationsFiles()[0], 'r', |
| encoding='utf-8') as f: |
| expectations = expectations_parser.TestExpectations() |
| expectations.parse_tagged_list(f.read()) |
| for pattern, _ in expectations.individual_exps.items(): |
| _CheckWebglConformanceTestPathIsValid(pattern) |
| |
| # Pixel tests are handled separately since some tests are only generated on |
| # certain platforms. |
| def testForBrokenPixelTestExpectations(self) -> None: |
| pixel_test_names = [] |
| for _, method in inspect.getmembers(pixel_test_pages.PixelTestPages, |
| predicate=inspect.isfunction): |
| pixel_test_names.extend([ |
| p.name for p in method( |
| pixel_integration_test.PixelIntegrationTest.test_base_name) |
| ]) |
| CheckTestExpectationsAreForExistingTests( |
| self, pixel_integration_test.PixelIntegrationTest, |
| gpu_helper.GetMockArgs(), pixel_test_names) |
| |
| # Trace tests are handled separately since some tests are only generated on |
| # certain platforms. |
| def testForBrokenTraceTestExpectations(self): |
| generator_functions = [] |
| for _, method in inspect.getmembers(trace_test_pages.TraceTestPages, |
| predicate=inspect.isfunction): |
| generator_functions.append(method) |
| |
| trace_test_names = [] |
| # This results in a set of tests which is a superset of actual tests that |
| # can be generated. |
| for prefix in trace_it.TraceIntegrationTest.known_test_prefixes: |
| for method in generator_functions: |
| trace_test_names.extend([p.name for p in method(prefix)]) |
| CheckTestExpectationsAreForExistingTests(self, |
| trace_it.TraceIntegrationTest, |
| gpu_helper.GetMockArgs(), |
| trace_test_names) |
| |
| # WebGL tests are handled separately since test case generation varies |
| # depending on inputs. |
| def testForBrokenWebGlTestExpectations(self) -> None: |
| webgl_test_classes_and_versions = ( |
| (webgl1_cit.WebGL1ConformanceIntegrationTest, '1.0.4'), |
| (webgl2_cit.WebGL2ConformanceIntegrationTest, '2.0.1'), |
| ) |
| for webgl_test_class, webgl_version in webgl_test_classes_and_versions: |
| args = gpu_helper.GetMockArgs(webgl_version=webgl_version) |
| test_names = [ |
| test[0] |
| for test in webgl_test_class.GenerateTestCases__RunGpuTest(args) |
| ] |
| CheckTestExpectationsAreForExistingTests(self, webgl_test_class, args, |
| test_names) |
| |
| def testForBrokenGpuTestExpectations(self) -> None: |
| options = gpu_helper.GetMockArgs() |
| for test_case in _FindTestCases(): |
| if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: |
| # Pixel, trace, and WebGL are handled in dedicated unittests. |
| if (test_case.Name() not in ('pixel', 'trace_test', |
| 'webgl1_conformance', 'webgl2_conformance') |
| and test_case.ExpectationsFiles()): |
| CheckTestExpectationsAreForExistingTests(self, test_case, options) |
| |
| def testExpectationBugValidity(self) -> None: |
| expectation_dir = os.path.join(os.path.dirname(__file__), |
| 'test_expectations') |
| for expectation_file in (f for f in os.listdir(expectation_dir) |
| if f.endswith('.txt')): |
| with open(os.path.join(expectation_dir, expectation_file), |
| encoding='utf-8') as f: |
| content = f.read() |
| list_parser = expectations_parser.TaggedTestListParser(content) |
| for expectation in list_parser.expectations: |
| reason = expectation.reason |
| if not reason: |
| continue |
| if not any(r.match(reason) for r in VALID_BUG_REGEXES): |
| self.fail(f'Bug string "{reason}" in expectation file ' |
| f'{expectation_file} is either not in a recognized format ' |
| f'or references an unknown project.') |
| |
| def testWebglTestExpectationsForDriverTags(self) -> None: |
| webgl_test_classes = ( |
| webgl1_cit.WebGL1ConformanceIntegrationTest, |
| webgl2_cit.WebGL2ConformanceIntegrationTest, |
| ) |
| expectations_driver_tags = set() |
| for webgl_version in range(1, 3): |
| webgl_conformance_test_class = webgl_test_classes[webgl_version - 1] |
| _ = list( |
| webgl_conformance_test_class.GenerateTestCases__RunGpuTest( |
| gpu_helper.GetMockArgs(webgl_version=f'{webgl_version}.0.0'))) |
| with open(webgl_conformance_test_class.ExpectationsFiles()[0], |
| 'r', |
| encoding='utf-8') as f: |
| parser = expectations_parser.TestExpectations() |
| parser.parse_tagged_list(f.read(), f.name) |
| driver_tag_set = set() |
| for tag_set in parser.tag_sets: |
| if gpu_helper.MatchDriverTag(list(tag_set)[0]): |
| for tag in tag_set: |
| match = gpu_helper.MatchDriverTag(tag) |
| self.assertIsNotNone(match) |
| if match.group(1) == 'intel': |
| self.assertTrue(check_intel_driver_version(match.group(3))) |
| |
| self.assertSetEqual(driver_tag_set, set()) |
| driver_tag_set = tag_set |
| else: |
| for tag in tag_set: |
| self.assertIsNone(gpu_helper.MatchDriverTag(tag)) |
| expectations_driver_tags |= driver_tag_set |
| |
| self.assertEqual(gpu_helper.GetAllExpectationFileDriverTags(), |
| expectations_driver_tags) |
| |
| |
| class TestGpuTestExpectationsValidators(unittest.TestCase): |
| |
| def setUp(self): |
| self.conflict_checker = ( |
| gpu_integration_test.GpuIntegrationTest.GetTagConflictChecker()) |
| |
| def testConflictInTestExpectationsWithGpuDriverTags(self) -> None: |
| failed_test_expectations = _ExtractUnitTestTestExpectations( |
| 'failed_test_expectations_with_driver_tags.txt') |
| self.assertTrue( |
| all( |
| CheckTestExpectationPatternsForConflicts( |
| test_expectations, 'test.txt', self.conflict_checker) |
| for test_expectations in failed_test_expectations)) |
| |
| def testNoConflictInTestExpectationsWithGpuDriverTags(self) -> None: |
| passed_test_expectations = _ExtractUnitTestTestExpectations( |
| 'passed_test_expectations_with_driver_tags.txt') |
| for test_expectations in passed_test_expectations: |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertFalse(errors) |
| |
| def testConflictsBetweenAngleAndNonAngleConfigurations(self) -> None: |
| test_expectations = """ |
| # tags: [ android ] |
| # tags: [ android-pixel-6 ] |
| # tags: [ opengles ] |
| # results: [ RetryOnFailure Skip ] |
| [ android android-pixel-6 ] a/b/c/d [ RetryOnFailure ] |
| [ android opengles ] a/b/c/d [ Skip ] |
| """ |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertTrue(errors) |
| |
| def testConflictBetweenTestExpectationsWithOsNameAndOSVersionTags(self |
| ) -> None: |
| test_expectations = """# tags: [ mac win linux win10 ] |
| # tags: [ intel amd nvidia ] |
| # tags: [ debug release ] |
| # results: [ Failure Skip ] |
| [ intel win10 ] a/b/c/d [ Failure ] |
| [ intel win debug ] a/b/c/d [ Skip ] |
| """ |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertTrue(errors) |
| |
| def testNoConflictBetweenOsVersionTags(self) -> None: |
| test_expectations = """# tags: [ mac win linux xp win7 ] |
| # tags: [ intel amd nvidia ] |
| # tags: [ debug release ] |
| # results: [ Failure Skip ] |
| [ intel win7 ] a/b/c/d [ Failure ] |
| [ intel xp debug ] a/b/c/d [ Skip ] |
| """ |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertFalse(errors) |
| |
| def testConflictBetweenGpuVendorAndGpuDeviceIdTags(self) -> None: |
| test_expectations = """# tags: [ mac win linux xp win7 ] |
| # tags: [ intel amd nvidia nvidia-0x1cb3 nvidia-0x2184 ] |
| # tags: [ debug release ] |
| # results: [ Failure Skip ] |
| [ nvidia-0x1cb3 ] a/b/c/d [ Failure ] |
| [ nvidia debug ] a/b/c/d [ Skip ] |
| """ |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertTrue(errors) |
| |
| def testNoConflictBetweenGpuDeviceIdTags(self) -> None: |
| test_expectations = """# tags: [ mac win linux xp win7 ] |
| # tags: [ intel amd nvidia nvidia-0x01 nvidia-0x02 ] |
| # tags: [ debug release ] |
| # results: [ Failure Skip ] |
| [ nvidia-0x01 win7 ] a/b/c/d [ Failure ] |
| [ nvidia-0x02 win7 debug ] a/b/c/d [ Skip ] |
| [ nvidia win debug ] a/b/c/* [ Skip ] |
| """ |
| errors = CheckTestExpectationPatternsForConflicts(test_expectations, |
| 'test.txt', |
| self.conflict_checker) |
| self.assertFalse(errors) |
| |
| def testFoundBrokenExpectations(self) -> None: |
| test_expectations = ('# tags: [ mac ]\n' |
| '# results: [ Failure ]\n' |
| '[ mac ] a/b/d [ Failure ]\n' |
| 'a/c/* [ Failure ]\n') |
| options = gpu_helper.GetMockArgs() |
| test_class = gpu_integration_test.GpuIntegrationTest |
| with tempfile_ext.NamedTemporaryFile(mode='w') as expectations_file, \ |
| mock.patch.object( |
| test_class, 'GenerateTestCases__RunGpuTest', |
| return_value=[('a/b/c', ())]), \ |
| mock.patch.object( |
| test_class, |
| 'ExpectationsFiles', return_value=[expectations_file.name]): |
| expectations_file.write(test_expectations) |
| expectations_file.close() |
| with self.assertRaises(AssertionError) as context: |
| CheckTestExpectationsAreForExistingTests(self, test_class, options) |
| self.assertIn( |
| 'The following expectations were found to not apply' |
| ' to any tests in the GpuIntegrationTest test suite', |
| str(context.exception)) |
| self.assertIn( |
| "4: Expectation with pattern 'a/c/*' does not match" |
| ' any tests in the GpuIntegrationTest test suite', |
| str(context.exception)) |
| self.assertIn( |
| "3: Expectation with pattern 'a/b/d' does not match" |
| ' any tests in the GpuIntegrationTest test suite', |
| str(context.exception)) |
| |
| |
| def testDriverVersionComparision(self) -> None: |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'eq', |
| '24.20.100.7000')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100', 'ne', |
| '24.20.100.7000')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'gt', |
| '24.20.100')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000a', 'gt', |
| '24.20.100.7000')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'lt', |
| '24.20.100.7001')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'lt', |
| '24.20.200.6000')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'lt', |
| '25.30.100.6000', 'linux', |
| 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'gt', |
| '25.30.100.6000', 'win', 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.101.6000', 'gt', |
| '25.30.100.7000', 'win', 'intel')) |
| self.assertFalse( |
| gpu_helper.EvaluateVersionComparison('24.20.99.7000', 'gt', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.99.7000', 'lt', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.99.7000', 'ne', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertFalse( |
| gpu_helper.EvaluateVersionComparison('24.20.100', 'lt', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertFalse( |
| gpu_helper.EvaluateVersionComparison('24.20.100', 'gt', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100', 'ne', |
| '24.20.100.7000', 'win', 'intel')) |
| self.assertTrue( |
| gpu_helper.EvaluateVersionComparison('24.20.100.7000', 'eq', |
| '25.20.100.7000', 'win', 'intel')) |
| |
| |
| if __name__ == '__main__': |
| unittest.main(verbosity=2) |
