| #!/usr/bin/env vpython3 |
| # Copyright 2021 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # pylint: disable=protected-access |
| |
| import subprocess |
| import unittest |
| import unittest.mock as mock |
| |
| from unexpected_passes import gpu_unittest_utils as gpu_uu |
| from unexpected_passes_common import builders |
| from unexpected_passes_common import constants |
| from unexpected_passes_common import expectations |
| from unexpected_passes_common import unittest_utils as uu |
| |
| |
| class QueryBuilderUnittest(unittest.TestCase): |
| def setUp(self) -> None: |
| self._patcher = mock.patch.object(subprocess, 'Popen') |
| self._popen_mock = self._patcher.start() |
| self.addCleanup(self._patcher.stop) |
| |
| builders.ClearInstance() |
| expectations.ClearInstance() |
| uu.RegisterGenericBuildersImplementation() |
| uu.RegisterGenericExpectationsImplementation() |
| |
| def testSuiteNameTranslation(self) -> None: |
| """Tests that the suite passed to the query is auto-translated.""" |
| # The key is the return value of Name() for a test suite, while the value is |
| # the last part of the Python module for the test file (i.e. the name of the |
| # file without .py). The former is used when running the tests, while the |
| # latter is used by ResultDB for reporting. |
| suites_to_modules = { |
| 'cast_streaming': 'cast_streaming_integration_test', |
| 'context_lost': 'context_lost_integration_test', |
| 'expected_color': 'expected_color_test', |
| 'gpu_process': 'gpu_process_integration_test', |
| 'hardware_accelerated_feature': |
| 'hardware_accelerated_feature_integration_test', |
| 'info_collection': 'info_collection_test', |
| 'noop_sleep': 'noop_sleep_integration_test', |
| 'pixel': 'pixel_integration_test', |
| 'power': 'power_measurement_integration_test', |
| 'screenshot_sync': 'screenshot_sync_integration_test', |
| 'trace_test': 'trace_integration_test', |
| 'webcodecs': 'webcodecs_integration_test', |
| 'webgl1_conformance': 'webgl1_conformance_integration_test', |
| 'webgl2_conformance': 'webgl2_conformance_integration_test', |
| 'webgpu_cts': 'webgpu_cts_integration_test', |
| } |
| |
| def assertSuiteInQuery(suite: str, call_args: tuple) -> None: |
| query = call_args[0][0] |
| s = 'gpu_tests\\\\.%s\\\\.' % suite |
| self.assertIn(s, query) |
| |
| for suite, module in suites_to_modules.items(): |
| querier = gpu_uu.CreateGenericGpuQuerier(suite=suite) |
| with mock.patch.object(querier, '_GetSeriesForQuery', |
| return_value=[]) as query_mock: |
| for _ in querier.GetBuilderGroupedQueryResults( |
| constants.BuilderTypes.CI, False): |
| pass |
| query_mock.assert_called_once() |
| assertSuiteInQuery(module, query_mock.call_args) |
| |
| |
| class GeneratedQueryUnittest(unittest.TestCase): |
| maxDiff = None |
| |
| def setUp(self): |
| self._querier = gpu_uu.CreateGenericGpuQuerier(suite='webgl1_conformance', |
| num_samples=15) |
| |
| def testPublicCi(self): |
| """Tests that the generated public CI query is as expected.""" |
| expected_query = """\ |
| WITH |
| builds AS ( |
| WITH |
| all_builds AS ( |
| SELECT |
| DISTINCT exported.id AS build_inv_id, |
| variant.*, |
| partition_time |
| FROM |
| `chrome-luci-data.chromium.gpu_ci_test_results` AS tr, |
| UNNEST(variant) AS variant |
| WHERE |
| DATE(partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.realm = "chromium:ci" |
| AND key = "builder" |
| ), |
| grouped_builds AS ( |
| SELECT |
| build_inv_id, |
| value AS builder, |
| partition_time, |
| RANK() OVER (PARTITION BY value ORDER BY partition_time DESC) AS rank_idx, |
| FROM all_builds |
| ) |
| SELECT |
| build_inv_id, |
| builder, |
| partition_time |
| FROM grouped_builds |
| WHERE rank_idx <= 15 |
| ), |
| results AS ( |
| SELECT |
| exported.id, |
| test_id, |
| status, |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "builder" |
| ) as builder_name, |
| IFNULL( |
| ( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "step_name"), |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "test_suite")) as step_name, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "typ_tag") as typ_tags, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "raw_typ_expectation") as typ_expectations |
| FROM |
| `chrome-luci-data.chromium.gpu_ci_test_results` tr, |
| builds b |
| WHERE |
| DATE(tr.partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.id = build_inv_id |
| AND status != "SKIP" |
| AND REGEXP_CONTAINS( |
| test_id, |
| "gpu_tests\\\\.webgl1_conformance_integration_test\\\\.") |
| ) |
| SELECT id, test_id, builder_name, status, step_name, typ_tags |
| FROM results |
| WHERE |
| "Failure" IN UNNEST(typ_expectations) |
| OR "RetryOnFailure" IN UNNEST(typ_expectations) |
| ORDER BY builder_name DESC |
| """ |
| self.assertEqual(self._querier._GetPublicCiQuery(), expected_query) |
| |
| def testInternalCi(self) -> None: |
| """Tests that the generated internal CI query is as expected.""" |
| expected_query = """\ |
| WITH |
| builds AS ( |
| WITH |
| all_builds AS ( |
| SELECT |
| DISTINCT exported.id AS build_inv_id, |
| variant.*, |
| partition_time |
| FROM |
| `chrome-luci-data.chrome.gpu_ci_test_results` AS tr, |
| UNNEST(variant) AS variant |
| WHERE |
| DATE(partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.realm = "chrome:ci" |
| AND key = "builder" |
| ), |
| grouped_builds AS ( |
| SELECT |
| build_inv_id, |
| value AS builder, |
| partition_time, |
| RANK() OVER (PARTITION BY value ORDER BY partition_time DESC) AS rank_idx, |
| FROM all_builds |
| ) |
| SELECT |
| build_inv_id, |
| builder, |
| partition_time |
| FROM grouped_builds |
| WHERE rank_idx <= 15 |
| ), |
| results AS ( |
| SELECT |
| exported.id, |
| test_id, |
| status, |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "builder" |
| ) as builder_name, |
| IFNULL( |
| ( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "step_name"), |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "test_suite")) as step_name, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "typ_tag") as typ_tags, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "raw_typ_expectation") as typ_expectations |
| FROM |
| `chrome-luci-data.chrome.gpu_ci_test_results` tr, |
| builds b |
| WHERE |
| DATE(tr.partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.id = build_inv_id |
| AND status != "SKIP" |
| AND REGEXP_CONTAINS( |
| test_id, |
| "gpu_tests\\\\.webgl1_conformance_integration_test\\\\.") |
| ) |
| SELECT id, test_id, builder_name, status, step_name, typ_tags |
| FROM results |
| WHERE |
| "Failure" IN UNNEST(typ_expectations) |
| OR "RetryOnFailure" IN UNNEST(typ_expectations) |
| ORDER BY builder_name DESC |
| """ |
| self.assertEqual(self._querier._GetInternalCiQuery(), expected_query) |
| |
| def testPublicTry(self) -> None: |
| """Tests that the generated public try query is as expected.""" |
| expected_query = """\ |
| WITH |
| submitted_builds AS ( |
| SELECT |
| CONCAT("build-", CAST(unnested_builds.id AS STRING)) as id |
| FROM |
| `commit-queue.chromium.attempts`, |
| UNNEST(builds) as unnested_builds, |
| UNNEST(gerrit_changes) as unnested_changes |
| WHERE |
| unnested_builds.host = "cr-buildbucket.appspot.com" |
| AND unnested_changes.submit_status = "SUCCESS" |
| AND start_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), |
| INTERVAL 30 DAY) |
| UNION ALL |
| SELECT |
| CONCAT("build-", CAST(unnested_builds.id AS STRING)) as id |
| FROM |
| `commit-queue.angle.attempts`, |
| UNNEST(builds) as unnested_builds, |
| UNNEST(gerrit_changes) as unnested_changes |
| WHERE |
| unnested_builds.host = "cr-buildbucket.appspot.com" |
| AND unnested_changes.submit_status = "SUCCESS" |
| AND start_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), |
| INTERVAL 30 DAY) |
| ), |
| builds AS ( |
| WITH |
| all_builds AS ( |
| SELECT |
| DISTINCT exported.id AS build_inv_id, |
| variant.*, |
| partition_time |
| FROM |
| `chrome-luci-data.chromium.gpu_try_test_results` AS tr, |
| UNNEST(variant) AS variant, |
| submitted_builds sb |
| WHERE |
| DATE(partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.realm = "chromium:try" |
| AND key = "builder" |
| AND exported.id = sb.id |
| ), |
| grouped_builds AS ( |
| SELECT |
| build_inv_id, |
| value AS builder, |
| partition_time, |
| RANK() OVER (PARTITION BY value ORDER BY partition_time DESC) AS rank_idx, |
| FROM all_builds |
| ) |
| SELECT |
| build_inv_id, |
| builder, |
| partition_time |
| FROM grouped_builds |
| WHERE rank_idx <= 15 |
| ), |
| results AS ( |
| SELECT |
| exported.id, |
| test_id, |
| status, |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "builder" |
| ) as builder_name, |
| IFNULL( |
| ( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "step_name"), |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "test_suite")) as step_name, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "typ_tag") as typ_tags, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "raw_typ_expectation") as typ_expectations |
| FROM |
| `chrome-luci-data.chromium.gpu_try_test_results` tr, |
| builds b |
| WHERE |
| DATE(tr.partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.id = build_inv_id |
| AND status != "SKIP" |
| AND REGEXP_CONTAINS( |
| test_id, |
| "gpu_tests\\\\.webgl1_conformance_integration_test\\\\.") |
| ) |
| SELECT id, test_id, builder_name, status, step_name, typ_tags |
| FROM results |
| WHERE |
| "Failure" IN UNNEST(typ_expectations) |
| OR "RetryOnFailure" IN UNNEST(typ_expectations) |
| ORDER BY builder_name DESC |
| """ |
| self.assertEqual(self._querier._GetPublicTryQuery(), expected_query) |
| |
| def testInternalTry(self) -> None: |
| """Tests that the generated internal try query is as expected.""" |
| expected_query = """\ |
| WITH |
| submitted_builds AS ( |
| SELECT |
| CONCAT("build-", CAST(unnested_builds.id AS STRING)) as id |
| FROM |
| `commit-queue.chrome.attempts`, |
| UNNEST(builds) as unnested_builds, |
| UNNEST(gerrit_changes) as unnested_changes |
| WHERE |
| unnested_builds.host = "cr-buildbucket.appspot.com" |
| AND unnested_changes.submit_status = "SUCCESS" |
| AND start_time > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), |
| INTERVAL 30 DAY) |
| ), |
| builds AS ( |
| WITH |
| all_builds AS ( |
| SELECT |
| DISTINCT exported.id AS build_inv_id, |
| variant.*, |
| partition_time |
| FROM |
| `chrome-luci-data.chrome.gpu_try_test_results` AS tr, |
| UNNEST(variant) AS variant, |
| submitted_builds sb |
| WHERE |
| DATE(partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.realm = "chrome:try" |
| AND key = "builder" |
| AND exported.id = sb.id |
| ), |
| grouped_builds AS ( |
| SELECT |
| build_inv_id, |
| value AS builder, |
| partition_time, |
| RANK() OVER (PARTITION BY value ORDER BY partition_time DESC) AS rank_idx, |
| FROM all_builds |
| ) |
| SELECT |
| build_inv_id, |
| builder, |
| partition_time |
| FROM grouped_builds |
| WHERE rank_idx <= 15 |
| ), |
| results AS ( |
| SELECT |
| exported.id, |
| test_id, |
| status, |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "builder" |
| ) as builder_name, |
| IFNULL( |
| ( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "step_name"), |
| ( |
| SELECT value |
| FROM tr.variant |
| WHERE key = "test_suite")) as step_name, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "typ_tag") as typ_tags, |
| ARRAY( |
| SELECT value |
| FROM tr.tags |
| WHERE key = "raw_typ_expectation") as typ_expectations |
| FROM |
| `chrome-luci-data.chrome.gpu_try_test_results` tr, |
| builds b |
| WHERE |
| DATE(tr.partition_time) > DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) |
| AND exported.id = build_inv_id |
| AND status != "SKIP" |
| AND REGEXP_CONTAINS( |
| test_id, |
| "gpu_tests\\\\.webgl1_conformance_integration_test\\\\.") |
| ) |
| SELECT id, test_id, builder_name, status, step_name, typ_tags |
| FROM results |
| WHERE |
| "Failure" IN UNNEST(typ_expectations) |
| OR "RetryOnFailure" IN UNNEST(typ_expectations) |
| ORDER BY builder_name DESC |
| """ |
| self.assertEqual(self._querier._GetInternalTryQuery(), expected_query) |
| |
| |
| class HelperMethodUnittest(unittest.TestCase): |
| def setUp(self) -> None: |
| self.instance = gpu_uu.CreateGenericGpuQuerier() |
| |
| def testStripPrefixFromTestIdValidId(self): |
| test_name = 'conformance/programs/program-handling.html' |
| prefix = ('ninja://chrome/test:telemetry_gpu_integration_test/' |
| 'gpu_tests.webgl_conformance_integration_test.' |
| 'WebGLConformanceIntegrationTest.') |
| test_id = prefix + test_name |
| self.assertEqual(self.instance._StripPrefixFromTestId(test_id), test_name) |
| |
| def testStripPrefixFromTestIdInvalidId(self) -> None: |
| test_name = 'conformance/programs/program-handling_html' |
| prefix = ('ninja://chrome/test:telemetry_gpu_integration_test/' |
| 'gpu_testse.webgl_conformance_integration_test.') |
| test_id = prefix + test_name |
| with self.assertRaises(AssertionError): |
| self.instance._StripPrefixFromTestId(test_id) |
| |
| |
| if __name__ == '__main__': |
| unittest.main(verbosity=2) |