| #!/usr/bin/env vpython3 |
| # Copyright 2025 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Unittest for json_util.py.""" |
| |
| import copy |
| import datetime |
| import unittest |
| from unittest import mock |
| import json_util |
| from parameterized import parameterized # pylint: disable=import-error |
| |
| # pylint: disable=too-many-lines |
| |
| |
| class JsonUtilTest(unittest.TestCase): |
| |
| @parameterized.expand([ |
| ( |
| 'empty_stories_and_tags', |
| [], |
| [], |
| ['', ''], |
| ), |
| ( |
| 'just_stories', |
| ['load:news:reddit:2018'], |
| [], |
| ['load:news:reddit:2018', ''], |
| ), |
| ( |
| 'just_tags', |
| [], |
| ['2018', 'case:load', 'group:news'], |
| ['load_news', ''], |
| ), |
| ( |
| 'just_tags_2', |
| ['no_colon_stories'], |
| ['2018', 'case:load', 'group:games'], |
| ['load_games', ''], |
| ), |
| ( |
| 'valid_stories_and_tags', |
| ['load:news:reddit:2018'], |
| ['2018', 'case:load', 'group:news'], |
| ['load_news', 'load_news_reddit_2018'], |
| ), |
| ( |
| 'valid_stories_and_tags_2', |
| ['load:games:lazors'], |
| [ |
| '2016', |
| 'case:load', |
| 'group:games', |
| ], |
| ['load_games', 'load_games_lazors'], |
| ), |
| ( |
| 'valid_stories_and_tags_3', |
| ['browse:tools:photoshop_warm:2021'], |
| [ |
| '2021', |
| 'wasm', |
| 'case:browse', |
| 'group:tools', |
| ], |
| ['browse_tools', 'browse_tools_photoshop_warm_2021'], |
| ), |
| ( |
| 'valid_stories_and_tags_4', |
| ['speedometer3'], |
| [ |
| '2021', |
| 'wasm', |
| 'case', |
| 'group', |
| ], |
| ['speedometer3', ''], |
| ), |
| ]) |
| def test_extract_subtest_from_stories_tags(self, _, stories, tags, expected): |
| got_1, got_2 = json_util.extract_subtest_from_stories_tags(stories, tags) |
| self.assertEqual( |
| [got_1, got_2], |
| expected, |
| ) |
| |
| def test_calculate_stats(self): |
| with self.subTest(name='normal_values'): |
| self.assertEqual( |
| json_util.calculate_stats([1, 2, 3]), |
| (2.0, 1.0, 3, 3, 1, 6), |
| ) |
| with self.subTest(name='mixed_with_None'): |
| self.assertEqual( |
| json_util.calculate_stats([1, None, 3, None]), |
| (2.0, 1.4142135623730951, 2, 3, 1, 4), |
| ) |
| with self.subTest(name='mixed_with_zero'): |
| self.assertEqual( |
| json_util.calculate_stats([1, 0, 3, 0]), |
| (1.0, 1.4142135623730951, 4, 3, 0, 4), |
| ) |
| with self.subTest(name='mixed_with_None_and_zero'): |
| self.assertEqual( |
| json_util.calculate_stats([1, None, 3, 0]), |
| (1.3333333333333333, 1.5275252316519468, 3, 3, 0, 4), |
| ) |
| with self.subTest(name='empty_list'): |
| self.assertEqual( |
| json_util.calculate_stats([]), |
| (0.0, 0.0, 0, 0, 0, 0), |
| ) |
| |
| def test_process(self): |
| result2_json = [ |
| { |
| 'type': 'GenericSet', |
| 'guid': '35d015dd-887e-4b34-a8d5-0be7599dec78', |
| 'values': ['speedometer3'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '9ef9da4a-3b79-4574-9603-bf9e2fe4bbe7', |
| 'values': ['speedometer3'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '69d51091-2cd5-4f0d-a87e-9c08254e1a16', |
| 'values': ['The latest version of the Speedometer3 benchmark.'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '6214d34c-9200-4271-9cea-895f24e99d5c', |
| 'values': ['win-222-e504'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '947e0701-97ef-4e0c-8bde-2b565aa1a087', |
| 'values': ['cbruni@chromium.org', 'vahl@chromium.org'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '93bae44b-c29d-48d6-99fa-a7c6bc962751', |
| 'values': ['Blink>JavaScript'], |
| }, |
| { |
| 'type': |
| 'GenericSet', |
| 'guid': |
| 'f0a439d6-f0d2-4dcd-8bca-8f3912cd760f', |
| 'values': [[ |
| 'Benchmark documentation link', |
| 'https://github.com/WebKit/Speedometer', |
| ]], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '454e29c4-57dd-4987-9e3e-792ec84f07ce', |
| 'values': ['AMD64'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': 'dfb2f28d-fb32-49f2-b011-1d26853a6d6c', |
| 'values': ['win'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '84d184f8-cfb0-4929-994b-85f908b91839', |
| 'values': ['win10'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': '6e8ba79c-9588-426f-92cf-a3ab803bcbba', |
| 'values': ['10.0.19045'], |
| }, |
| { |
| 'type': 'DateRange', |
| 'guid': 'b5c3d0fa-e3dd-4e32-ab41-31c8a3ab0e2a', |
| 'min': 1736637289209.919, |
| }, |
| { |
| # this is the stories |
| 'type': 'GenericSet', |
| 'guid': '6903cc09-ca9e-47a4-82a0-d77d214d37fc', |
| 'values': ['Speedometer3'], |
| }, |
| { |
| 'type': 'GenericSet', |
| 'guid': 'a7f54f55-870b-4b76-bb87-d64df4bf3e7b', |
| 'values': [0], |
| }, |
| { |
| # this is the storyTags |
| 'type': 'GenericSet', |
| 'guid': 'f0bb92d7-5ab2-42ed-ad7f-d79018aa3b60', |
| 'values': ['all'], |
| }, |
| { |
| # this is the stories that yield subtest_2 |
| 'type': 'GenericSet', |
| 'guid': '6903cc09-ca9e-47a4-82a0-d77d214d37fd', |
| 'values': ['browse:news:nytimes:2020'], |
| }, |
| { |
| # this is the storyTags that yield subtest_1 |
| 'type': 'GenericSet', |
| 'guid': 'f0bb92d7-5ab2-42ed-ad7f-d79018aa3b61', |
| 'values': ['2020', 'case:browse', 'group:news'], |
| }, |
| { |
| 'type': 'DateRange', |
| 'guid': 'cf09d1a1-8b3b-4d3c-bee6-b8d341d5e31e', |
| 'min': 1736637289209.919, |
| }, |
| { |
| 'name': |
| 'Editor-TipTap', |
| 'unit': |
| 'ms_smallerIsBetter', |
| 'binBoundaries': [0.001, [1, 1000000.0, 100]], |
| 'diagnostics': { |
| 'benchmarks': '9ef9da4a-3b79-4574-9603-bf9e2fe4bbe7', |
| 'benchmarkDescriptions': '69d51091-2cd5-4f0d-a87e-9c08254e1a16', |
| 'botId': '6214d34c-9200-4271-9cea-895f24e99d5c', |
| 'owners': '947e0701-97ef-4e0c-8bde-2b565aa1a087', |
| 'bugComponents': '93bae44b-c29d-48d6-99fa-a7c6bc962751', |
| 'documentationLinks': 'f0a439d6-f0d2-4dcd-8bca-8f3912cd760f', |
| 'architectures': '454e29c4-57dd-4987-9e3e-792ec84f07ce', |
| 'osNames': 'dfb2f28d-fb32-49f2-b011-1d26853a6d6c', |
| 'osVersions': '84d184f8-cfb0-4929-994b-85f908b91839', |
| 'osDetailedVersions': '6e8ba79c-9588-426f-92cf-a3ab803bcbba', |
| 'benchmarkStart': 'b5c3d0fa-e3dd-4e32-ab41-31c8a3ab0e2a', |
| 'stories': '6903cc09-ca9e-47a4-82a0-d77d214d37fc', |
| 'storysetRepeats': 'a7f54f55-870b-4b76-bb87-d64df4bf3e7b', |
| 'storyTags': 'f0bb92d7-5ab2-42ed-ad7f-d79018aa3b60', |
| 'traceStart': 'cf09d1a1-8b3b-4d3c-bee6-b8d341d5e31e', |
| }, |
| 'sampleValues': [ |
| 172.90000000130385, |
| 135.3999999994412, |
| 137.10000000149012, |
| 133, |
| 134.5, |
| 130.90000000037253, |
| 137.70000000018626, |
| 158.1000000005588, |
| 133.09999999962747, |
| 134.19999999925494, |
| ], |
| 'running': [ |
| 10, |
| 172.90000000130385, |
| 4.942674143981753, |
| 140.6900000002235, |
| 130.90000000037253, |
| 1406.9000000022352, |
| 1683.429000105582, |
| ], |
| 'allBins': { |
| '57': [5], |
| '58': [4], |
| '59': [1] |
| }, |
| }, |
| ] |
| key = { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap', |
| 'subtest_1': 'Speedometer3', |
| } |
| measurement = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 140.6900000002235 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 13.676537086499565 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 10.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 172.90000000130385 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 130.90000000037253 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 1406.9000000022352 |
| }, |
| ] |
| }, |
| 'key': key, |
| } |
| links = { |
| 'Build Page': |
| ('https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376'), |
| 'OS Version': |
| '10.0.19045', |
| 'Bot Id': |
| 'win-222-e504', |
| 'Chromium Commit Position': |
| 'https://crrev.com/1405221', |
| 'V8': ('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| 'WebRTC': ('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| } |
| expected = { |
| 'version': 1, |
| 'git_hash': 'CP:1405221', |
| 'key': { |
| 'master': 'ChromiumPerf', |
| 'bot': 'win-10-perf', |
| 'benchmark': 'speedometer3', |
| }, |
| 'results': [measurement], |
| 'links': links, |
| } |
| details = json_util.PerfBuilderDetails( |
| bot='win-10-perf', |
| builder_page=( |
| 'https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376' |
| ), |
| git_hash='CP:1405221', |
| chromium_commit_position='https://crrev.com/1405221', |
| master='ChromiumPerf', |
| v8_git_hash=('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| webrtc_git_hash=('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| ) |
| with self.subTest(name='no_synthetic_measurements'): |
| agent = json_util.JsonUtil(generate_synthetic_measurements=False) |
| agent.add(result2_json) |
| got = agent.process(details) |
| self.assertDictEqual(got, expected) |
| |
| with self.subTest(name='no_synthetic_measurements_with_benchmark_name'): |
| expected2 = copy.deepcopy(expected) |
| expected2['key']['benchmark'] = 'speedometer3_modified' |
| agent = json_util.JsonUtil(generate_synthetic_measurements=False) |
| agent.add(result2_json) |
| got = agent.process(details, benchmark_name='speedometer3_modified') |
| self.assertDictEqual(got, expected2) |
| |
| with self.subTest(name='generate_synthetic_measurements'): |
| agent = json_util.JsonUtil(generate_synthetic_measurements=True) |
| agent.add(result2_json) |
| synthetic_measurements_1 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'average', |
| 'measurement': 140.6900000002235 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap_avg', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| synthetic_measurements_2 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'min', |
| 'measurement': 130.90000000037253 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap_min', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| synthetic_measurements_3 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'max', |
| 'measurement': 172.90000000130385 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap_max', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| synthetic_measurements_4 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'sum', |
| 'measurement': 1406.9000000022352 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap_sum', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| synthetic_measurements_5 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'count', |
| 'measurement': 10.0 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'up', |
| 'unit': 'unitless_biggerIsBetter', |
| 'test': 'Editor-TipTap_count', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| synthetic_measurements_6 = { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'error', |
| 'measurement': 13.676537086499565 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap_std', |
| 'subtest_1': 'Speedometer3', |
| }, |
| } |
| expected['results'].extend([ |
| synthetic_measurements_1, |
| synthetic_measurements_2, |
| synthetic_measurements_3, |
| synthetic_measurements_4, |
| synthetic_measurements_5, |
| synthetic_measurements_6]) |
| got = agent.process(details) |
| self.assertDictEqual(got, expected) |
| with self.subTest(name='generate_synthetic_measurements_with_subtest'): |
| agent = json_util.JsonUtil(generate_synthetic_measurements=True) |
| # modifies result2_json_copy to support subtest_1 and subtest_2 |
| result2_json_copy = copy.deepcopy(result2_json) |
| for item in result2_json_copy: |
| if 'diagnostics' in item: |
| item['diagnostics'][ |
| 'stories'] = '6903cc09-ca9e-47a4-82a0-d77d214d37fd' |
| item['diagnostics'][ |
| 'storyTags'] = 'f0bb92d7-5ab2-42ed-ad7f-d79018aa3b61' |
| key['subtest_1'] = 'browse_news' |
| key['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_1['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_1['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_2['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_2['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_3['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_3['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_4['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_4['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_5['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_5['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| synthetic_measurements_6['key']['subtest_1'] = 'browse_news' |
| synthetic_measurements_6['key']['subtest_2'] = 'browse_news_nytimes_2020' |
| expected = { |
| 'version': 1, |
| 'git_hash': 'CP:1405221', |
| 'key': { |
| 'master': 'ChromiumPerf', |
| 'bot': 'win-10-perf', |
| 'benchmark': 'speedometer3', |
| }, |
| 'results': [ |
| measurement, |
| synthetic_measurements_1, |
| synthetic_measurements_2, |
| synthetic_measurements_3, |
| synthetic_measurements_4, |
| synthetic_measurements_5, |
| synthetic_measurements_6], |
| 'links': links, |
| } |
| agent.add(result2_json_copy) |
| got = agent.process(details) |
| self.assertDictEqual(got, expected) |
| |
| @parameterized.expand([ |
| ( |
| 'empty_data', |
| False, |
| None, |
| [], |
| ), |
| ( |
| 'without_subtest', |
| False, |
| { |
| ('abc', 'ms_smallerIsBetter', 'down'): [1, 2, 3], |
| }, |
| [{ |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 2.0 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc', |
| }, |
| }], |
| ), |
| ( |
| 'with_subtest', |
| False, |
| { |
| ('abc', 'ms_smallerIsBetter', 'down', 'subtest', 'subtest2'): [ |
| 1, |
| 2, |
| 3, |
| ], |
| }, |
| [{ |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 2.0 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }], |
| ), |
| ( |
| 'without_subtest_with_synthetic_measurements', |
| True, |
| { |
| ('abc', 'ms_smallerIsBetter', 'down'): [1, 2, 3], |
| }, |
| [ |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 2.0 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'average', |
| 'measurement': 2.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_avg', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_min', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_max', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_sum', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'up', |
| 'unit': 'unitless_biggerIsBetter', |
| 'test': 'abc_count', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_std', |
| }, |
| }, |
| ], |
| ), |
| ( |
| 'with_subtest_with_synthetic_measurements', |
| True, |
| { |
| ('abc', 'ms_smallerIsBetter', 'down', 'subtest', 'subtest2'): [ |
| 1, |
| 2, |
| 3, |
| ], |
| }, |
| [ |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 2.0 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'average', |
| 'measurement': 2.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_avg', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'min', |
| 'measurement': 1.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_min', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'max', |
| 'measurement': 3.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_max', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'sum', |
| 'measurement': 6.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_sum', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'count', |
| 'measurement': 3.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'up', |
| 'unit': 'unitless_biggerIsBetter', |
| 'test': 'abc_count', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| { |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'error', |
| 'measurement': 1.0 |
| }, |
| ], |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'abc_std', |
| 'subtest_1': 'subtest', |
| 'subtest_2': 'subtest2', |
| }, |
| }, |
| ], |
| ), |
| ]) |
| def test_measurement(self, _, synthetic_measurements, data, expected): |
| instance = json_util.JsonUtil( |
| generate_synthetic_measurements=synthetic_measurements) |
| got = instance.measurements_from_results(data=data) |
| self.assertEqual(expected, got) |
| |
| def test_key_from_builder_details(self): |
| builder_details = json_util.PerfBuilderDetails( |
| bot='win-10-perf', |
| builder_page=( |
| 'https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376' |
| ), |
| git_hash='CP:1405221', |
| chromium_commit_position='https://crrev.com/1405221', |
| master='ChromiumPerf', |
| v8_git_hash=('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| webrtc_git_hash=('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| ) |
| got = json_util.key_from_builder_details(builder_details=builder_details, |
| benchmark_key='speedometer3') |
| expected = { |
| 'master': 'ChromiumPerf', |
| 'bot': 'win-10-perf', |
| 'benchmark': 'speedometer3', |
| } |
| self.assertDictEqual(got, expected) |
| |
| def test_links_from_builder_details_no_benchmark_key(self): |
| builder_details = json_util.PerfBuilderDetails( |
| bot='win-10-perf', |
| builder_page=( |
| 'https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376' |
| ), |
| git_hash='CP:1405221', |
| chromium_commit_position='https://crrev.com/1405221', |
| master='ChromiumPerf', |
| v8_git_hash=('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| webrtc_git_hash=('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| ) |
| got = json_util.links_from_builder_details( |
| builder_details=builder_details, |
| bot_ids={'win-222-e504', 'win-223-e504', 'win-224-e504'}, |
| os_versions={'10.0.19045'}, |
| ) |
| expected = { |
| 'Build Page': |
| ('https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376'), |
| 'OS Version': |
| '10.0.19045', |
| 'Bot Id': |
| 'win-222-e504, win-223-e504, win-224-e504', |
| 'Chromium Commit Position': |
| 'https://crrev.com/1405221', |
| 'V8': ('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| 'WebRTC': ('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| } |
| self.assertDictEqual(got, expected) |
| |
| def test_perf_builder_details_from_build_properties(self): |
| build_properties = { |
| 'builder_group': 'chromium.perf', |
| 'buildername': 'win-11-perf', |
| 'buildnumber': 9719, |
| 'git_cache_epoch': '1738403861', |
| 'got_angle_revision': '56d796a9f12a4bc2388f0fb37ba473eceefe4a8e', |
| 'got_dawn_revision': '2a3e80d33d4855c8ff8d01f1ba3b076b3088ffd1', |
| 'got_revision': 'bae4f05ce6002349a5b2ac01decb4ce27c47a30b', |
| 'got_revision_cp': 'refs/heads/main@{#1415171}', |
| 'got_src_internal_revision': 'f517862818abc1fed7ed8b3eeed9419f2ce17a71', |
| 'got_swiftshader_revision': '86cf34f50cbe5a9f35da7eedad0f4d4127fb8342', |
| 'got_v8_revision': '0f87a54dade4353b6ece1d7591ca8c66f90c1c93', |
| 'got_v8_revision_cp': 'refs/heads/13.4.114@{#1}', |
| 'got_webrtc_revision': '0533b5eafe69b744f10fa178f5a6f9657eaeeb25', |
| 'got_webrtc_revision_cp': 'refs/heads/main@{#43844}', |
| 'perf_dashboard_machine_group': 'ChromiumPerf' |
| } |
| got = json_util.perf_builder_details_from_build_properties( |
| build_properties, 'win64-builder-perf', 'ChromiumPerf') |
| expected = json_util.PerfBuilderDetails( |
| bot='win64-builder-perf', |
| builder_page= |
| 'https://ci.chromium.org/ui/p/chrome/builders/ci/win-11-perf/9719', |
| git_hash='CP:1415171', |
| chromium_commit_position='refs/heads/main@{#1415171}', |
| master='ChromiumPerf', |
| v8_git_hash='0f87a54dade4353b6ece1d7591ca8c66f90c1c93', |
| webrtc_git_hash='0533b5eafe69b744f10fa178f5a6f9657eaeeb25', |
| ) |
| self.assertEqual(got, expected) |
| |
| @parameterized.expand([ |
| ( |
| 'empty_filename', |
| { |
| 'buildername': 'win-11-perf', |
| 'buildnumber': 9719, |
| }, |
| json_util.PerfBuilderDetails( |
| bot='win64-builder-perf', |
| master='ChromiumPerf', |
| builder_page='http://foo.link', |
| git_hash='CP:deadbeef', |
| chromium_commit_position='refs/heads/master@{#1234}', |
| v8_git_hash='beef1234', |
| webrtc_git_hash='fee123', |
| ), |
| 'benchmark.example', |
| datetime.datetime(2024, 8, 29), |
| '', |
| ('ingest/2024/08/29/ChromiumPerf/' |
| 'win-11-perf/9719/benchmark.example'), |
| ), |
| ( |
| 'non_empty_filename', |
| { |
| 'buildername': 'win-11-perf', |
| 'buildnumber': 9719, |
| }, |
| json_util.PerfBuilderDetails( |
| bot='win64-builder-perf', |
| master='ChromiumPerf', |
| builder_page='http://foo.link', |
| git_hash='CP:deadbeef', |
| chromium_commit_position='refs/heads/master@{#1234}', |
| v8_git_hash='beef1234', |
| webrtc_git_hash='fee123', |
| ), |
| 'benchmark.example', |
| datetime.datetime(2024, 8, 29), |
| 'skia_results.json', |
| ('ingest/2024/08/29/ChromiumPerf/' |
| 'win-11-perf/9719/benchmark.example/skia_results.json'), |
| ), |
| ]) |
| def test_get_gcs_prefix_path( |
| self, _, build_properties, builder_details, |
| benchmark_name, given_datetime, filename, expected): |
| got = json_util.get_gcs_prefix_path( |
| build_properties=build_properties, |
| builder_details=builder_details, |
| benchmark_name=benchmark_name, |
| given_datetime=given_datetime, |
| filename=filename) |
| self.assertEqual(got, expected) |
| |
| @mock.patch('builtins.open', new_callable=mock.mock_open) |
| def test_is_public_builder(self, mock_open): |
| mock_open.return_value.__enter__.return_value.read.return_value = ( |
| '{"public_perf_builders": ["win-10-perf"]}') |
| self.assertTrue(json_util.is_public_builder('')) |
| self.assertTrue(json_util.is_public_builder('win-10-perf')) |
| self.assertFalse(json_util.is_public_builder('win-11-perf')) |
| |
| @parameterized.expand([ |
| ( |
| 'empty_master_name', |
| 'win-11-perf', |
| '', |
| False, |
| False, |
| [], |
| ), |
| ( |
| 'empty_builder_name', |
| '', |
| 'ChromiumPerf', |
| False, |
| False, |
| [], |
| ), |
| ( |
| 'public_builder', |
| 'win-10-perf', |
| 'ChromiumPerf', |
| False, |
| False, |
| ['chrome-perf-public', 'chrome-perf-non-public'], |
| ), |
| ( |
| 'public_builder_with_copy_to_experiment', |
| 'win-10-perf', |
| 'ChromiumPerf', |
| False, |
| True, |
| ['chrome-perf-public', 'chrome-perf-non-public', |
| 'chrome-perf-experiment-non-public'], |
| ), |
| ( |
| 'internal_builder', |
| 'win-11-perf', |
| 'ChromiumPerf', |
| False, |
| False, |
| ['chrome-perf-non-public'], |
| ), |
| ( |
| 'internal_builder_with_copy_to_experiment', |
| 'win-11-perf', |
| 'ChromiumPerf', |
| False, |
| True, |
| ['chrome-perf-non-public'], |
| ), |
| ( |
| 'another_internal_builder', |
| 'android-pixel6-perf', |
| 'ChromiumPerf', |
| False, |
| False, |
| ['chrome-perf-non-public'], |
| ), |
| ( |
| 'experiment_only', |
| 'win-11-perf', |
| 'ChromiumPerf', |
| True, |
| False, |
| ['chrome-perf-experiment-non-public'], |
| ), |
| ]) |
| def test_gcs_buckets_from_builder_name( |
| self, _, builder_name, master_name, experiment_only, |
| copy_to_experiment, expected): |
| with mock.patch('builtins.open', new_callable=mock.mock_open) as mock_open: |
| mock_open.return_value.__enter__.return_value.read.return_value = ( |
| '{"public_perf_builders": ["win-10-perf"]}') |
| got = json_util.gcs_buckets_from_builder_name( |
| builder_name=builder_name, |
| master_name=master_name, |
| experiment_only=experiment_only, |
| public_copy_to_experiment=copy_to_experiment) |
| self.assertEqual(got, expected) |
| |
| @parameterized.expand([ |
| ( |
| 'default_up_on_empty_string', |
| '', |
| 'up', |
| ), |
| ( |
| 'foo_smallerIsBetter', |
| 'foo_smallerIsBetter', |
| 'down', |
| ), |
| ( |
| 'smallerIsBetter', |
| '_smallerIsBetter', |
| 'down', |
| ), |
| ( |
| 'abc_smallerIsBetter', |
| 'abc_smallerIsBetter', |
| 'down', |
| ), |
| ( |
| 'default_up', |
| 'foo_bar', |
| 'up', |
| ), |
| ( |
| 'bar_biggerIsBetter', |
| 'bar_biggerIsBetter', |
| 'up', |
| ), |
| ]) |
| def test_get_improvement_direction( |
| self, _, unit, expected): |
| # pylint: disable=protected-access |
| got= json_util._get_improvement_direction(unit) |
| self.assertEqual(got, expected) |
| |
| |
| @parameterized.expand([ |
| ('none', None, True), |
| ( |
| 'with_empty_results', |
| { |
| 'version': 1, |
| 'git_hash': 'CP:1405221', |
| 'key': { |
| 'master': 'ChromiumPerf', |
| 'bot': 'win-10-perf', |
| 'benchmark': 'speedometer3', |
| }, |
| 'results': [], |
| 'links': { |
| 'Build Page': |
| ('https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376' |
| ), |
| 'OS Version': |
| '10.0.19045', |
| 'Bot Id': |
| 'win-222-e504', |
| 'Chromium Commit Position': |
| 'https://crrev.com/1405221', |
| 'V8': ('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| 'WebRTC': ('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| } |
| }, |
| True, |
| ), |
| ( |
| 'with_some_results', |
| { |
| 'version': |
| 1, |
| 'git_hash': |
| 'CP:1405221', |
| 'key': { |
| 'master': 'ChromiumPerf', |
| 'bot': 'win-10-perf', |
| 'benchmark': 'speedometer3', |
| }, |
| 'results': [{ |
| 'measurements': { |
| 'stat': [ |
| { |
| 'value': 'value', |
| 'measurement': 140.6900000002235 |
| }, |
| { |
| 'value': 'error', |
| 'measurement': 13.676537086499565 |
| }, |
| { |
| 'value': 'count', |
| 'measurement': 10.0 |
| }, |
| { |
| 'value': 'max', |
| 'measurement': 172.90000000130385 |
| }, |
| { |
| 'value': 'min', |
| 'measurement': 130.90000000037253 |
| }, |
| { |
| 'value': 'sum', |
| 'measurement': 1406.9000000022352 |
| }, |
| ] |
| }, |
| 'key': { |
| 'improvement_direction': 'down', |
| 'unit': 'ms_smallerIsBetter', |
| 'test': 'Editor-TipTap', |
| 'subtest_1': 'Speedometer3', |
| }, |
| }], |
| 'links': { |
| 'Build Page': |
| ('https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376' |
| ), |
| 'OS Version': |
| '10.0.19045', |
| 'Bot Id': |
| 'win-222-e504', |
| 'Chromium Commit Position': |
| 'https://crrev.com/1405221', |
| 'V8': ('https://chromium.googlesource.com/v8/v8/+/' |
| '60e67b93909a1c858305b27111d9988f94fff0f8'), |
| 'WebRTC': ('https://webrtc.googlesource.com/src/+/' |
| '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), |
| } |
| }, |
| False, |
| ) |
| ]) |
| def test_is_empty(self, _, data, expected): |
| self.assertEqual(json_util.is_empty(data), expected) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
