Always put Telemetry tests at the end to avoid starving fast tests
This also add this check to PRESUBMIT perf_json_config_validator script
Bug: 873389
Cq-Include-Trybots: master.tryserver.chromium.perf:obbs_fyi
Change-Id: I16fd762e567d5a41d75a221456e12e14a03f0b85
Reviewed-on: https://chromium-review.googlesource.com/1171578
Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#582422}diff --git a/testing/buildbot/chromium.perf.fyi.json b/testing/buildbot/chromium.perf.fyi.json
index 5d6eb15..f4f9e529 100644
--- a/testing/buildbot/chromium.perf.fyi.json
+++ b/testing/buildbot/chromium.perf.fyi.json
@@ -184,6 +184,49 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "views_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "views_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "/creds/service_accounts/service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "views_perftests",
+ "override_compile_targets": [
+ "views_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "gpu": "8086:0a2e",
+ "os": "Mac-10.12",
+ "pool": "chrome.tests.perf-fyi"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=release",
"--upload-results",
@@ -225,49 +268,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "views_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "views_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "/creds/service_accounts/service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "views_perftests",
- "override_compile_targets": [
- "views_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:0a2e",
- "os": "Mac-10.12",
- "pool": "chrome.tests.perf-fyi"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
},
diff --git a/testing/buildbot/chromium.perf.json b/testing/buildbot/chromium.perf.json
index 34980d8..141c5a0c 100644
--- a/testing/buildbot/chromium.perf.json
+++ b/testing/buildbot/chromium.perf.json
@@ -147,6 +147,51 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "tracing_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "tracing_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "/creds/service_accounts/service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "tracing_perftests",
+ "override_compile_targets": [
+ "tracing_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "device_os": "KOT49H",
+ "device_os_flavor": "google",
+ "device_type": "hammerhead",
+ "os": "Android",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=android-chromium",
"--upload-results",
@@ -191,51 +236,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "tracing_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "tracing_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "/creds/service_accounts/service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "tracing_perftests",
- "override_compile_targets": [
- "tracing_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "device_os": "KOT49H",
- "device_os_flavor": "google",
- "device_type": "hammerhead",
- "os": "Android",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
},
@@ -535,51 +535,6 @@
},
{
"args": [
- "-v",
- "--browser=release_x64",
- "--upload-results",
- "--run-ref-build",
- "--test-shard-map-filename=win7_nvidia_shard_map.json",
- "--assert-gpu-compositing"
- ],
- "isolate_name": "performance_test_suite",
- "merge": {
- "args": [
- "--service-account-file",
- "C:\\creds\\service_accounts\\service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "performance_test_suite",
- "override_compile_targets": [
- "performance_test_suite"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "10de:1cb3",
- "os": "Windows-2008ServerR2-SP1",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 5,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
- },
- {
- "args": [
"--gtest-benchmark-name",
"validating_command_buffer_perftests",
"--non-telemetry=true",
@@ -622,6 +577,51 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
+ },
+ {
+ "args": [
+ "-v",
+ "--browser=release_x64",
+ "--upload-results",
+ "--run-ref-build",
+ "--test-shard-map-filename=win7_nvidia_shard_map.json",
+ "--assert-gpu-compositing"
+ ],
+ "isolate_name": "performance_test_suite",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "C:\\creds\\service_accounts\\service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "performance_test_suite",
+ "override_compile_targets": [
+ "performance_test_suite"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "gpu": "10de:1cb3",
+ "os": "Windows-2008ServerR2-SP1",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 5,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
}
]
},
@@ -1047,6 +1047,51 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "tracing_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "tracing_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "/creds/service_accounts/service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "tracing_perftests",
+ "override_compile_targets": [
+ "tracing_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "device_os": "MMB29Q",
+ "device_os_flavor": "google",
+ "device_type": "bullhead",
+ "os": "Android",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=android-chromium",
"--upload-results",
@@ -1091,51 +1136,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "tracing_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "tracing_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "/creds/service_accounts/service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "tracing_perftests",
- "override_compile_targets": [
- "tracing_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "device_os": "MMB29Q",
- "device_os_flavor": "google",
- "device_type": "bullhead",
- "os": "Android",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
},
@@ -1272,6 +1272,49 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "tracing_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "tracing_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "/creds/service_accounts/service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "tracing_perftests",
+ "override_compile_targets": [
+ "tracing_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "gpu": "10de:1cb3",
+ "os": "Ubuntu-14.04",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=release",
"--upload-results",
@@ -1314,49 +1357,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "tracing_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "tracing_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "/creds/service_accounts/service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "tracing_perftests",
- "override_compile_targets": [
- "tracing_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "10de:1cb3",
- "os": "Ubuntu-14.04",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
},
@@ -1584,6 +1584,49 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "views_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "views_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "/creds/service_accounts/service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "views_perftests",
+ "override_compile_targets": [
+ "views_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "gpu": "1002:6821",
+ "os": "Mac-10.13",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=release",
"--upload-results",
@@ -1625,49 +1668,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "views_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "views_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "/creds/service_accounts/service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "views_perftests",
- "override_compile_targets": [
- "views_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "1002:6821",
- "os": "Mac-10.13",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
},
@@ -1761,6 +1761,49 @@
},
{
"args": [
+ "--gtest-benchmark-name",
+ "views_perftests",
+ "--non-telemetry=true",
+ "--migrated-test=true"
+ ],
+ "isolate_name": "views_perftests",
+ "merge": {
+ "args": [
+ "--service-account-file",
+ "C:\\creds\\service_accounts\\service-account-chromium-perf-histograms.json"
+ ],
+ "script": "//tools/perf/process_perf_results.py"
+ },
+ "name": "views_perftests",
+ "override_compile_targets": [
+ "views_perftests"
+ ],
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "gpu": "8086:5912",
+ "os": "Windows-10",
+ "pool": "chrome.tests.perf"
+ }
+ ],
+ "expiration": 7200,
+ "hard_timeout": 25200,
+ "ignore_task_failure": false,
+ "io_timeout": 1800,
+ "shards": 1,
+ "upload_test_results": true
+ },
+ "trigger_script": {
+ "args": [
+ "--multiple-dimension-script-verbose",
+ "True"
+ ],
+ "script": "//testing/trigger_scripts/perf_device_trigger.py"
+ }
+ },
+ {
+ "args": [
"-v",
"--browser=release_x64",
"--upload-results",
@@ -1803,49 +1846,6 @@
],
"script": "//testing/trigger_scripts/perf_device_trigger.py"
}
- },
- {
- "args": [
- "--gtest-benchmark-name",
- "views_perftests",
- "--non-telemetry=true",
- "--migrated-test=true"
- ],
- "isolate_name": "views_perftests",
- "merge": {
- "args": [
- "--service-account-file",
- "C:\\creds\\service_accounts\\service-account-chromium-perf-histograms.json"
- ],
- "script": "//tools/perf/process_perf_results.py"
- },
- "name": "views_perftests",
- "override_compile_targets": [
- "views_perftests"
- ],
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:5912",
- "os": "Windows-10",
- "pool": "chrome.tests.perf"
- }
- ],
- "expiration": 7200,
- "hard_timeout": 25200,
- "ignore_task_failure": false,
- "io_timeout": 1800,
- "shards": 1,
- "upload_test_results": true
- },
- "trigger_script": {
- "args": [
- "--multiple-dimension-script-verbose",
- "True"
- ],
- "script": "//testing/trigger_scripts/perf_device_trigger.py"
- }
}
]
}
diff --git a/tools/perf/core/perf_data_generator.py b/tools/perf/core/perf_data_generator.py
index 9d5d890..a99b6cf 100755
--- a/tools/perf/core/perf_data_generator.py
+++ b/tools/perf/core/perf_data_generator.py
@@ -71,14 +71,6 @@
'OBBS Mac 10.12 Perf': {
'tests': [
{
- 'isolate': 'performance_test_suite',
- 'extra_args': [
- '--run-ref-build',
- '--test-shard-map-filename=mac1012_5_shard_map.json',
- ],
- 'num_shards': 5
- },
- {
'isolate': 'net_perftests',
'num_shards': 1,
'telemetry': False,
@@ -87,6 +79,14 @@
'isolate': 'views_perftests',
'num_shards': 1,
'telemetry': False,
+ },
+ {
+ 'isolate': 'performance_test_suite',
+ 'extra_args': [
+ '--run-ref-build',
+ '--test-shard-map-filename=mac1012_5_shard_map.json',
+ ],
+ 'num_shards': 5
}
],
'platform': 'mac',
@@ -949,11 +949,20 @@
def generate_telemetry_tests(testers, tests):
for tester, tester_config in testers['testers'].iteritems():
- isolated_scripts = []
+ telemetry_tests = []
+ gtest_tests = []
for test in tester_config['tests']:
- isolated_scripts.append(generate_performance_test(tester_config, test))
+ generated_script = generate_performance_test(tester_config, test)
+ if test.get('telemetry', True):
+ telemetry_tests.append(generated_script)
+ else:
+ gtest_tests.append(generated_script)
+ telemetry_tests.sort(key=lambda x: x['name'])
+ gtest_tests.sort(key=lambda x: x['name'])
tests[tester] = {
- 'isolated_scripts': sorted(isolated_scripts, key=lambda x: x['name'])
+ # Put Telemetry tests as the end since they tend to run longer to avoid
+ # starving gtests (see crbug.com/873389).
+ 'isolated_scripts': gtest_tests + telemetry_tests
}
diff --git a/tools/perf/core/perf_json_config_validator.py b/tools/perf/core/perf_json_config_validator.py
index 3eb2e616..dfefede8 100644
--- a/tools/perf/core/perf_json_config_validator.py
+++ b/tools/perf/core/perf_json_config_validator.py
@@ -101,7 +101,9 @@
def ValidateTestingBuilder(builder_name, builder_data):
isolated_scripts = builder_data['isolated_scripts']
+ test_names = []
for test_config in isolated_scripts:
+ test_names.append(test_config['name'])
_ValidateSwarmingDimension(
builder_name,
swarming_dimensions=test_config['swarming'].get('dimension_sets', {}))
@@ -110,6 +112,17 @@
_ValidateShardingData(builder_name, test_config)
_ValidateBrowserType(builder_name, test_config)
+ if ('performance_test_suite' in test_names or
+ 'performance_webview_test_suite' in test_names):
+ if test_names[-1] not in ('performance_test_suite',
+ 'performance_webview_test_suite'):
+ raise ValueError(
+ 'performance_test_suite or performance_webview_test_suite must run '
+ 'at the end of builder %s to avoid starving other test step '
+ '(see crbug.com/873389). Instead found %s' % (
+ repr(builder_name), test_names[-1]))
+
+
def _IsBuilderName(name):
return not name.startswith('AAA')
