blob: 7c12563c584756efdec6e3246014b9cb2ee41ea4 [file] [log] [blame]
#!/usr/bin/python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Tests for generate_buildbot_json.py."""
import argparse
import os
import unittest
import generate_buildbot_json
class FakeBBGen(generate_buildbot_json.BBJSONGenerator):
def __init__(self, waterfalls, test_suites, exceptions, mixins,
luci_milo_cfg):
super(FakeBBGen, self).__init__()
infra_config_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..',
'infra', 'config', 'global'))
luci_milo_cfg_path = os.path.join(infra_config_dir, 'luci-milo.cfg')
luci_milo_dev_cfg_path = os.path.join(infra_config_dir, 'luci-milo-dev.cfg')
self.files = {
'waterfalls.pyl': waterfalls,
'test_suites.pyl': test_suites,
'test_suite_exceptions.pyl': exceptions,
'mixins.pyl': mixins,
luci_milo_cfg_path: luci_milo_cfg,
luci_milo_dev_cfg_path: '',
}
self.printed_lines = []
def print_line(self, line):
self.printed_lines.append(line)
def read_file(self, relative_path):
return self.files[relative_path]
def write_file(self, relative_path, contents):
self.files[relative_path] = contents
# pragma pylint: disable=arguments-differ
def check_output_file_consistency(self, verbose=False, dump=True):
try:
super(FakeBBGen, self).check_output_file_consistency(verbose)
except generate_buildbot_json.BBGenErr:
if verbose and dump:
# Assume we want to see the difference in the waterfalls'
# generated output to make it easier to rebaseline the test.
for line in self.printed_lines:
print line
raise
# pragma pylint: enable=arguments-differ
FOO_GTESTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {
'dimension_sets': [
{
'kvm': '1',
},
],
},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'gtest_tests': 'foo_tests',
},
'args': [
'--enable-features=Baz',
],
},
},
},
]
"""
FOO_GTESTS_MULTI_DIMENSION_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {
'dimension_sets': [
{
"gpu": "none",
"os": "1",
},
],
},
'use_multi_dimension_trigger_script': True,
'alternate_swarming_dimensions': [
{
"gpu": "none",
"os": "2",
},
],
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {
'dimension_sets': [
{
"device_type": "foo_device",
},
],
},
'test_suites': {
'gtest_tests': 'foo_tests',
},
'os_type': 'chromeos',
},
},
},
]
"""
FOO_LINUX_GTESTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'os_type': 'linux',
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
COMPOSITION_GTEST_SUITE_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'gtest_tests': 'composition_tests',
},
},
},
},
]
"""
COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'gtest_tests': 'composition_tests',
},
'args': [
'--this-is-an-argument',
],
},
},
},
]
"""
FOO_ISOLATED_SCRIPTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'isolated_scripts': 'composition_tests',
},
},
},
},
]
"""
FOO_SCRIPT_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'scripts': 'foo_scripts',
},
},
},
},
]
"""
FOO_JUNIT_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'junit_tests': 'composition_tests',
},
},
},
},
]
"""
FOO_CTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'cts_tests': 'foo_cts_tests',
},
},
},
},
]
"""
FOO_ISOLATED_CTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'isolated_scripts': 'isolated_foo_cts_tests',
},
'use_android_presentation': True,
},
},
},
]
"""
FOO_INSTRUMENTATION_TEST_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'composition_tests',
},
},
},
},
]
"""
FOO_GPU_TELEMETRY_TEST_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'os_type': 'win',
'browser_config': 'release',
'swarming': {
'dimension_sets': [
{
'gpu': '10de:1cb3',
},
],
},
'test_suites': {
'gpu_telemetry_tests': 'composition_tests',
},
},
},
},
]
"""
UNKNOWN_TEST_SUITE_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'gtest_tests': 'baz_tests',
},
},
},
},
]
"""
UNKNOWN_TEST_SUITE_TYPE_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'gtest_tests': 'foo_tests',
'foo_test_type': 'foo_tests',
},
},
},
},
]
"""
ANDROID_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Android Builder': {
'additional_compile_targets': [
'bar_test',
],
},
'Fake Android K Tester': {
'additional_compile_targets': [
'bar_test',
],
'swarming': {
'dimension_sets': [
{
'device_os': 'KTU84P',
'device_type': 'hammerhead',
},
],
},
'os_type': 'android',
'skip_merge_script': True,
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
'Fake Android L Tester': {
'swarming': {
'dimension_sets': [
{
'device_os': 'KTU84P',
'device_type': 'hammerhead',
},
],
},
'os_type': 'android',
'skip_merge_script': True,
'skip_output_links': True,
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
'Fake Android M Tester': {
'swarming': {
'dimension_sets': [
{
'device_os': 'MMB29Q',
'device_type': 'bullhead',
},
],
},
'os_type': 'android',
'use_swarming': False,
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
UNKNOWN_BOT_GTESTS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Unknown Bot': {
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_TEST_SUITE = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {
'swarming': {
'dimension_sets': [
{
'integrity': 'high',
}
],
'expiration': 120,
},
},
},
},
}
"""
FOO_TEST_SUITE_WITH_ARGS = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {
'args': [
'--c_arg',
],
},
},
},
}
"""
FOO_TEST_SUITE_WITH_LINUX_ARGS = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {
'linux_args': [
'--no-xvfb',
],
},
},
},
}
"""
FOO_TEST_SUITE_WITH_ENABLE_FEATURES = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {
'args': [
'--enable-features=Foo,Bar',
],
},
},
},
}
"""
FOO_SCRIPT_SUITE = """\
{
'basic_suites': {
'foo_scripts': {
'foo_test': {
'script': 'foo.py',
},
'bar_test': {
'script': 'bar.py',
},
},
},
}
"""
FOO_CTS_SUITE = """\
{
'basic_suites': {
'foo_cts_tests': {
'arch': 'arm64',
'platform': 'L',
},
},
}
"""
FOO_ISOLATED_CTS_SUITE = """\
{
'basic_suites': {
'isolated_foo_cts_tests': {
'foo_cts_tests': {},
},
},
}
"""
GOOD_COMPOSITION_TEST_SUITES = """\
{
'basic_suites': {
'bar_tests': {
'bar_test': {},
},
'foo_tests': {
'foo_test': {},
},
},
'compound_suites': {
'composition_tests': [
'foo_tests',
'bar_tests',
],
},
}
"""
BAD_COMPOSITION_TEST_SUITES = """\
{
'basic_suites': {
'bar_tests': {},
'foo_tests': {},
},
'compound_suites': {
'buggy_composition_tests': [
'bar_tests',
],
'composition_tests': [
'foo_tests',
'buggy_composition_tests',
],
},
}
"""
DUPLICATES_COMPOSITION_TEST_SUITES = """\
{
'basic_suites': {
'bar_tests': {},
'foo_tests': {},
},
'compound_suites': {
'bar_tests': [
'foo_tests',
],
'composition_tests': [
'foo_tests',
'buggy_composition_tests',
],
},
}
"""
INSTRUMENTATION_TESTS_WITH_DIFFERENT_NAMES = """\
{
'basic_suites': {
'composition_tests': {
'foo_tests': {
'test': 'foo_test',
},
'bar_tests': {
'test': 'foo_test',
},
},
},
}
"""
SCRIPT_SUITE = """\
{
'basic_suites': {
'foo_scripts': {
'foo_test': {
'script': 'foo.py',
},
},
},
}
"""
UNREFED_TEST_SUITE = """\
{
'basic_suites': {
'bar_tests': {},
'foo_tests': {},
},
}
"""
REUSING_TEST_WITH_DIFFERENT_NAME = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {},
'variation_test': {
'args': [
'--variation',
],
'test': 'foo_test',
},
},
},
}
"""
COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST = """\
{
'basic_suites': {
'foo_tests': {
'foo': {},
},
'bar_tests': {
'bar_test': {},
},
},
'compound_suites': {
'composition_tests': [
'foo_tests',
'bar_tests',
],
},
}
"""
EMPTY_PYL_FILE = """\
{
}
"""
SCRIPT_WITH_ARGS_EXCEPTIONS = """\
{
'foo_test': {
'modifications': {
'Fake Tester': {
'args': ['--fake-arg'],
},
},
},
}
"""
SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS = """\
{
'foo_test': {
'modifications': {
'Fake Tester': {
'swarming': {
'value': 'exception',
},
},
},
},
}
"""
NO_BAR_TEST_EXCEPTIONS = """\
{
'bar_test': {
'remove_from': [
'Fake Tester',
]
}
}
"""
EMPTY_BAR_TEST_EXCEPTIONS = """\
{
'bar_test': {
}
}
"""
EXCEPTIONS_SORTED = """\
{
'suite_c': {
'modifications': {
'Fake Tester': {
'foo': 'bar',
},
},
},
'suite_d': {
'modifications': {
'Fake Tester': {
'foo': 'baz',
},
},
},
}
"""
EXCEPTIONS_UNSORTED = """\
{
'suite_d': {
'modifications': {
'Fake Tester': {
'foo': 'baz',
},
},
},
'suite_c': {
'modifications': {
'Fake Tester': {
'foo': 'bar',
},
},
},
}
"""
FOO_TEST_MODIFICATIONS = """\
{
'foo_test': {
'modifications': {
'Fake Tester': {
'args': [
'--bar',
],
'swarming': {
'hard_timeout': 600,
},
},
},
}
}
"""
NONEXISTENT_REMOVAL = """\
{
'foo_test': {
'remove_from': [
'Nonexistent Tester',
]
}
}
"""
NONEXISTENT_MODIFICATION = """\
{
'foo_test': {
'modifications': {
'Nonexistent Tester': {
'args': [],
},
},
}
}
"""
COMPOSITION_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "bar_test"
},
{
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--this-is-an-argument"
],
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "bar_test"
},
{
"args": [
"--this-is-an-argument"
],
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
VARIATION_GTEST_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"kvm": "1"
}
]
},
"test": "foo_test"
},
{
"args": [
"--variation"
],
"name": "variation_test",
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"kvm": "1"
}
]
},
"test": "foo_test"
}
]
}
}
"""
COMPOSITION_WATERFALL_FILTERED_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
MERGED_ARGS_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--c_arg",
"--bar"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"kvm": "1"
}
],
"hard_timeout": 600
},
"test": "foo_test"
}
]
}
}
"""
LINUX_ARGS_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--no-xvfb"
],
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
MERGED_ENABLE_FEATURES_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--enable-features=Foo,Bar,Baz"
],
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
MODIFIED_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--bar"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high",
"kvm": "1"
}
],
"expiration": 120,
"hard_timeout": 600
},
"test": "foo_test"
}
]
}
}
"""
ISOLATED_SCRIPT_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"isolated_scripts": [
{
"isolate_name": "foo_test",
"name": "foo_test",
"swarming": {
"can_use_on_swarming_builders": true
}
}
]
}
}
"""
SCRIPT_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"scripts": [
{
"name": "foo_test",
"script": "foo.py"
}
]
}
}
"""
SCRIPT_WITH_ARGS_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"scripts": [
{
"args": [
"--fake-arg"
],
"name": "foo_test",
"script": "foo.py"
}
]
}
}
"""
JUNIT_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"junit_tests": [
{
"test": "foo_test"
}
]
}
}
"""
CTS_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"cts_tests": [
{
"arch": "arm64",
"platform": "L"
}
]
}
}
"""
CTS_ISOLATED_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"isolated_scripts": [
{
"args": [
"--gs-results-bucket=chromium-result-details"
],
"isolate_name": "foo_cts_tests",
"merge": {
"args": [
"--bucket",
"chromium-result-details",
"--test-name",
"foo_cts_tests"
],
"script": \
"//build/android/pylib/results/presentation/test_results_presentation.py"
},
"name": "foo_cts_tests",
"swarming": {
"can_use_on_swarming_builders": true,
"cipd_packages": [
{
"cipd_package": "infra/tools/luci/logdog/butler/${platform}",
"location": "bin",
"revision": \
"git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
}
],
"output_links": [
{
"link": [
"https://luci-logdog.appspot.com/v/?s",
"=android%2Fswarming%2Flogcats%2F",
"${TASK_ID}%2F%2B%2Funified_logcats"
],
"name": "shard #${SHARD_INDEX} logcats"
}
]
}
}
]
}
}
"""
INSTRUMENTATION_TEST_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"instrumentation_tests": [
{
"test": "foo_test"
}
]
}
}
"""
INSTRUMENTATION_TEST_DIFFERENT_NAMES_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"instrumentation_tests": [
{
"name": "bar_tests",
"test": "foo_test"
},
{
"name": "foo_tests",
"test": "foo_test"
}
]
}
}
"""
GPU_TELEMETRY_TEST_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"isolated_scripts": [
{
"args": [
"foo",
"--show-stdout",
"--browser=release",
"--passthrough",
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_integration_test",
"name": "foo_tests",
"should_retry_with_patch": false,
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "10de:1cb3"
}
],
"idempotent": false
}
}
]
}
}
"""
ANDROID_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Android Builder": {
"additional_compile_targets": [
"bar_test"
]
},
"Fake Android K Tester": {
"additional_compile_targets": [
"bar_test"
],
"gtest_tests": [
{
"args": [
"--gs-results-bucket=chromium-result-details",
"--recover-devices"
],
"swarming": {
"can_use_on_swarming_builders": true,
"cipd_packages": [
{
"cipd_package": "infra/tools/luci/logdog/butler/${platform}",
"location": "bin",
"revision": \
"git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
}
],
"dimension_sets": [
{
"device_os": "KTU84P",
"device_type": "hammerhead",
"integrity": "high"
}
],
"expiration": 120,
"output_links": [
{
"link": [
"https://luci-logdog.appspot.com/v/?s",
"=android%2Fswarming%2Flogcats%2F",
"${TASK_ID}%2F%2B%2Funified_logcats"
],
"name": "shard #${SHARD_INDEX} logcats"
}
]
},
"test": "foo_test"
}
]
},
"Fake Android L Tester": {
"gtest_tests": [
{
"args": [
"--gs-results-bucket=chromium-result-details",
"--recover-devices"
],
"swarming": {
"can_use_on_swarming_builders": true,
"cipd_packages": [
{
"cipd_package": "infra/tools/luci/logdog/butler/${platform}",
"location": "bin",
"revision": \
"git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
}
],
"dimension_sets": [
{
"device_os": "KTU84P",
"device_type": "hammerhead",
"integrity": "high"
}
],
"expiration": 120
},
"test": "foo_test"
}
]
},
"Fake Android M Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": false
},
"test": "foo_test"
}
]
}
}
"""
MULTI_DIMENSION_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "none",
"integrity": "high",
"os": "1"
}
],
"expiration": 120
},
"test": "foo_test",
"trigger_script": {
"args": [
"--multiple-trigger-configs",
"[{\\"gpu\\": \\"none\\", \\"integrity\\": \\"high\\", \
\\"os\\": \\"1\\"}, \
{\\"gpu\\": \\"none\\", \\"os\\": \\"2\\"}]",
"--multiple-dimension-script-verbose",
"True"
],
"script": "//testing/trigger_scripts/trigger_multiple_dimensions.py"
}
}
]
}
}
"""
CHROMEOS_TRIGGER_SCRIPT_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"device_type": "foo_device",
"integrity": "high"
}
],
"expiration": 120
},
"test": "foo_test",
"trigger_script": {
"script": "//testing/trigger_scripts/chromeos_device_trigger.py"
}
}
]
}
}
"""
GPU_DIMENSIONS_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"isolated_scripts": [
{
"args": [
"foo_test",
"--show-stdout",
"--browser=release",
"--passthrough",
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_integration_test",
"name": "foo_test",
"should_retry_with_patch": false,
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"iama": "mixin",
"integrity": "high"
}
],
"expiration": 120,
"idempotent": false,
"value": "test"
}
}
]
}
}
"""
LUCI_MILO_CFG = """\
consoles {
builders {
name: "buildbucket/luci.chromium.ci/Fake Tester"
}
}
"""
LUCI_MILO_CFG_WATERFALL_SORTING = """\
consoles {
builders {
name: "buildbucket/luci.chromium.ci/Fake Tester"
name: "buildbucket/luci.chromium.ci/Really Fake Tester"
}
}
"""
TEST_SUITE_SORTING_WATERFALL = """
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
]
"""
TEST_SUITE_SORTED_WATERFALL = """
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
{
'name': 'chromium.zz.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
]
"""
TEST_SUITE_UNSORTED_WATERFALL_1 = """
[
{
'name': 'chromium.zz.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
]
"""
TEST_SUITE_UNSORTED_WATERFALL_2 = """
[
{
'name': 'chromium.test',
'machines': {
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
{
'name': 'chromium.zz.test',
'machines': {
'Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
'Really Fake Tester': {
'test_suites': {
'instrumentation_tests': 'suite_a',
'scripts': 'suite_b',
},
},
},
},
]
"""
# Note that the suites in basic_suites would be sorted after the suites in
# compound_suites. This is valid though, because each set of suites is sorted
# separately.
# suite_c is an 'instrumentation_tests' test
# suite_d is an 'scripts' test
TEST_SUITE_SORTED = """\
{
'basic_suites': {
'suite_c': {
'suite_c': {},
},
'suite_d': {
'script': {
'script': 'suite_d.py',
}
},
},
'compound_suites': {
'suite_a': [
'suite_c',
],
'suite_b': [
'suite_d',
],
},
}
"""
TEST_SUITE_UNSORTED_1 = """\
{
'basic_suites': {
'suite_d': {
'a': 'b',
},
'suite_c': {
'a': 'b',
},
},
'compound_suites': {
'suite_a': [
'suite_c',
],
'suite_b': [
'suite_d',
],
},
}
"""
TEST_SUITE_UNSORTED_2 = """\
{
'basic_suites': {
'suite_c': {
'a': 'b',
},
'suite_d': {
'a': 'b',
},
},
'compound_suites': {
'suite_b': [
'suite_c',
],
'suite_a': [
'suite_d',
],
},
}
"""
TEST_SUITE_UNSORTED_3 = """\
{
'basic_suites': {
'suite_d': {
'a': 'b',
},
'suite_c': {
'a': 'b',
},
},
'compound_suites': {
'suite_b': [
'suite_c',
],
'suite_a': [
'suite_d',
],
},
}
"""
TEST_SUITES_SYNTAX_ERROR = """\
{
'basic_suites': {
3: {
'suite_c': {},
},
},
'compound_suites': {},
}
"""
class UnitTest(unittest.TestCase):
def test_base_generator(self):
# Only needed for complete code coverage.
self.assertRaises(NotImplementedError,
generate_buildbot_json.BaseGenerator(None).generate,
None, None, None, None)
self.assertRaises(NotImplementedError,
generate_buildbot_json.BaseGenerator(None).sort,
None)
def test_good_test_suites_are_ok(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_good_multi_dimension_test_suites_are_ok(self):
fbb = FakeBBGen(FOO_GTESTS_MULTI_DIMENSION_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_good_composition_test_suites_are_ok(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_bad_composition_test_suites_are_caught(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
BAD_COMPOSITION_TEST_SUITES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'Composition test suites may not refer to.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_composition_test_suites_no_duplicate_names(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
DUPLICATES_COMPOSITION_TEST_SUITES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'.*may not duplicate basic test suite.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_unknown_test_suites_are_caught(self):
fbb = FakeBBGen(UNKNOWN_TEST_SUITE_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'Test suite baz_tests from machine.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_unknown_test_suite_types_are_caught(self):
fbb = FakeBBGen(UNKNOWN_TEST_SUITE_TYPE_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'Unknown test suite type foo_test_type.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_unrefed_test_suite_caught(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
UNREFED_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'.*unreferenced.*bar_tests.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_good_waterfall_output(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = COMPOSITION_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_reusing_gtest_targets(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
REUSING_TEST_WITH_DIFFERENT_NAME,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = VARIATION_GTEST_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_noop_exception_does_nothing(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
EMPTY_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = COMPOSITION_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_test_arg_merges(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE_WITH_ARGS,
FOO_TEST_MODIFICATIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = MERGED_ARGS_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_enable_features_arg_merges(self):
fbb = FakeBBGen(FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL,
FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = MERGED_ENABLE_FEATURES_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_linux_args(self):
fbb = FakeBBGen(FOO_LINUX_GTESTS_WATERFALL,
FOO_TEST_SUITE_WITH_LINUX_ARGS,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = LINUX_ARGS_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_test_filtering(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = COMPOSITION_WATERFALL_FILTERED_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_test_modifications(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE,
FOO_TEST_MODIFICATIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = MODIFIED_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_isolated_script_tests(self):
fbb = FakeBBGen(FOO_ISOLATED_SCRIPTS_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = ISOLATED_SCRIPT_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_script_with_args(self):
fbb = FakeBBGen(FOO_SCRIPT_WATERFALL,
SCRIPT_SUITE,
SCRIPT_WITH_ARGS_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = SCRIPT_WITH_ARGS_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_script(self):
fbb = FakeBBGen(FOO_SCRIPT_WATERFALL,
FOO_SCRIPT_SUITE,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = SCRIPT_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_junit_tests(self):
fbb = FakeBBGen(FOO_JUNIT_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = JUNIT_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_cts_tests(self):
fbb = FakeBBGen(FOO_CTS_WATERFALL,
FOO_CTS_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = CTS_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_isolated_cts_tests(self):
fbb = FakeBBGen(FOO_ISOLATED_CTS_WATERFALL,
FOO_ISOLATED_CTS_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = CTS_ISOLATED_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_instrumentation_tests(self):
fbb = FakeBBGen(FOO_INSTRUMENTATION_TEST_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = INSTRUMENTATION_TEST_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_gpu_telemetry_tests(self):
fbb = FakeBBGen(FOO_GPU_TELEMETRY_TEST_WATERFALL,
COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = GPU_TELEMETRY_TEST_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_instrumentation_tests_with_different_names(self):
fbb = FakeBBGen(FOO_INSTRUMENTATION_TEST_WATERFALL,
INSTRUMENTATION_TESTS_WITH_DIFFERENT_NAMES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = \
INSTRUMENTATION_TEST_DIFFERENT_NAMES_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_ungenerated_output_files_are_caught(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
NO_BAR_TEST_EXCEPTIONS,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = (
'\n' + COMPOSITION_WATERFALL_FILTERED_OUTPUT)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_output_file_consistency(verbose=True, dump=False)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, 'Waterfall chromium.test did not have the following'
' expected contents:.*')
self.assertRegexpMatches(joined_lines, '.*--- expected.*')
self.assertRegexpMatches(joined_lines, '.*\+\+\+ current.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_android_output_options(self):
fbb = FakeBBGen(ANDROID_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = ANDROID_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_nonexistent_removal_raises(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE,
NONEXISTENT_REMOVAL,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'The following nonexistent machines.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_nonexistent_modification_raises(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE,
NONEXISTENT_MODIFICATION,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'The following nonexistent machines.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_waterfall_args(self):
fbb = FakeBBGen(COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL,
GOOD_COMPOSITION_TEST_SUITES,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_multi_dimension_output(self):
fbb = FakeBBGen(FOO_GTESTS_MULTI_DIMENSION_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = MULTI_DIMENSION_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_chromeos_trigger_script_output(self):
fbb = FakeBBGen(FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = CHROMEOS_TRIGGER_SCRIPT_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_relative_pyl_file_dir(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
REUSING_TEST_WITH_DIFFERENT_NAME,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.args = argparse.Namespace(pyl_files_dir='relative/path/')
for file_name in list(fbb.files):
if not 'luci-milo.cfg' in file_name:
fbb.files[os.path.join('relative/path/', file_name)] = (
fbb.files.pop(file_name))
fbb.check_input_file_consistency(verbose=True)
fbb.files['relative/path/chromium.test.json'] = VARIATION_GTEST_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_nonexistent_bot_raises(self):
fbb = FakeBBGen(UNKNOWN_BOT_GTESTS_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_waterfalls_must_be_sorted(self):
fbb = FakeBBGen(TEST_SUITE_SORTED_WATERFALL,
TEST_SUITE_SORTED,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG_WATERFALL_SORTING)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_UNSORTED_WATERFALL_1,
TEST_SUITE_SORTED,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG_WATERFALL_SORTING)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'The following files have invalid keys: waterfalls.pyl'):
fbb.check_input_file_consistency(verbose=True)
joined_lines = '\n'.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+chromium\..*test.*')
self.assertRegexpMatches(
joined_lines, '.*\-chromium\..*test.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_UNSORTED_WATERFALL_2,
TEST_SUITE_SORTED,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG_WATERFALL_SORTING)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'The following files have invalid keys: waterfalls.pyl'):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+.*Fake Tester.*')
self.assertRegexpMatches(
joined_lines, '.*\-.*Fake Tester.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_test_suite_exceptions_must_be_sorted(self):
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
EXCEPTIONS_SORTED,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
EXCEPTIONS_UNSORTED,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+suite_.*')
self.assertRegexpMatches(
joined_lines, '.*\-suite_.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_test_suites_must_be_sorted(self):
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
TEST_SUITE_SORTED,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
for unsorted in (
TEST_SUITE_UNSORTED_1,
TEST_SUITE_UNSORTED_2,
TEST_SUITE_UNSORTED_3,
):
fbb = FakeBBGen(TEST_SUITE_SORTING_WATERFALL,
unsorted,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+suite_.*')
self.assertRegexpMatches(
joined_lines, '.*\-suite_.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
[
{
'mixins': ['waterfall_mixin'],
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['builder_mixin'],
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['random_mixin'],
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['dimension_mixin'],
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['dimension_mixin'],
'os_type': 'win',
'browser_config': 'release',
'test_suites': {
'gpu_telemetry_tests': 'foo_tests',
},
},
},
},
]
"""
# Swarming mixins must be a list, a single string is not allowed.
FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': 'dimension_mixin',
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['nonexistant'],
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
[
{
'mixins': ['waterfall_mixin'],
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
[
{
'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'swarming': {
'dimension_sets': [
{
'kvm': '1',
},
],
},
'test_suites': {
'gtest_tests': 'foo_tests',
},
},
},
},
]
"""
FOO_TEST_SUITE_WITH_MIXIN = """\
{
'basic_suites': {
'foo_tests': {
'foo_test': {
'swarming': {
'dimension_sets': [
{
'integrity': 'high',
}
],
'expiration': 120,
},
'mixins': ['test_mixin'],
},
},
},
}
"""
# These mixins are invalid; if passed to check_input_file_consistency, they will
# fail. These are used for output file consistency checks.
SWARMING_MIXINS = """\
{
'builder_mixin': {
'swarming': {
'value': 'builder',
},
},
'dimension_mixin': {
'swarming': {
'dimensions': {
'iama': 'mixin',
},
},
},
'random_mixin': {
'value': 'random',
},
'test_mixin': {
'swarming': {
'value': 'test',
},
},
'waterfall_mixin': {
'swarming': {
'value': 'waterfall',
},
},
}
"""
SWARMING_MIXINS_APPEND = """\
{
'builder_mixin': {
'$mixin_append': {
'args': [ '--mixin-argument' ],
},
},
}
"""
SWARMING_MIXINS_APPEND_NOT_LIST = """\
{
'builder_mixin': {
'$mixin_append': {
'args': 'I am not a list',
},
},
}
"""
SWARMING_MIXINS_APPEND_TO_SWARMING = """\
{
'builder_mixin': {
'$mixin_append': {
'swarming': [ 'swarming!' ],
},
},
}
"""
SWARMING_MIXINS_DUPLICATED = """\
{
'builder_mixin': {
'value': 'builder',
},
'builder_mixin': {
'value': 'builder',
},
}
"""
SWARMING_MIXINS_UNSORTED = """\
{
'b_mixin': {
'b': 'b',
},
'a_mixin': {
'a': 'a',
},
'c_mixin': {
'c': 'c',
},
}
"""
SWARMING_MIXINS_SORTED = """\
{
'a_mixin': {
'a': 'a',
},
'b_mixin': {
'b': 'b',
},
'c_mixin': {
'c': 'c',
},
}
"""
FOO_CTS_WATERFALL_MIXINS = """\
[
{
'name': 'chromium.test',
'machines': {
'Fake Tester': {
'mixins': ['test_mixin'],
'test_suites': {
'cts_tests': 'foo_cts_tests',
},
},
},
},
]
"""
WATERFALL_MIXIN_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high"
}
],
"expiration": 120,
"value": "waterfall"
},
"test": "foo_test"
}
]
}
}
"""
WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high"
}
],
"expiration": 120,
"value": "exception"
},
"test": "foo_test"
}
]
}
}
"""
BUILDER_MIXIN_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high"
}
],
"expiration": 120,
"value": "builder"
},
"test": "foo_test"
}
]
}
}
"""
BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high"
}
],
"expiration": 120
},
"test": "foo_test",
"value": "random"
}
]
}
}
"""
BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"args": [
"--c_arg",
"--mixin-argument"
],
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "foo_test"
}
]
}
}
"""
TEST_MIXIN_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"integrity": "high",
"kvm": "1"
}
],
"expiration": 120,
"value": "test"
},
"test": "foo_test"
}
]
}
}
"""
DIMENSIONS_MIXIN_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Fake Tester": {
"gtest_tests": [
{
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"iama": "mixin",
"integrity": "high"
}
],
"expiration": 120,
"value": "test"
},
"test": "foo_test"
}
]
}
}
"""
class MixinTests(unittest.TestCase):
"""Tests for the mixins feature."""
def test_mixins_must_be_sorted(self):
fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS_SORTED,
LUCI_MILO_CFG)
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS_UNSORTED,
LUCI_MILO_CFG)
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, '.*\+._mixin.*')
self.assertRegexpMatches(
joined_lines, '.*\-._mixin.*')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_waterfall(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_waterfall_exception_overrides(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
FOO_TEST_SUITE,
SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_builder(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = BUILDER_MIXIN_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_builder_non_swarming(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = (
BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_test_suite(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = TEST_MIXIN_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_dimension(self):
fbb = FakeBBGen(FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_dimension_gpu(self):
fbb = FakeBBGen(FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = GPU_DIMENSIONS_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_unreferenced(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
'.*mixins are unreferenced.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_cts(self):
fbb = FakeBBGen(FOO_CTS_WATERFALL_MIXINS,
FOO_CTS_SUITE ,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = CTS_OUTPUT
fbb.check_input_file_consistency(verbose=True)
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_unused(self):
fbb = FakeBBGen(FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_list(self):
fbb = FakeBBGen(FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
FOO_TEST_SUITE_WITH_MIXIN,
EMPTY_PYL_FILE,
SWARMING_MIXINS,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
with self.assertRaises(generate_buildbot_json.BBGenErr):
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_no_duplicate_keys(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS_DUPLICATED,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'The following files have invalid keys: mixins.pyl'):
fbb.check_input_file_consistency(verbose=True)
joined_lines = ' '.join(fbb.printed_lines)
self.assertRegexpMatches(
joined_lines, 'Key .* is duplicated')
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
def test_type_assert_printing_help(self):
fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
TEST_SUITES_SYNTAX_ERROR,
EMPTY_PYL_FILE,
EMPTY_PYL_FILE,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'Invalid \.pyl file \'test_suites.pyl\'.*'):
fbb.check_input_file_consistency(verbose=True)
self.assertEquals(
fbb.printed_lines, [
'== test_suites.pyl ==',
'<snip>',
'1 {',
"2 'basic_suites': {",
'--------------------------------------------------------------------'
'------------',
'3 3: {',
'-------^------------------------------------------------------------'
'------------',
"4 'suite_c': {},",
'5 },',
'<snip>',
])
def test_mixin_append_args(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE_WITH_ARGS,
EMPTY_PYL_FILE,
SWARMING_MIXINS_APPEND,
LUCI_MILO_CFG)
fbb.files['chromium.test.json'] = BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_mixin_append_mixin_field_not_list(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE_WITH_ARGS,
EMPTY_PYL_FILE,
SWARMING_MIXINS_APPEND_NOT_LIST,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'Key "args" in \$mixin_append must be a list.'):
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
def test_mixin_append_test_field_not_list(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
SWARMING_MIXINS_APPEND_TO_SWARMING,
LUCI_MILO_CFG)
with self.assertRaisesRegexp(
generate_buildbot_json.BBGenErr,
'Cannot apply \$mixin_append to non-list "swarming".'):
fbb.check_output_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
if __name__ == '__main__':
unittest.main()