blob: 5cde1afa236d78b655e88b58acf1c21ae108f24c [file] [log] [blame]
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from import FAILURE, SUCCESS
from PB.recipe_engine.result import RawResult
DEPS = [
# Mapping from a builder name to a list of GOOS-GOARCH variants it should build
# CIPD packages for. 'native' means "do not cross-compile, build for the host
# platform". Targeting 'native' will also usually build non-go based packages.
# Additionally, a variant may have a sequence of options appended to it,
# separated by colons. e.g. 'VARIANT:option:option'. Currently the supported
# options are:
# * 'test' - Run the tests. By default no tests are run.
# * 'legacy' - Switch the builder that builds this variant to use go "legacy"
# Go version 1.15.* instead of the "bleeding_edge" version. This is
# primarily needed by builders targeting OSX amd64 that need to produce
# binaries that can run on OSX 10.10 and OSX 10.11. What exact versions
# correspond to "legacy" and "bleeding_edge" is defined in in
# infra.git. Note that this option applies to the entire builder (not only
# the individual build variant).
# If the builder is not in this set, or the list of GOOS-GOARCH for it is empty,
# it won't be used for building CIPD packages.
# Only builders named '*-packager-*' builders will actually upload CIPD
# packages, while '*-continuous-*' builders merely verify that CIPD packages can
# be built.
# TODO(iannucci): make packager role explicit with `package=cipd_prefix` option.
# TODO(iannucci): remove this dict and put this all configuration as explicit
# property inputs to the recipe :)
# trusty-64 is the primary builder for linux-amd64, and the rest just
# cross-compile to different platforms (to speed up the overall cycle time
# by doing stuff in parallel).
'infra-continuous-trusty-64': [
'infra-continuous-xenial-64': [
'infra-continuous-bionic-64': [
# 10.13 is the primary builder for darwin-amd64.
'infra-continuous-mac-10.13-64': ['native:test:legacy',],
'infra-continuous-mac-10.14-64': [],
'infra-continuous-mac-10.15-64': ['darwin-arm64',],
# Windows 64 bit builder runs and tests for both 64 && 32 bit.
'infra-continuous-win10-64': [
'windows-arm64', # cross compiled, so don't run tests.
'infra-continuous-win11-64': [
# Internal builders, they use exact same recipe.
'infra-internal-continuous-trusty-64': [
'infra-internal-continuous-xenial-64': [
'darwin-arm64', # note: can't do it on mac-10.15 since we need >=go1.16
'infra-internal-continuous-win-64': [
'windows-arm64', # cross compiled, so don't run tests.
'infra-internal-continuous-mac-11-64': ['native:test:legacy',],
# Builders that upload CIPD packages.
# In comments is approximate runtime for building and testing packages, per
# platform (as of Feb 23 2021). We try to balance xc1 and xc2.
'infra-packager-linux-64': [
'native', # ~120 sec
'infra-packager-linux-xc1': [
'linux-386', # ~60 sec
'linux-arm', # ~60 sec
'linux-arm64', # ~60 sec
'infra-packager-linux-xc2': [
'aix-ppc64', # ~5 sec
'linux-mips64', # ~40 sec
'linux-mips64le', # ~40 sec
'linux-mipsle', # ~40 sec
'linux-ppc64', # ~40 sec
'linux-ppc64le', # ~40 sec
'linux-riscv64', # ~5 sec
'linux-s390x', # ~40 sec
'infra-packager-mac-64': [
# NOTE: Mac packages need to be codesigned and so can ONLY be packaged
# on Mac.
'native:legacy', # ~150 sec
'darwin-arm64', # ~60 sec
'infra-packager-win-64': [
'native', # ~60 sec
'windows-386', # ~100 sec
'windows-arm64', # not timed
'infra-internal-packager-linux-64': [
'native', # ~60 sec
'linux-arm', # ~30 sec
'linux-arm64', # ~30 sec
'infra-internal-packager-mac-64': [
# NOTE: Mac packages need to be codesigned and so can ONLY be packaged
# on Mac.
'native:legacy', # ~40 sec
'darwin-arm64', # ~30 sec
'infra-internal-packager-win-64': [
'native', # ~60 sec
'windows-386', # ~40 sec
'windows-arm64', # not timed
def RunSteps(api):
buildername = api.buildbucket.builder_name
if (buildername.startswith('infra-internal-continuous') or
project_name = 'infra_internal'
repo_url = INTERNAL_REPO
elif (buildername.startswith('infra-continuous') or
project_name = 'infra'
repo_url = PUBLIC_REPO
else: # pragma: no cover
raise ValueError(
'This recipe is not intended for builder %s. ' % buildername)
# Use the latest bleeding edge version of Go unless asked for the legacy one.
go_version_variant = 'bleeding_edge'
for variant in CIPD_PACKAGE_BUILDERS.get(buildername, []):
if 'legacy' in variant.split(':'):
go_version_variant = 'legacy'
co = api.infra_checkout.checkout(
internal=(project_name == 'infra_internal'),
# Whatever is checked out by bot_update. It is usually equal to
# except when the build was triggered
# manually (commit id is empty in that case).
rev =['got_revision']
return build_main(api, co, buildername, project_name, repo_url, rev)
def build_main(api, checkout, buildername, project_name, repo_url, rev):
is_packager = 'packager' in buildername
# Do not run python tests on packager builders, since most of them are
# irrelevant to the produced packages. Relevant portion of tests will be run
# from api.infra_cipd.test() below, when testing packages that pack python
# code.
if api.platform.arch != 'arm' and not is_packager:
run_python_tests(api, project_name)
# Some third_party go packages on OSX rely on cgo and thus a configured
# clang toolchain.
with api.osx_sdk('mac'), checkout.go_env():
if not is_packager:
with api.depot_tools.on_path():
# Some go tests test interactions with depot_tools binaries, so put
# depot_tools on the path.
'infra go tests',
['vpython', '-u', api.path['checkout'].join('go', '')]))
fails = []
for plat in CIPD_PACKAGE_BUILDERS.get(buildername, []):
options = plat.split(':')
plat = options.pop(0)
# Mac packages can only be produced by a mac builder.
if 'darwin-' in plat and is_packager and not api.platform.is_mac:
"Invalid package configuration",
"Mac packages require codesign and can not run on %s."
% (,)
if plat == 'native':
goos, goarch = None, None
goos, goarch = plat.split('-', 1)
with api.infra_cipd.context(api.path['checkout'], goos, goarch):
if api.platform.is_mac:
if 'test' in options:
if is_packager:
if api.runtime.is_experimental:
api.step('no CIPD package upload in experimental mode', cmd=None)
api.infra_cipd.upload(api.infra_cipd.tags(repo_url, rev))
except api.step.StepFailure:
status = SUCCESS
summary = []
if fails:
status = FAILURE
summary.append('Failed to build some CIPD packages for platform(s):')
summary.extend(' * %s' % f for f in fails)
return RawResult(status=status, summary_markdown='\n'.join(summary))
def run_python_tests(api, project_name):
with api.step.defer_results():
with api.context(cwd=api.path['checkout']):
# Run Linux tests everywhere, Windows tests only on public CI.
if api.platform.is_linux or project_name == 'infra':
api.step('infra python tests',
['python', '', 'test'])
# Validate ccompute configs.
if api.platform.is_linux and project_name == 'infra_internal':
ccompute_config = api.path['checkout'].join(
'ccompute', 'scripts', '')
'ccompute config test', ['python', ccompute_config, 'test'])
def GenTests(api):
# This allows us to test potential misconfigurations. As long as this runs
# before the yield statement for the test that needs it, it will be added to
# the module global by the time the test runs.
# We put it here so that ALL tests have a consistent view of
CIPD_PACKAGE_BUILDERS['infra-packager-TEST-linux-xcMac'] = [
def test(name, builder, repo, project, bucket, plat, is_experimental=False,
return (api.test(name) + api.platform(plat, 64, arch) +
api.runtime(is_experimental=is_experimental) +
project, bucket, builder, git_repo=repo, build_number=123))
yield test('public-ci-linux', 'infra-continuous-trusty-64',
PUBLIC_REPO, 'infra', 'ci', 'linux')
yield test('public-ci-linux-arm64', 'infra-continuous-trusty-64',
PUBLIC_REPO, 'infra', 'ci', 'linux', arch='arm')
yield test('public-ci-win', 'infra-continuous-win10-64',
PUBLIC_REPO, 'infra', 'ci', 'win')
yield test('internal-ci-linux', 'infra-internal-continuous-trusty-64',
INTERNAL_REPO, 'infra-internal', 'ci', 'linux')
yield test('internal-ci-mac', 'infra-internal-continuous-mac-11-64',
INTERNAL_REPO, 'infra-internal', 'ci', 'mac')
yield test('public-packager-mac', 'infra-packager-mac-64',
PUBLIC_REPO, 'infra', 'prod', 'mac')
yield test('public-packager-mac_experimental', 'infra-packager-mac-64',
PUBLIC_REPO, 'infra', 'prod', 'mac',
yield test('public-packager-mac_codesign', 'infra-packager-mac-64',
PUBLIC_REPO, 'infra', 'prod', 'mac') +
yield test('public-packager-win', 'infra-packager-win-64', PUBLIC_REPO,
'infra', 'prod', 'win')
yield test('internal-packager-linux', 'infra-internal-packager-linux-64',
INTERNAL_REPO, 'infra-internal', 'prod', 'linux')
yield (
test('packager-cipd-fail', 'infra-packager-linux-xc1',
PUBLIC_REPO, 'infra', 'prod', 'linux') +
api.step_data('[GOOS:linux GOARCH:arm]cipd - build packages', retcode=1)
yield test('cross-packager-mac-fail', 'infra-packager-TEST-linux-xcMac',
PUBLIC_REPO, 'infra', 'prod', 'linux')