blob: a82ea6f43f80ebe7627bffffdac0391eedd6547f [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Main builder code for Chromium OS.
Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
from __future__ import print_function
import distutils.version # pylint: disable=import-error,no-name-in-module
import glob
import json
import mock
import optparse # pylint: disable=deprecated-module
import os
import pickle
import sys
from chromite.cbuildbot import builders
from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot import remote_try
from chromite.cbuildbot import repository
from chromite.cbuildbot import tee
from chromite.cbuildbot import topology
from chromite.cbuildbot import trybot_patch_pool
from chromite.cbuildbot.stages import completion_stages
from chromite.lib.const import waterfall
from chromite.lib import builder_status_lib
from chromite.lib import cidb
from chromite.lib import cgroups
from chromite.lib import cleanup
from chromite.lib import commandline
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import failures_lib
from chromite.lib import git
from chromite.lib import gob_util
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import retry_stats
from chromite.lib import sudo
from chromite.lib import timeout_util
from chromite.lib import tree_status
from chromite.lib import ts_mon_config
_DEFAULT_LOG_DIR = 'cbuildbot_logs'
_BUILDBOT_LOG_FILE = 'cbuildbot.log'
_DEFAULT_INT_BUILDROOT = 'trybot-internal'
_API_VERSION_ATTR = 'api_version'
def _PrintValidConfigs(site_config, display_all=False):
"""Print a list of valid buildbot configs.
site_config: config_lib.SiteConfig containing all config info.
display_all: Print all configs. Otherwise, prints only configs with
def _GetSortKey(config_name):
config_dict = site_config[config_name]
return (not config_dict['trybot_list'], config_dict['description'],
if not display_all:
print('Note: This is the common list; for all configs, use --all.')
print('config'.ljust(COLUMN_WIDTH), 'description')
print('------'.ljust(COLUMN_WIDTH), '-----------')
config_names = site_config.keys()
for name in config_names:
if display_all or site_config[name]['trybot_list']:
desc = site_config[name].get('description')
desc = desc if desc else ''
print(name.ljust(COLUMN_WIDTH), desc)
def _ConfirmBuildRoot(buildroot):
"""Confirm with user the inferred buildroot, and mark it as confirmed."""
logging.warning('Using default directory %s as buildroot', buildroot)
if not cros_build_lib.BooleanPrompt(default=False):
print('Please specify a different buildroot via the --buildroot option.')
if not os.path.exists(buildroot):
def _ConfirmRemoteBuildbotRun():
"""Confirm user wants to run with --buildbot --remote."""
'You are about to launch a PRODUCTION job! This is *NOT* a '
'trybot run! Are you sure?')
if not cros_build_lib.BooleanPrompt(default=False):
print('Please specify --pass-through="--debug".')
def _DetermineDefaultBuildRoot(sourceroot, internal_build):
"""Default buildroot to be under the directory that contains current checkout.
internal_build: Whether the build is an internal build
sourceroot: Use specified sourceroot.
if not repository.IsARepoRoot(sourceroot):
'Could not find root of local checkout at %s. Please specify '
'using the --sourceroot option.' % sourceroot)
# Place trybot buildroot under the directory containing current checkout.
top_level = os.path.dirname(os.path.realpath(sourceroot))
if internal_build:
buildroot = os.path.join(top_level, _DEFAULT_INT_BUILDROOT)
buildroot = os.path.join(top_level, _DEFAULT_EXT_BUILDROOT)
return buildroot
def _BackupPreviousLog(log_file, backup_limit=25):
"""Rename previous log.
log_file: The absolute path to the previous log.
backup_limit: Maximum number of old logs to keep.
if os.path.exists(log_file):
old_logs = sorted(glob.glob(log_file + '.*'),
if len(old_logs) >= backup_limit:
last = 0
if old_logs:
last = int(old_logs.pop().rpartition('.')[2])
os.rename(log_file, log_file + '.' + str(last + 1))
def _IsDistributedBuilder(options, chrome_rev, build_config):
"""Determines whether the builder should be a DistributedBuilder.
options: options passed on the commandline.
chrome_rev: Chrome revision to build.
build_config: Builder configuration dictionary.
True if the builder should be a distributed_builder
if build_config['pre_cq']:
return True
elif not options.buildbot:
return False
elif chrome_rev in (constants.CHROME_REV_TOT,
# We don't do distributed logic to TOT Chrome PFQ's, nor local
# chrome roots (e.g. chrome try bots)
# TODO(davidjames): Update any builders that rely on this logic to use
# manifest_version=False instead.
return False
elif build_config['manifest_version']:
return True
return False
def _RunBuildStagesWrapper(options, site_config, build_config):
"""Helper function that wraps RunBuildStages()."""'cbuildbot was executed with args %s' %
chrome_rev = build_config['chrome_rev']
if options.chrome_rev:
chrome_rev = options.chrome_rev
if chrome_rev == constants.CHROME_REV_TOT:
options.chrome_version = gob_util.GetTipOfTrunkRevision(
options.chrome_rev = constants.CHROME_REV_SPEC
# If it's likely we'll need to build Chrome, fetch the source.
if build_config['sync_chrome'] is None:
options.managed_chrome = (
chrome_rev != constants.CHROME_REV_LOCAL and
(not build_config['usepkg_build_packages'] or chrome_rev or
build_config['profile'] or options.rietveld_patches))
options.managed_chrome = build_config['sync_chrome']
if options.managed_chrome:
# Tell Chrome to fetch the source locally.
internal = constants.USE_CHROME_INTERNAL in build_config['useflags']
chrome_src = 'chrome-src-internal' if internal else 'chrome-src'
target_name = 'target'
if options.branch:
# Tie the cache per branch
target_name = 'target-%s' % options.branch
options.chrome_root = os.path.join(options.cache_dir, 'distfiles',
target_name, chrome_src)
# Create directory if in need
elif options.rietveld_patches:
cros_build_lib.Die('This builder does not support Rietveld patches.')
metadata_dump_dict = {}
if options.metadata_dump:
with open(options.metadata_dump, 'r') as metadata_file:
metadata_dump_dict = json.loads(
# We are done munging options values, so freeze options object now to avoid
# further abuse of it.
# TODO(mtennant): one by one identify each options value override and see if
# it can be handled another way. Try to push this freeze closer and closer
# to the start of the script (e.g. in or after _PostParseCheck).
with parallel.Manager() as manager:
builder_run = cbuildbot_run.BuilderRun(
options, site_config, build_config, manager)
if metadata_dump_dict:
if builder_run.config.builder_class_name is None:
# TODO: This should get relocated to chromeos_config.
if _IsDistributedBuilder(options, chrome_rev, build_config):
builder_cls_name = 'simple_builders.DistributedBuilder'
builder_cls_name = 'simple_builders.SimpleBuilder'
builder_cls = builders.GetBuilderClass(builder_cls_name)
builder = builder_cls(builder_run)
builder = builders.Builder(builder_run)
if not builder.Run():
# Parser related functions
def _CheckLocalPatches(sourceroot, local_patches):
"""Do an early quick check of the passed-in patches.
If the branch of a project is not specified we append the current branch the
project is on.
TODO(davidjames): The project:branch format isn't unique, so this means that
we can't differentiate what directory the user intended to apply patches to.
We should references by directory instead.
sourceroot: The checkout where patches are coming from.
local_patches: List of patches to check in project:branch format.
A list of patches that have been verified, in project:branch format.
verified_patches = []
manifest = git.ManifestCheckout.Cached(sourceroot)
for patch in local_patches:
project, _, branch = patch.partition(':')
checkouts = manifest.FindCheckouts(project)
if not checkouts:
cros_build_lib.Die('Project %s does not exist.' % (project,))
if len(checkouts) > 1:
'We do not yet support local patching for projects that are checked '
'out to multiple directories. Try uploading your patch to gerrit '
'and referencing it via the -g option instead.'
ok = False
for checkout in checkouts:
project_dir = checkout.GetPath(absolute=True)
# If no branch was specified, we use the project's current branch.
if not branch:
local_branch = git.GetCurrentBranch(project_dir)
local_branch = branch
if local_branch and git.DoesCommitExistInRepo(project_dir, local_branch):
verified_patches.append('%s:%s' % (project, local_branch))
ok = True
if not ok:
if branch:
cros_build_lib.Die('Project %s does not have branch %s'
% (project, branch))
cros_build_lib.Die('Project %s is not on a branch!' % (project,))
return verified_patches
def _CheckChromeVersionOption(_option, _opt_str, value, parser):
"""Upgrade other options based on chrome_version being passed."""
value = value.strip()
if parser.values.chrome_rev is None and value:
parser.values.chrome_rev = constants.CHROME_REV_SPEC
parser.values.chrome_version = value
def _CheckChromeRootOption(_option, _opt_str, value, parser):
"""Validate and convert chrome_root to full-path form."""
if parser.values.chrome_rev is None:
parser.values.chrome_rev = constants.CHROME_REV_LOCAL
parser.values.chrome_root = value
def FindCacheDir(_parser, _options):
return None
class CustomGroup(optparse.OptionGroup):
"""Custom option group which supports arguments passed-through to trybot."""
def add_remote_option(self, *args, **kwargs):
"""For arguments that are passed-through to remote trybot."""
return optparse.OptionGroup.add_option(self, *args,
class CustomOption(commandline.FilteringOption):
"""Subclass FilteringOption class to implement pass-through and api."""
ACTIONS = commandline.FilteringOption.ACTIONS + ('extend',)
STORE_ACTIONS = commandline.FilteringOption.STORE_ACTIONS + ('extend',)
TYPED_ACTIONS = commandline.FilteringOption.TYPED_ACTIONS + ('extend',)
def __init__(self, *args, **kwargs):
# The remote_pass_through argument specifies whether we should directly
# pass the argument (with its value) onto the remote trybot.
self.pass_through = kwargs.pop('remote_pass_through', False)
self.api_version = int(kwargs.pop('api', '0'))
commandline.FilteringOption.__init__(self, *args, **kwargs)
def take_action(self, action, dest, opt, value, values, parser):
if action == 'extend':
# If there is extra spaces between each argument, we get '' which later
# code barfs on, so skip those. e.g. We see this with the forms:
# cbuildbot -p 'proj:branch ' ...
# cbuildbot -p ' proj:branch' ...
# cbuildbot -p 'proj:branch proj2:branch' ...
lvalue = value.split()
values.ensure_value(dest, []).extend(lvalue)
self, action, dest, opt, value, values, parser)
class CustomParser(commandline.FilteringParser):
"""Custom option parser which supports arguments passed-trhough to trybot"""
def add_remote_option(self, *args, **kwargs):
"""For arguments that are passed-through to remote trybot."""
return self.add_option(*args, remote_pass_through=True, **kwargs)
def CreateParser():
"""Expose _CreateParser publicly."""
# Name _CreateParser is needed for commandline library.
return _CreateParser()
def _CreateParser():
"""Generate and return the parser with all the options."""
# Parse options
usage = 'usage: %prog [options] buildbot_config [buildbot_config ...]'
parser = CustomParser(usage=usage, caching=FindCacheDir)
# Main options
parser.add_option('-l', '--list', action='store_true', dest='list',
help='List the suggested trybot configs to use (see --all)')
parser.add_option('-a', '--all', action='store_true', dest='print_all',
help='List all of the buildbot configs available w/--list')
parser.add_option('--local', action='store_true', default=False,
help='Specifies that this tryjob should be run locally. '
'Implies --debug.')
parser.add_option('--remote', action='store_true', default=False,
help='Specifies that this tryjob should be run remotely.')
parser.add_remote_option('-b', '--branch',
help='The manifest branch to test. The branch to '
'check the buildroot out to.')
parser.add_option('-r', '--buildroot', type='path', dest='buildroot',
help='Root directory where source is checked out to, and '
'where the build occurs. For external build configs, '
"defaults to 'trybot' directory at top level of your "
'repo-managed checkout.')
parser.add_option('--bootstrap-dir', type='path',
help='Bootstrapping cbuildbot may involve checking out '
'multiple copies of chromite. All these checkouts '
'will be contained in the directory specified here. '
'Default:%s' % osutils.GetGlobalTempDir())
parser.add_remote_option('--android_rev', type='choice',
help=('Revision of Android to use, of type [%s]'
% '|'.join(constants.VALID_ANDROID_REVISIONS)))
parser.add_remote_option('--chrome_rev', type='choice',
help=('Revision of Chrome to use, of type [%s]'
% '|'.join(constants.VALID_CHROME_REVISIONS)))
help='Name of profile to sub-specify board variant.')
parser.add_option('-c', '--config_repo',
help='Deprecated option. Do not use!')
# TODO( Running GOMA is under development. Following
# flags are added for development purpose due to repository dependency,
# but not officially supported yet.
parser.add_option('--goma_dir', type='path',
help='Specify a directory containing goma. When this is '
'set, GOMA is used to build Chrome.')
parser.add_option('--goma_client_json', type='path',
help='Specify a service-account-goma-client.json path. '
'The file is needed on bots to run GOMA.')
# Patch selection options.
group = CustomGroup(
'Patch Options')
group.add_remote_option('-g', '--gerrit-patches', action='extend',
type='string', default=[],
metavar="'Id1 *int_Id2...IdN'",
help='Space-separated list of short-form Gerrit '
"Change-Id's or change numbers to patch. "
"Please prepend '*' to internal Change-Id's")
group.add_remote_option('-G', '--rietveld-patches', action='extend',
type='string', default=[],
help='Space-separated list of short-form Rietveld '
'issue numbers to patch. If no subdir is '
'specified, the src directory is used.')
group.add_option('-p', '--local-patches', action='extend', default=[],
help='Space-separated list of project branches with '
'patches to apply. Projects are specified by name. '
'If no branch is specified the current branch of the '
'project will be used.')
# Remote trybot options.
group = CustomGroup(
'Remote Trybot Options (--remote)')
# TODO(dgarrett): Remove after a reasonable delay.
group.add_option('--use-buildbucket', action='store_true',
help='Deprecated option. Ignored.')
group.add_option('--do-not-use-buildbucket', action='store_false',
dest='use_buildbucket', default=True,
help='Use buildbucket instead of git to request'
'the tryjob(s).')
group.add_remote_option('--hwtest', action='store_true', default=False,
help='Run the HWTest stage (tests on real hardware)')
help='Attach an optional description to a --remote run '
'to make it easier to identify the results when it '
group.add_option('--slaves', action='extend', default=[],
help='Specify specific remote tryslaves to run on (e.g. '
'build149-m2); if the bot is busy, it will be queued')
group.add_remote_option('--channel', action='extend', dest='channels',
help='Specify a channel for a payloads trybot. Can '
'be specified multiple times. No valid for '
'non-payloads configs.')
group.add_option('--test-tryjob', action='store_true', default=False,
help='Submit a tryjob to the test repository. Will not '
'show up on the production trybot waterfall.')
group.add_option('--committer-email', type='string',
help='Override default git committer email.')
# Branch creation options.
group = CustomGroup(
'Branch Creation Options (used with branch-util)')
help='The branch to create or delete.')
group.add_remote_option('--delete-branch', action='store_true', default=False,
help='Delete the branch specified in --branch-name.')
help='Rename a branch to the specified name.')
group.add_remote_option('--force-create', action='store_true', default=False,
help='Overwrites an existing branch.')
group.add_remote_option('--skip-remote-push', action='store_true',
help='Do not actually push to remote git repos. '
'Used for end-to-end testing branching.')
# Advanced options.
group = CustomGroup(
'Advanced Options',
'Caution: use these options at your own risk.')
group.add_remote_option('--bootstrap-args', action='append', default=[],
help='Args passed directly to the bootstrap re-exec '
'to skip verification by the bootstrap code')
group.add_remote_option('--buildbot', action='store_true', dest='buildbot',
help='This is running on a buildbot. '
'This can be used to make a build operate '
'like an official builder, e.g. generate '
'new version numbers and archive official '
'artifacts and such. This should only be '
'used if you are confident in what you are '
'doing, as it will make automated commits.')
parser.add_remote_option('--repo-cache', type='path', dest='_repo_cache',
help='Present for backwards compatibility, ignored.')
group.add_remote_option('--no-buildbot-tags', action='store_false',
dest='enable_buildbot_tags', default=True,
help='Suppress buildbot specific tags from log '
'output. This is used to hide recursive '
'cbuilbot runs on the waterfall.')
group.add_remote_option('--buildnumber', type='int', default=0,
help='build number')
group.add_option('--chrome_root', action='callback', type='path',
help='Local checkout of Chrome to use.')
group.add_remote_option('--chrome_version', action='callback', type='string',
help='Used with SPEC logic to force a particular '
'git revision of chrome rather than the '
group.add_remote_option('--clobber', action='store_true', default=False,
help='Clears an old checkout before syncing')
group.add_remote_option('--latest-toolchain', action='store_true',
help='Use the latest toolchain.')
parser.add_option('--log_dir', dest='log_dir', type='path',
help='Directory where logs are stored.')
group.add_remote_option('--maxarchives', type='int',
dest='max_archive_builds', default=3,
help='Change the local saved build count limit.')
help='Overrides the default manifest repo url.')
group.add_remote_option('--compilecheck', action='store_true', default=False,
help='Only verify compilation and unit tests.')
group.add_remote_option('--noarchive', action='store_false', dest='archive',
default=True, help="Don't run archive stage.")
group.add_remote_option('--nobootstrap', action='store_false',
dest='bootstrap', default=True,
help="Don't checkout and run from a standalone "
'chromite repo.')
group.add_remote_option('--nobuild', action='store_false', dest='build',
help="Don't actually build (for cbuildbot dev)")
group.add_remote_option('--noclean', action='store_false', dest='clean',
default=True, help="Don't clean the buildroot")
group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
help='Disable cbuildbots usage of cgroups.')
group.add_remote_option('--nochromesdk', action='store_false',
dest='chrome_sdk', default=True,
help="Don't run the ChromeSDK stage which builds "
'Chrome outside of the chroot.')
group.add_remote_option('--noprebuilts', action='store_false',
dest='prebuilts', default=True,
help="Don't upload prebuilts.")
group.add_remote_option('--nopatch', action='store_false',
dest='postsync_patch', default=True,
help="Don't run PatchChanges stage. This does not "
'disable patching in of chromite patches '
'during BootstrapStage.')
group.add_remote_option('--nopaygen', action='store_false',
dest='paygen', default=True,
help="Don't generate payloads.")
group.add_remote_option('--noreexec', action='store_false',
dest='postsync_reexec', default=True,
help="Don't reexec into the buildroot after syncing.")
group.add_remote_option('--nosdk', action='store_true', default=False,
help='Re-create the SDK from scratch.')
group.add_remote_option('--nosync', action='store_false', dest='sync',
default=True, help="Don't sync before building.")
group.add_remote_option('--notests', action='store_false', dest='tests',
help='Override values from buildconfig, run no '
'tests, and build no autotest and artifacts.')
group.add_remote_option('--novmtests', action='store_false', dest='vmtests',
help='Override values from buildconfig, run no '
group.add_remote_option('--noimagetests', action='store_false',
dest='image_test', default=True,
help='Override values from buildconfig and run no '
'image tests.')
group.add_remote_option('--nouprev', action='store_false', dest='uprev',
help='Override values from buildconfig and never '
help='Reuse git data stored in an existing repo '
'checkout. This can drastically reduce the network '
'time spent setting up the trybot checkout. By '
"default, if this option isn't given but cbuildbot "
'is invoked from a repo checkout, cbuildbot will '
'use the repo root.')
group.add_option('--resume', action='store_true', default=False,
help='Skip stages already successfully completed.')
group.add_remote_option('--timeout', type='int', default=0,
help='Specify the maximum amount of time this job '
'can run for, at which point the build will be '
'aborted. If set to zero, then there is no '
group.add_remote_option('--version', dest='force_version',
help='Used with manifest logic. Forces use of this '
'version rather than create or get latest. '
'Examples: 4815.0.0-rc1, 4815.1.2')
group.add_remote_option('--git-cache-dir', type='path',
help='Specify the cache directory to store the '
'project caches populated by the git-cache '
'tool. Bootstrap the projects based on the git '
'cache files instead of fetching them directly '
'from the GoB servers.')
group.add_remote_option('--sanity-check-build', action='store_true',
default=False, dest='sanity_check_build',
help='Run the build as a sanity check build.')
# Internal options.
group = CustomGroup(
'Internal Chromium OS Build Team Options',
'Caution: these are for meant for the Chromium OS build team only')
group.add_remote_option('--archive-base', type='gs_path',
help='Base GS URL (gs://<bucket_name>/<path>) to '
'upload archive artifacts to')
'--cq-gerrit-query', dest='cq_gerrit_override',
help='If given, this gerrit query will be used to find what patches to '
"test, rather than the normal 'CommitQueue>=1 AND Verified=1 AND "
"CodeReview=2' query it defaults to. Use with care- note "
'additionally this setting only has an effect if the buildbot '
"target is a cq target, and we're in buildbot mode.")
group.add_option('--pass-through', action='append', type='string',
dest='pass_through_args', default=[])
group.add_option('--reexec-api-version', action='store_true',
dest='output_api_version', default=False,
help='Used for handling forwards/backwards compatibility '
'with --resume and --bootstrap')
group.add_option('--remote-trybot', action='store_true', default=False,
help='Indicates this is running on a remote trybot machine')
help='The unique ID in buildbucket of current build '
'generated by buildbucket.')
group.add_remote_option('--remote-patches', action='extend', default=[],
help='Patches uploaded by the trybot client when '
'run using the -p option')
# Note the default here needs to be hardcoded to 3; that is the last version
# that lacked this functionality.
group.add_option('--remote-version', type='int', default=3,
help='Used for compatibility checks w/tryjobs running in '
'older chromite instances')
group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT)
group.add_option('--ts-mon-task-num', type='int', default=0,
help='The task number of this process. Defaults to 0. '
'This argument is useful for running multiple copies '
'of cbuildbot without their metrics colliding.')
group.add_remote_option('--test-bootstrap', action='store_true',
help='Causes cbuildbot to bootstrap itself twice, '
'in the sequence A->B->C: A(unpatched) patches '
'and bootstraps B; B patches and bootstraps C')
help='Path to a pickled validation pool. Intended '
'for use only with the commit queue.')
help='Path to a json dumped metadata file. This '
'will be used as the initial metadata.')
group.add_remote_option('--master-build-id', type='int',
help='cidb build id of the master build to this '
'slave build.')
help='Override the tree status value that would be '
'returned from the the actual tree. Example '
'values: open, closed, throttled. When used '
'in conjunction with --debug, the tree status '
'will not be ignored as it usually is in a '
'--debug run.')
help='Override the result of the _FetchSlaveStatuses '
'method of MasterSlaveSyncCompletionStage, by '
'specifying a file with a pickle of the result '
'to be returned.')
# Debug options
# Temporary hack; in place till --dry-run replaces --debug.
# pylint: disable=W0212
group = parser.debug_group
debug = [x for x in group.option_list if x._long_opts == ['--debug']][0] += ' Currently functions as --dry-run in addition.'
debug.pass_through = True
group.add_option('--notee', action='store_false', dest='tee', default=True,
help='Disable logging and internal tee process. Primarily '
'used for debugging cbuildbot itself.')
return parser
def _FinishParsing(options, args):
"""Perform some parsing tasks that need to take place after optparse.
This function needs to be easily testable! Keep it free of
environment-dependent code. Put more detailed usage validation in
options: The options object returned by optparse
args: The args object returned by optparse
# Populate options.pass_through_args.
accepted, _ = commandline.FilteringParser.FilterArgs(
options.parsed_args, lambda x: x.opt_inst.pass_through)
if options.chrome_root:
if options.chrome_rev != constants.CHROME_REV_LOCAL:
cros_build_lib.Die('Chrome rev must be %s if chrome_root is set.' %
elif options.chrome_rev == constants.CHROME_REV_LOCAL:
cros_build_lib.Die('Chrome root must be set if chrome_rev is %s.' %
if options.chrome_version:
if options.chrome_rev != constants.CHROME_REV_SPEC:
cros_build_lib.Die('Chrome rev must be %s if chrome_version is set.' %
elif options.chrome_rev == constants.CHROME_REV_SPEC:
'Chrome rev must not be %s if chrome_version is not set.'
% constants.CHROME_REV_SPEC)
patches = bool(options.gerrit_patches or options.local_patches or
if options.remote:
if options.local:
cros_build_lib.Die('Cannot specify both --remote and --local')
# options.channels is a convenient way to detect payloads builds.
if (not options.list and not options.buildbot and not options.channels and
not patches):
prompt = ('No patches were provided; are you sure you want to just '
'run a remote build of %s?' % (
options.branch if options.branch else 'ToT'))
if not cros_build_lib.BooleanPrompt(prompt=prompt, default=False):
cros_build_lib.Die('Must provide patches when running with --remote.')
# --debug needs to be explicitly passed through for remote invocations.
release_mode_with_patches = (options.buildbot and patches and
'--debug' not in options.pass_through_args)
if len(args) > 1:
cros_build_lib.Die('Multiple configs not supported if not running with '
'--remote. Got %r', args)
if options.slaves:
cros_build_lib.Die('Cannot use --slaves if not running with --remote.')
release_mode_with_patches = (options.buildbot and patches and
not options.debug)
# When running in release mode, make sure we are running with checked-in code.
# We want checked-in cbuildbot/scripts to prevent errors, and we want to build
# a release image with checked-in code for CrOS packages.
if release_mode_with_patches:
'Cannot provide patches when running with --buildbot!')
if options.buildbot and options.remote_trybot:
'--buildbot and --remote-trybot cannot be used together.')
# Record whether --debug was set explicitly vs. it was inferred.
options.debug_forced = False
if options.debug:
options.debug_forced = True
if not options.debug:
# We don't set debug by default for
# 1. --buildbot invocations.
# 2. --remote invocations, because it needs to push changes to the tryjob
# repo.
options.debug = not options.buildbot and not options.remote
# Record the configs targeted.
options.build_targets = args[:]
if constants.BRANCH_UTIL_CONFIG in options.build_targets:
if options.remote:
'Running %s as a remote tryjob is not yet supported.',
if len(options.build_targets) > 1:
'Cannot run %s with any other configs.',
if not options.branch_name:
'Must specify --branch-name with the %s config.',
if options.branch and options.branch != options.branch_name:
'If --branch is specified with the %s config, it must'
' have the same value as --branch-name.',
exclusive_opts = {'--version': options.force_version,
'--delete-branch': options.delete_branch,
'--rename-to': options.rename_to}
if 1 != sum(1 for x in exclusive_opts.values() if x):
cros_build_lib.Die('When using the %s config, you must'
' specifiy one and only one of the following'
' options: %s.', constants.BRANCH_UTIL_CONFIG,
', '.join(exclusive_opts.keys()))
# When deleting or renaming a branch, the --branch and --nobootstrap
# options are implied.
if options.delete_branch or options.rename_to:
if not options.branch:'Automatically enabling sync to branch %s for this %s '
'flow.', options.branch_name,
options.branch = options.branch_name
if options.bootstrap:'Automatically disabling bootstrap step for this %s flow.',
options.bootstrap = False
elif any([options.delete_branch, options.rename_to, options.branch_name]):
'Cannot specify --delete-branch, --rename-to or --branch-name when not '
'running the %s config', constants.BRANCH_UTIL_CONFIG)
# pylint: disable=W0613
def _PostParseCheck(parser, options, args, site_config):
"""Perform some usage validation after we've parsed the arguments
parser: Option parser that was used to parse arguments.
options: The options returned by optparse.
args: The args returned by optparse.
site_config: config_lib.SiteConfig containing all config info.
if not args:
parser.error('Invalid usage: no configuration targets provided.'
'Use -h to see usage. Use -l to list supported configs.')
if not options.branch:
options.branch = git.GetChromiteTrackingBranch()
if not repository.IsARepoRoot(options.sourceroot):
if options.local_patches:
raise Exception('Could not find repo checkout at %s!'
% options.sourceroot)
# Because the default cache dir depends on other options, FindCacheDir
# always returns None, and we setup the default here.
if options.cache_dir is None:
# Note, options.sourceroot is set regardless of the path
# actually existing.
if options.buildroot is not None:
options.cache_dir = os.path.join(options.buildroot, '.cache')
elif os.path.exists(options.sourceroot):
options.cache_dir = os.path.join(options.sourceroot, '.cache')
options.cache_dir = parser.FindCacheDir(parser, options)
options.cache_dir = os.path.abspath(options.cache_dir)
if options.local_patches:
options.local_patches = _CheckLocalPatches(
options.sourceroot, options.local_patches)
default = os.environ.get('CBUILDBOT_DEFAULT_MODE')
if (default and not any([options.local, options.buildbot,
options.remote, options.remote_trybot])):'CBUILDBOT_DEFAULT_MODE=%s env var detected, using it.'
% default)
default = default.lower()
if default == 'local':
options.local = True
elif default == 'remote':
options.remote = True
elif default == 'buildbot':
options.buildbot = True
cros_build_lib.Die("CBUILDBOT_DEFAULT_MODE value %s isn't supported. "
% default)
# Ensure that all args are legitimate config targets.
invalid_targets = []
for arg in args:
if arg not in site_config:
logging.error('No such configuraton target: "%s".', arg)
build_config = site_config[arg]
is_payloads_build = build_config.build_type == constants.PAYLOADS_TYPE
if options.channels and not is_payloads_build:
cros_build_lib.Die('--channel must only be used with a payload config,'
' not target (%s).' % arg)
if not options.channels and is_payloads_build:
cros_build_lib.Die('payload configs (%s) require --channel to do anything'
' useful.' % arg)
# The --version option is not compatible with an external target unless the
# --buildbot option is specified. More correctly, only "paladin versions"
# will work with external targets, and those are only used with --buildbot.
# If --buildbot is specified, then user should know what they are doing and
# only specify a version that will work. See
if (options.force_version and
not (options.buildbot or build_config.internal)):
cros_build_lib.Die('Cannot specify --version without --buildbot for an'
' external target (%s).' % arg)
if invalid_targets:
cros_build_lib.Die('One or more invalid configuration targets specified. '
'You can check the available configs by running '
'`cbuildbot --list --all`')
def ParseCommandLine(parser, argv):
"""Completely parse the commandline arguments"""
(options, args) = parser.parse_args(argv)
# Strip out null arguments.
# TODO(rcui): Remove when buildbot is fixed
args = [arg for arg in args if arg]
if options.deprecated_use_buildbucket:
logging.warning('--use-buildbucket is deprecated, and ignored.')
if options.output_api_version:
_FinishParsing(options, args)
return options, args
def _GetRunEnvironment(options, build_config):
"""Determine whether this is a prod/debug/standalone run."""
# TODO(akeshet): This is a temporary workaround to make sure that the cidb
# is not used on waterfalls that the db schema does not support (in particular
# the waterfall).
# See
wfall = os.environ.get('BUILDBOT_MASTERNAME', '')
if not wfall in waterfall.CIDB_KNOWN_WATERFALLS:
# TODO(akeshet): Clean up this code once we have better defined flags to
# specify on-or-off waterfall and on-or-off production runs of cbuildbot.
# See
# --buildbot runs should use the production services, unless the --debug flag
# is also present.
if options.buildbot:
if options.debug:
# --remote-trybot runs should use the debug services, with the exception of
# pre-cq builds, which should use the production services.
if options.remote_trybot:
if build_config['pre_cq']:
# If neither --buildbot nor --remote-trybot flag was used, don't use external
# services.
def _SetupConnections(options, build_config):
"""Set up CIDB connections using the appropriate Setup call.
options: Command line options structure.
build_config: Config object for this build.
# Outline:
# 1) Based on options and build_config, decide whether we are a production
# run, debug run, or standalone run.
# 2) Set up cidb instance accordingly.
# 3) Update topology info from cidb, so that any other service set up can use
# topology.
# 4) Set up any other services.
run_type = _GetRunEnvironment(options, build_config)
if run_type == _ENVIRONMENT_PROD:
context = ts_mon_config.SetupTsMonGlobalState(
'cbuildbot', indirect=True, task_num=options.ts_mon_task_num)
elif run_type == _ENVIRONMENT_DEBUG:
context = ts_mon_config.TrivialContextManager()
context = ts_mon_config.TrivialContextManager()
db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
return context
# TODO(build): This function is too damn long.
def main(argv):
# Turn on strict sudo checks.
cros_build_lib.STRICT_SUDO = True
# Set umask to 022 so files created by buildbot are readable.
parser = _CreateParser()
options, args = ParseCommandLine(parser, argv)
if options.config_repo:
cros_build_lib.Die('Deprecated usage. Ping you need it.')
# Fetch our site_config now, because we need it to do anything else.
site_config = config_lib.GetConfig()
if options.list:
_PrintValidConfigs(site_config, options.print_all)
_PostParseCheck(parser, options, args, site_config)
if options.enable_buildbot_tags:
if options.remote:
# Verify configs are valid.
# If hwtest flag is enabled, post a warning that HWTest step may fail if the
# specified board is not a released platform or it is a generic overlay.
for bot in args:
build_config = site_config[bot]
if options.hwtest:
'If %s is not a released platform or it is a generic overlay, '
'the HWTest step will most likely not run; please ask the lab '
'team for help if this is unexpected.' % build_config['boards'])
# Verify gerrit patches are valid.
print('Verifying patches...')
patch_pool = trybot_patch_pool.TrybotPatchPool.FromOptions(
# --debug need to be explicitly passed through for remote invocations.
if options.buildbot and '--debug' not in options.pass_through_args:
print('Submitting tryjob...')
with _SetupConnections(options, build_config):
description = options.remote_description
if description is None:
description = remote_try.DefaultDescription(
tryjob = remote_try.RemoteTryJob(args, patch_pool.local_patches,
tryjob.Submit(testjob=options.test_tryjob, dryrun=False)
print('Tryjob submitted!')
print(('Go to %s to view the status of your job.'
% tryjob.GetTrybotWaterfallLink()))
elif (not options.buildbot and not options.remote_trybot
and not options.resume and not options.local):
cros_build_lib.Die('Please use --remote or --local to run trybots')
elif options.buildbot and not options.debug:
if not cros_build_lib.HostIsCIBuilder():
# Cannot run --buildbot if both --debug and --remote aren't specified.
cros_build_lib.Die('This host isn\'t a continuous-integration builder.')
# Only one config arg is allowed in this mode, which was confirmed earlier.
bot_id = args[-1]
build_config = site_config[bot_id]
# TODO: Re-enable this block when reference_repo support handles this
# properly. (see chromium:330775)
# if options.reference_repo is None:
# repo_path = os.path.join(options.sourceroot, '.repo')
# # If we're being run from a repo checkout, reuse the repo's git pool to
# # cut down on sync time.
# if os.path.exists(repo_path):
# options.reference_repo = options.sourceroot
if options.reference_repo:
if not os.path.exists(options.reference_repo):
parser.error('Reference path %s does not exist'
% (options.reference_repo,))
elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
parser.error('Reference path %s does not look to be the base of a '
'repo checkout; no .repo exists in the root.'
% (options.reference_repo,))
if (options.buildbot or options.remote_trybot) and not options.resume:
if not options.cgroups:
parser.error('Options --buildbot/--remote-trybot and --nocgroups cannot '
'be used together. Cgroup support is required for '
'buildbot/remote-trybot mode.')
if not cgroups.Cgroup.IsSupported():
parser.error('Option --buildbot/--remote-trybot was given, but this '
'system does not support cgroups. Failing.')
missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
if missing:
parser.error('Option --buildbot/--remote-trybot requires the following '
"binaries which couldn't be found in $PATH: %s"
% (', '.join(missing)))
if options.reference_repo:
options.reference_repo = os.path.abspath(options.reference_repo)
if not options.buildroot:
if options.buildbot:
parser.error('Please specify a buildroot with the --buildroot option.')
options.buildroot = _DetermineDefaultBuildRoot(options.sourceroot,
# We use a marker file in the buildroot to indicate the user has
# consented to using this directory.
if not os.path.exists(repository.GetTrybotMarkerPath(options.buildroot)):
# Sanity check of buildroot- specifically that it's not pointing into the
# midst of an existing repo since git-repo doesn't support nesting.
if (not repository.IsARepoRoot(options.buildroot) and
parser.error('Configured buildroot %s points into a repository checkout, '
'rather than the root of it. This is not supported.'
% options.buildroot)
if not options.log_dir:
options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
log_file = None
if options.tee:
log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
with cros_build_lib.ContextManagerStack() as stack:
options.preserve_paths = set()
if log_file is not None:
# We don't want the critical section to try to clean up the tee process,
# so we run Tee (forked off) outside of it. This prevents a deadlock
# because the Tee process only exits when its pipe is closed, and the
# critical section accidentally holds on to that file handle.
stack.Add(tee.Tee, log_file)
critical_section = stack.Add(cleanup.EnforcedCleanupSection)
if not options.resume:
# If we're in resume mode, use our parents tempdir rather than
# nesting another layer.
stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))
if options.cgroups:
stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
# Mark everything between EnforcedCleanupSection and here as having to
# be rolled back via the contextmanager cleanup handlers. This
# ensures that sudo bits cannot outlive cbuildbot, that anything
# cgroups would kill gets killed, etc.
if not options.buildbot:
build_config = config_lib.OverrideConfigForTrybot(
build_config, options)
if options.mock_tree_status is not None:
stack.Add(mock.patch.object, tree_status, '_GetStatus',
if options.mock_slave_status is not None:
with open(options.mock_slave_status, 'r') as f:
mock_statuses = pickle.load(f)
for key, value in mock_statuses.iteritems():
mock_statuses[key] = builder_status_lib.BuilderStatus(**value)
stack.Add(_SetupConnections, options, build_config)
timeout_display_message = None
# For master-slave builds: Update slave's timeout using master's published
# deadline.
if options.buildbot and options.master_build_id is not None:
slave_timeout = None
if cidb.CIDBConnectionFactory.IsCIDBSetup():
cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
if cidb_handle:
slave_timeout = cidb_handle.GetTimeToDeadline(options.master_build_id)
if slave_timeout is not None:
# We artificially set a minimum slave_timeout because '0' is handled
# specially, and because we don't want to timeout while trying to set
# things up.
slave_timeout = max(slave_timeout, 20)
if options.timeout == 0 or slave_timeout < options.timeout:'Updating slave build timeout to %d seconds enforced '
'by the master', slave_timeout)
options.timeout = slave_timeout
timeout_display_message = (
'This build has reached the timeout deadline set by the master. '
'Either this stage or a previous one took too long (see stage '
'timing historical summary in ReportStage) or the build failed '
'to start on time.')
logging.warning('Could not get master deadline for master-slave build. '
'Can not set slave timeout.')
if options.timeout > 0:
stack.Add(timeout_util.FatalTimeout, options.timeout,
_RunBuildStagesWrapper(options, site_config, build_config)
except failures_lib.ExitEarlyException as ex:
# This build finished successfully. Do not re-raise ExitEarlyException.'One stage exited early: %s', ex)