GoogleGit

blob: f08e3d23db75f5f71c9fc75106a0c1262dce5c8c [file] [log] [blame]
  1. # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Main builder code for Chromium OS.
  5. Used by Chromium OS buildbot configuration for all Chromium OS builds including
  6. full and pre-flight-queue builds.
  7. """
  8. from __future__ import print_function
  9. import distutils.version
  10. import glob
  11. import json
  12. import mock
  13. import optparse
  14. import os
  15. import pickle
  16. import sys
  17. from chromite.cbuildbot import builders
  18. from chromite.cbuildbot import cbuildbot_run
  19. from chromite.cbuildbot import config_lib
  20. from chromite.cbuildbot import constants
  21. from chromite.cbuildbot import manifest_version
  22. from chromite.cbuildbot import remote_try
  23. from chromite.cbuildbot import repository
  24. from chromite.cbuildbot import tee
  25. from chromite.cbuildbot import topology
  26. from chromite.cbuildbot import tree_status
  27. from chromite.cbuildbot import trybot_patch_pool
  28. from chromite.cbuildbot.stages import completion_stages
  29. from chromite.lib import cidb
  30. from chromite.lib import cgroups
  31. from chromite.lib import cleanup
  32. from chromite.lib import commandline
  33. from chromite.lib import cros_build_lib
  34. from chromite.lib import cros_logging as logging
  35. from chromite.lib import git
  36. from chromite.lib import graphite
  37. from chromite.lib import gob_util
  38. from chromite.lib import osutils
  39. from chromite.lib import parallel
  40. from chromite.lib import retry_stats
  41. from chromite.lib import sudo
  42. from chromite.lib import timeout_util
  43. _DEFAULT_LOG_DIR = 'cbuildbot_logs'
  44. _BUILDBOT_LOG_FILE = 'cbuildbot.log'
  45. _DEFAULT_EXT_BUILDROOT = 'trybot'
  46. _DEFAULT_INT_BUILDROOT = 'trybot-internal'
  47. _BUILDBOT_REQUIRED_BINARIES = ('pbzip2',)
  48. _API_VERSION_ATTR = 'api_version'
  49. def _PrintValidConfigs(site_config, display_all=False):
  50. """Print a list of valid buildbot configs.
  51. Args:
  52. site_config: config_lib.SiteConfig containing all config info.
  53. display_all: Print all configs. Otherwise, prints only configs with
  54. trybot_list=True.
  55. """
  56. def _GetSortKey(config_name):
  57. config_dict = site_config[config_name]
  58. return (not config_dict['trybot_list'], config_dict['description'],
  59. config_name)
  60. COLUMN_WIDTH = 45
  61. if not display_all:
  62. print('Note: This is the common list; for all configs, use --all.')
  63. print('config'.ljust(COLUMN_WIDTH), 'description')
  64. print('------'.ljust(COLUMN_WIDTH), '-----------')
  65. config_names = site_config.keys()
  66. config_names.sort(key=_GetSortKey)
  67. for name in config_names:
  68. if display_all or site_config[name]['trybot_list']:
  69. desc = site_config[name].get('description')
  70. desc = desc if desc else ''
  71. print(name.ljust(COLUMN_WIDTH), desc)
  72. def _ConfirmBuildRoot(buildroot):
  73. """Confirm with user the inferred buildroot, and mark it as confirmed."""
  74. logging.warning('Using default directory %s as buildroot', buildroot)
  75. if not cros_build_lib.BooleanPrompt(default=False):
  76. print('Please specify a different buildroot via the --buildroot option.')
  77. sys.exit(0)
  78. if not os.path.exists(buildroot):
  79. os.mkdir(buildroot)
  80. repository.CreateTrybotMarker(buildroot)
  81. def _ConfirmRemoteBuildbotRun():
  82. """Confirm user wants to run with --buildbot --remote."""
  83. logging.warning(
  84. 'You are about to launch a PRODUCTION job! This is *NOT* a '
  85. 'trybot run! Are you sure?')
  86. if not cros_build_lib.BooleanPrompt(default=False):
  87. print('Please specify --pass-through="--debug".')
  88. sys.exit(0)
  89. def _DetermineDefaultBuildRoot(sourceroot, internal_build):
  90. """Default buildroot to be under the directory that contains current checkout.
  91. Args:
  92. internal_build: Whether the build is an internal build
  93. sourceroot: Use specified sourceroot.
  94. """
  95. if not repository.IsARepoRoot(sourceroot):
  96. cros_build_lib.Die(
  97. 'Could not find root of local checkout at %s. Please specify '
  98. 'using the --sourceroot option.' % sourceroot)
  99. # Place trybot buildroot under the directory containing current checkout.
  100. top_level = os.path.dirname(os.path.realpath(sourceroot))
  101. if internal_build:
  102. buildroot = os.path.join(top_level, _DEFAULT_INT_BUILDROOT)
  103. else:
  104. buildroot = os.path.join(top_level, _DEFAULT_EXT_BUILDROOT)
  105. return buildroot
  106. def _BackupPreviousLog(log_file, backup_limit=25):
  107. """Rename previous log.
  108. Args:
  109. log_file: The absolute path to the previous log.
  110. backup_limit: Maximum number of old logs to keep.
  111. """
  112. if os.path.exists(log_file):
  113. old_logs = sorted(glob.glob(log_file + '.*'),
  114. key=distutils.version.LooseVersion)
  115. if len(old_logs) >= backup_limit:
  116. os.remove(old_logs[0])
  117. last = 0
  118. if old_logs:
  119. last = int(old_logs.pop().rpartition('.')[2])
  120. os.rename(log_file, log_file + '.' + str(last + 1))
  121. def _IsDistributedBuilder(options, chrome_rev, build_config):
  122. """Determines whether the builder should be a DistributedBuilder.
  123. Args:
  124. options: options passed on the commandline.
  125. chrome_rev: Chrome revision to build.
  126. build_config: Builder configuration dictionary.
  127. Returns:
  128. True if the builder should be a distributed_builder
  129. """
  130. if build_config['pre_cq']:
  131. return True
  132. elif not options.buildbot:
  133. return False
  134. elif chrome_rev in (constants.CHROME_REV_TOT,
  135. constants.CHROME_REV_LOCAL,
  136. constants.CHROME_REV_SPEC):
  137. # We don't do distributed logic to TOT Chrome PFQ's, nor local
  138. # chrome roots (e.g. chrome try bots)
  139. # TODO(davidjames): Update any builders that rely on this logic to use
  140. # manifest_version=False instead.
  141. return False
  142. elif build_config['manifest_version']:
  143. return True
  144. return False
  145. def _RunBuildStagesWrapper(options, site_config, build_config):
  146. """Helper function that wraps RunBuildStages()."""
  147. logging.info('cbuildbot was executed with args %s' %
  148. cros_build_lib.CmdToStr(sys.argv))
  149. chrome_rev = build_config['chrome_rev']
  150. if options.chrome_rev:
  151. chrome_rev = options.chrome_rev
  152. if chrome_rev == constants.CHROME_REV_TOT:
  153. options.chrome_version = gob_util.GetTipOfTrunkRevision(
  154. constants.CHROMIUM_GOB_URL)
  155. options.chrome_rev = constants.CHROME_REV_SPEC
  156. # If it's likely we'll need to build Chrome, fetch the source.
  157. if build_config['sync_chrome'] is None:
  158. options.managed_chrome = (
  159. chrome_rev != constants.CHROME_REV_LOCAL and
  160. (not build_config['usepkg_build_packages'] or chrome_rev or
  161. build_config['profile'] or options.rietveld_patches))
  162. else:
  163. options.managed_chrome = build_config['sync_chrome']
  164. if options.managed_chrome:
  165. # Tell Chrome to fetch the source locally.
  166. internal = constants.USE_CHROME_INTERNAL in build_config['useflags']
  167. chrome_src = 'chrome-src-internal' if internal else 'chrome-src'
  168. options.chrome_root = os.path.join(options.cache_dir, 'distfiles', 'target',
  169. chrome_src)
  170. elif options.rietveld_patches:
  171. cros_build_lib.Die('This builder does not support Rietveld patches.')
  172. metadata_dump_dict = {}
  173. if options.metadata_dump:
  174. with open(options.metadata_dump, 'r') as metadata_file:
  175. metadata_dump_dict = json.loads(metadata_file.read())
  176. # We are done munging options values, so freeze options object now to avoid
  177. # further abuse of it.
  178. # TODO(mtennant): one by one identify each options value override and see if
  179. # it can be handled another way. Try to push this freeze closer and closer
  180. # to the start of the script (e.g. in or after _PostParseCheck).
  181. options.Freeze()
  182. with parallel.Manager() as manager:
  183. builder_run = cbuildbot_run.BuilderRun(
  184. options, site_config, build_config, manager)
  185. if metadata_dump_dict:
  186. builder_run.attrs.metadata.UpdateWithDict(metadata_dump_dict)
  187. if builder_run.config.builder_class_name is None:
  188. # TODO: This should get relocated to chromeos_config.
  189. if _IsDistributedBuilder(options, chrome_rev, build_config):
  190. builder_cls_name = 'simple_builders.DistributedBuilder'
  191. else:
  192. builder_cls_name = 'simple_builders.SimpleBuilder'
  193. builder_cls = builders.GetBuilderClass(builder_cls_name)
  194. builder = builder_cls(builder_run)
  195. else:
  196. builder = builders.Builder(builder_run)
  197. if not builder.Run():
  198. sys.exit(1)
  199. # Parser related functions
  200. def _CheckLocalPatches(sourceroot, local_patches):
  201. """Do an early quick check of the passed-in patches.
  202. If the branch of a project is not specified we append the current branch the
  203. project is on.
  204. TODO(davidjames): The project:branch format isn't unique, so this means that
  205. we can't differentiate what directory the user intended to apply patches to.
  206. We should references by directory instead.
  207. Args:
  208. sourceroot: The checkout where patches are coming from.
  209. local_patches: List of patches to check in project:branch format.
  210. Returns:
  211. A list of patches that have been verified, in project:branch format.
  212. """
  213. verified_patches = []
  214. manifest = git.ManifestCheckout.Cached(sourceroot)
  215. for patch in local_patches:
  216. project, _, branch = patch.partition(':')
  217. checkouts = manifest.FindCheckouts(project, only_patchable=True)
  218. if not checkouts:
  219. cros_build_lib.Die('Project %s does not exist.' % (project,))
  220. if len(checkouts) > 1:
  221. cros_build_lib.Die(
  222. 'We do not yet support local patching for projects that are checked '
  223. 'out to multiple directories. Try uploading your patch to gerrit '
  224. 'and referencing it via the -g option instead.'
  225. )
  226. ok = False
  227. for checkout in checkouts:
  228. project_dir = checkout.GetPath(absolute=True)
  229. # If no branch was specified, we use the project's current branch.
  230. if not branch:
  231. local_branch = git.GetCurrentBranch(project_dir)
  232. else:
  233. local_branch = branch
  234. if local_branch and git.DoesCommitExistInRepo(project_dir, local_branch):
  235. verified_patches.append('%s:%s' % (project, local_branch))
  236. ok = True
  237. if not ok:
  238. if branch:
  239. cros_build_lib.Die('Project %s does not have branch %s'
  240. % (project, branch))
  241. else:
  242. cros_build_lib.Die('Project %s is not on a branch!' % (project,))
  243. return verified_patches
  244. def _CheckChromeVersionOption(_option, _opt_str, value, parser):
  245. """Upgrade other options based on chrome_version being passed."""
  246. value = value.strip()
  247. if parser.values.chrome_rev is None and value:
  248. parser.values.chrome_rev = constants.CHROME_REV_SPEC
  249. parser.values.chrome_version = value
  250. def _CheckChromeRootOption(_option, _opt_str, value, parser):
  251. """Validate and convert chrome_root to full-path form."""
  252. if parser.values.chrome_rev is None:
  253. parser.values.chrome_rev = constants.CHROME_REV_LOCAL
  254. parser.values.chrome_root = value
  255. def _CheckChromeRevOption(_option, _opt_str, value, parser):
  256. """Validate the chrome_rev option."""
  257. value = value.strip()
  258. if value not in constants.VALID_CHROME_REVISIONS:
  259. raise optparse.OptionValueError('Invalid chrome rev specified')
  260. parser.values.chrome_rev = value
  261. def FindCacheDir(_parser, _options):
  262. return None
  263. class CustomGroup(optparse.OptionGroup):
  264. """Custom option group which supports arguments passed-through to trybot."""
  265. def add_remote_option(self, *args, **kwargs):
  266. """For arguments that are passed-through to remote trybot."""
  267. return optparse.OptionGroup.add_option(self, *args,
  268. remote_pass_through=True,
  269. **kwargs)
  270. class CustomOption(commandline.FilteringOption):
  271. """Subclass FilteringOption class to implement pass-through and api."""
  272. ACTIONS = commandline.FilteringOption.ACTIONS + ('extend',)
  273. STORE_ACTIONS = commandline.FilteringOption.STORE_ACTIONS + ('extend',)
  274. TYPED_ACTIONS = commandline.FilteringOption.TYPED_ACTIONS + ('extend',)
  275. ALWAYS_TYPED_ACTIONS = (commandline.FilteringOption.ALWAYS_TYPED_ACTIONS +
  276. ('extend',))
  277. def __init__(self, *args, **kwargs):
  278. # The remote_pass_through argument specifies whether we should directly
  279. # pass the argument (with its value) onto the remote trybot.
  280. self.pass_through = kwargs.pop('remote_pass_through', False)
  281. self.api_version = int(kwargs.pop('api', '0'))
  282. commandline.FilteringOption.__init__(self, *args, **kwargs)
  283. def take_action(self, action, dest, opt, value, values, parser):
  284. if action == 'extend':
  285. # If there is extra spaces between each argument, we get '' which later
  286. # code barfs on, so skip those. e.g. We see this with the forms:
  287. # cbuildbot -p 'proj:branch ' ...
  288. # cbuildbot -p ' proj:branch' ...
  289. # cbuildbot -p 'proj:branch proj2:branch' ...
  290. lvalue = value.split()
  291. values.ensure_value(dest, []).extend(lvalue)
  292. commandline.FilteringOption.take_action(
  293. self, action, dest, opt, value, values, parser)
  294. class CustomParser(commandline.FilteringParser):
  295. """Custom option parser which supports arguments passed-trhough to trybot"""
  296. DEFAULT_OPTION_CLASS = CustomOption
  297. def add_remote_option(self, *args, **kwargs):
  298. """For arguments that are passed-through to remote trybot."""
  299. return self.add_option(*args, remote_pass_through=True, **kwargs)
  300. def _CreateParser():
  301. """Generate and return the parser with all the options."""
  302. # Parse options
  303. usage = 'usage: %prog [options] buildbot_config [buildbot_config ...]'
  304. parser = CustomParser(usage=usage, caching=FindCacheDir)
  305. # Main options
  306. parser.add_option('-l', '--list', action='store_true', dest='list',
  307. default=False,
  308. help='List the suggested trybot configs to use (see --all)')
  309. parser.add_option('-a', '--all', action='store_true', dest='print_all',
  310. default=False,
  311. help='List all of the buildbot configs available w/--list')
  312. parser.add_option('--local', default=False, action='store_true',
  313. help=('Specifies that this tryjob should be run locally. '
  314. 'Implies --debug.'))
  315. parser.add_option('--remote', default=False, action='store_true',
  316. help='Specifies that this tryjob should be run remotely.')
  317. parser.add_remote_option('-b', '--branch',
  318. help=('The manifest branch to test. The branch to '
  319. 'check the buildroot out to.'))
  320. parser.add_option('-r', '--buildroot', dest='buildroot', type='path',
  321. help=('Root directory where source is checked out to, and '
  322. 'where the build occurs. For external build configs, '
  323. "defaults to 'trybot' directory at top level of your "
  324. 'repo-managed checkout.'))
  325. parser.add_option('--bootstrap-dir', type='path', default=None,
  326. help='Bootstrapping cbuildbot may involve checking out '
  327. 'multiple copies of chromite. All these checkouts '
  328. 'will be contained in the directory specified here. '
  329. 'Default:%s' % osutils.GetGlobalTempDir())
  330. parser.add_remote_option('--chrome_rev', default=None, type='string',
  331. action='callback', dest='chrome_rev',
  332. callback=_CheckChromeRevOption,
  333. help=('Revision of Chrome to use, of type [%s]'
  334. % '|'.join(constants.VALID_CHROME_REVISIONS)))
  335. parser.add_remote_option('--profile', default=None, type='string',
  336. action='store', dest='profile',
  337. help='Name of profile to sub-specify board variant.')
  338. parser.add_option('-c', '--config_repo',
  339. help=('Cloneable path to the git repository containing '
  340. 'the site configuration to use.'))
  341. #
  342. # Patch selection options.
  343. #
  344. group = CustomGroup(
  345. parser,
  346. 'Patch Options')
  347. group.add_remote_option('-g', '--gerrit-patches', action='extend',
  348. default=[], type='string',
  349. metavar="'Id1 *int_Id2...IdN'",
  350. help=('Space-separated list of short-form Gerrit '
  351. "Change-Id's or change numbers to patch. "
  352. "Please prepend '*' to internal Change-Id's"))
  353. group.add_remote_option('-G', '--rietveld-patches', action='extend',
  354. default=[], type='string',
  355. metavar="'id1[:subdir1]...idN[:subdirN]'",
  356. help=('Space-separated list of short-form Rietveld '
  357. 'issue numbers to patch. If no subdir is '
  358. 'specified, the src directory is used.'))
  359. group.add_option('-p', '--local-patches', action='extend', default=[],
  360. metavar="'<project1>[:<branch1>]...<projectN>[:<branchN>]'",
  361. help=('Space-separated list of project branches with '
  362. 'patches to apply. Projects are specified by name. '
  363. 'If no branch is specified the current branch of the '
  364. 'project will be used.'))
  365. parser.add_option_group(group)
  366. #
  367. # Remote trybot options.
  368. #
  369. group = CustomGroup(
  370. parser,
  371. 'Remote Trybot Options (--remote)')
  372. group.add_remote_option('--hwtest', dest='hwtest', action='store_true',
  373. default=False,
  374. help='Run the HWTest stage (tests on real hardware)')
  375. group.add_option('--remote-description', default=None,
  376. help=('Attach an optional description to a --remote run '
  377. 'to make it easier to identify the results when it '
  378. 'finishes'))
  379. group.add_option('--slaves', action='extend', default=[],
  380. help=('Specify specific remote tryslaves to run on (e.g. '
  381. 'build149-m2); if the bot is busy, it will be queued'))
  382. group.add_remote_option('--channel', dest='channels', action='extend',
  383. default=[],
  384. help=('Specify a channel for a payloads trybot. Can '
  385. 'be specified multiple times. No valid for '
  386. 'non-payloads configs.'))
  387. group.add_option('--test-tryjob', action='store_true',
  388. default=False,
  389. help=('Submit a tryjob to the test repository. Will not '
  390. 'show up on the production trybot waterfall.'))
  391. parser.add_option_group(group)
  392. #
  393. # Branch creation options.
  394. #
  395. group = CustomGroup(
  396. parser,
  397. 'Branch Creation Options (used with branch-util)')
  398. group.add_remote_option('--branch-name',
  399. help='The branch to create or delete.')
  400. group.add_remote_option('--delete-branch', default=False, action='store_true',
  401. help='Delete the branch specified in --branch-name.')
  402. group.add_remote_option('--rename-to', type='string',
  403. help='Rename a branch to the specified name.')
  404. group.add_remote_option('--force-create', default=False, action='store_true',
  405. help='Overwrites an existing branch.')
  406. group.add_remote_option('--skip-remote-push', default=False,
  407. action='store_true',
  408. help='Do not actually push to remote git repos. '
  409. 'Used for end-to-end testing branching.')
  410. parser.add_option_group(group)
  411. #
  412. # Advanced options.
  413. #
  414. group = CustomGroup(
  415. parser,
  416. 'Advanced Options',
  417. 'Caution: use these options at your own risk.')
  418. group.add_remote_option('--bootstrap-args', action='append', default=[],
  419. help=('Args passed directly to the bootstrap re-exec '
  420. 'to skip verification by the bootstrap code'))
  421. group.add_remote_option('--buildbot', dest='buildbot', action='store_true',
  422. default=False, help='This is running on a buildbot')
  423. group.add_remote_option('--no-buildbot-tags', action='store_false',
  424. dest='enable_buildbot_tags', default=True,
  425. help='Suppress buildbot specific tags from log '
  426. 'output. This is used to hide recursive '
  427. 'cbuilbot runs on the waterfall.')
  428. group.add_remote_option('--buildnumber', help='build number', type='int',
  429. default=0)
  430. group.add_option('--chrome_root', default=None, type='path',
  431. action='callback', callback=_CheckChromeRootOption,
  432. dest='chrome_root', help='Local checkout of Chrome to use.')
  433. group.add_remote_option('--chrome_version', default=None, type='string',
  434. action='callback', dest='chrome_version',
  435. callback=_CheckChromeVersionOption,
  436. help=('Used with SPEC logic to force a particular '
  437. 'git revision of chrome rather than the '
  438. 'latest.'))
  439. group.add_remote_option('--clobber', action='store_true', dest='clobber',
  440. default=False,
  441. help='Clears an old checkout before syncing')
  442. group.add_remote_option('--latest-toolchain', action='store_true',
  443. default=False,
  444. help='Use the latest toolchain.')
  445. parser.add_option('--log_dir', dest='log_dir', type='path',
  446. help=('Directory where logs are stored.'))
  447. group.add_remote_option('--maxarchives', dest='max_archive_builds',
  448. default=3, type='int',
  449. help='Change the local saved build count limit.')
  450. parser.add_remote_option('--manifest-repo-url',
  451. help=('Overrides the default manifest repo url.'))
  452. group.add_remote_option('--compilecheck', action='store_true', default=False,
  453. help='Only verify compilation and unit tests.')
  454. group.add_remote_option('--noarchive', action='store_false', dest='archive',
  455. default=True, help="Don't run archive stage.")
  456. group.add_remote_option('--nobootstrap', action='store_false',
  457. dest='bootstrap', default=True,
  458. help=("Don't checkout and run from a standalone "
  459. 'chromite repo.'))
  460. group.add_remote_option('--nobuild', action='store_false', dest='build',
  461. default=True,
  462. help="Don't actually build (for cbuildbot dev)")
  463. group.add_remote_option('--noclean', action='store_false', dest='clean',
  464. default=True, help="Don't clean the buildroot")
  465. group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
  466. default=True,
  467. help='Disable cbuildbots usage of cgroups.')
  468. group.add_remote_option('--nochromesdk', action='store_false',
  469. dest='chrome_sdk', default=True,
  470. help=("Don't run the ChromeSDK stage which builds "
  471. 'Chrome outside of the chroot.'))
  472. group.add_remote_option('--noprebuilts', action='store_false',
  473. dest='prebuilts', default=True,
  474. help="Don't upload prebuilts.")
  475. group.add_remote_option('--nopatch', action='store_false',
  476. dest='postsync_patch', default=True,
  477. help=("Don't run PatchChanges stage. This does not "
  478. 'disable patching in of chromite patches '
  479. 'during BootstrapStage.'))
  480. group.add_remote_option('--nopaygen', action='store_false',
  481. dest='paygen', default=True,
  482. help="Don't generate payloads.")
  483. group.add_remote_option('--noreexec', action='store_false',
  484. dest='postsync_reexec', default=True,
  485. help="Don't reexec into the buildroot after syncing.")
  486. group.add_remote_option('--nosdk', action='store_true',
  487. default=False,
  488. help='Re-create the SDK from scratch.')
  489. group.add_remote_option('--nosync', action='store_false', dest='sync',
  490. default=True, help="Don't sync before building.")
  491. group.add_remote_option('--notests', action='store_false', dest='tests',
  492. default=True,
  493. help=('Override values from buildconfig and run no '
  494. 'tests.'))
  495. group.add_remote_option('--noimagetests', action='store_false',
  496. dest='image_test', default=True,
  497. help=('Override values from buildconfig and run no '
  498. 'image tests.'))
  499. group.add_remote_option('--nouprev', action='store_false', dest='uprev',
  500. default=True,
  501. help=('Override values from buildconfig and never '
  502. 'uprev.'))
  503. group.add_option('--reference-repo', action='store', default=None,
  504. dest='reference_repo',
  505. help=('Reuse git data stored in an existing repo '
  506. 'checkout. This can drastically reduce the network '
  507. 'time spent setting up the trybot checkout. By '
  508. "default, if this option isn't given but cbuildbot "
  509. 'is invoked from a repo checkout, cbuildbot will '
  510. 'use the repo root.'))
  511. group.add_option('--resume', action='store_true', default=False,
  512. help='Skip stages already successfully completed.')
  513. group.add_remote_option('--timeout', action='store', type='int', default=0,
  514. help=('Specify the maximum amount of time this job '
  515. 'can run for, at which point the build will be '
  516. 'aborted. If set to zero, then there is no '
  517. 'timeout.'))
  518. group.add_remote_option('--version', dest='force_version', default=None,
  519. help=('Used with manifest logic. Forces use of this '
  520. 'version rather than create or get latest. '
  521. 'Examples: 4815.0.0-rc1, 4815.1.2'))
  522. parser.add_option_group(group)
  523. #
  524. # Internal options.
  525. #
  526. group = CustomGroup(
  527. parser,
  528. 'Internal Chromium OS Build Team Options',
  529. 'Caution: these are for meant for the Chromium OS build team only')
  530. group.add_remote_option('--archive-base', type='gs_path',
  531. help=('Base GS URL (gs://<bucket_name>/<path>) to '
  532. 'upload archive artifacts to'))
  533. group.add_remote_option(
  534. '--cq-gerrit-query', dest='cq_gerrit_override', default=None,
  535. help=('If given, this gerrit query will be used to find what patches to '
  536. "test, rather than the normal 'CommitQueue>=1 AND Verified=1 AND "
  537. "CodeReview=2' query it defaults to. Use with care- note "
  538. 'additionally this setting only has an effect if the buildbot '
  539. "target is a cq target, and we're in buildbot mode."))
  540. group.add_option('--pass-through', dest='pass_through_args', action='append',
  541. type='string', default=[])
  542. group.add_option('--reexec-api-version', dest='output_api_version',
  543. action='store_true', default=False,
  544. help=('Used for handling forwards/backwards compatibility '
  545. 'with --resume and --bootstrap'))
  546. group.add_option('--remote-trybot', dest='remote_trybot',
  547. action='store_true', default=False,
  548. help='Indicates this is running on a remote trybot machine')
  549. group.add_remote_option('--remote-patches', action='extend', default=[],
  550. help=('Patches uploaded by the trybot client when '
  551. 'run using the -p option'))
  552. # Note the default here needs to be hardcoded to 3; that is the last version
  553. # that lacked this functionality.
  554. group.add_option('--remote-version', default=3, type=int, action='store',
  555. help=('Used for compatibility checks w/tryjobs running in '
  556. 'older chromite instances'))
  557. group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT)
  558. group.add_remote_option('--test-bootstrap', action='store_true',
  559. default=False,
  560. help=('Causes cbuildbot to bootstrap itself twice, '
  561. 'in the sequence A->B->C: A(unpatched) patches '
  562. 'and bootstraps B; B patches and bootstraps C'))
  563. group.add_remote_option('--validation_pool', default=None,
  564. help=('Path to a pickled validation pool. Intended '
  565. 'for use only with the commit queue.'))
  566. group.add_remote_option('--metadata_dump', default=None,
  567. help=('Path to a json dumped metadata file. This '
  568. 'will be used as the initial metadata.'))
  569. group.add_remote_option('--master-build-id', default=None, type=int,
  570. api=constants.REEXEC_API_MASTER_BUILD_ID,
  571. help=('cidb build id of the master build to this '
  572. 'slave build.'))
  573. group.add_remote_option('--mock-tree-status', dest='mock_tree_status',
  574. default=None, action='store',
  575. help=('Override the tree status value that would be '
  576. 'returned from the the actual tree. Example '
  577. 'values: open, closed, throttled. When used '
  578. 'in conjunction with --debug, the tree status '
  579. 'will not be ignored as it usually is in a '
  580. '--debug run.'))
  581. group.add_remote_option(
  582. '--mock-slave-status', dest='mock_slave_status', default=None,
  583. action='store', metavar='MOCK_SLAVE_STATUS_PICKLE_FILE',
  584. help=('Override the result of the _FetchSlaveStatuses method of '
  585. 'MasterSlaveSyncCompletionStage, by specifying a file with a '
  586. 'pickle of the result to be returned.'))
  587. parser.add_option_group(group)
  588. #
  589. # Debug options
  590. #
  591. # Temporary hack; in place till --dry-run replaces --debug.
  592. # pylint: disable=W0212
  593. group = parser.debug_group
  594. debug = [x for x in group.option_list if x._long_opts == ['--debug']][0]
  595. debug.help += ' Currently functions as --dry-run in addition.'
  596. debug.pass_through = True
  597. group.add_option('--notee', action='store_false', dest='tee', default=True,
  598. help=('Disable logging and internal tee process. Primarily '
  599. 'used for debugging cbuildbot itself.'))
  600. return parser
  601. def _FinishParsing(options, args):
  602. """Perform some parsing tasks that need to take place after optparse.
  603. This function needs to be easily testable! Keep it free of
  604. environment-dependent code. Put more detailed usage validation in
  605. _PostParseCheck().
  606. Args:
  607. options: The options object returned by optparse
  608. args: The args object returned by optparse
  609. """
  610. # Populate options.pass_through_args.
  611. accepted, _ = commandline.FilteringParser.FilterArgs(
  612. options.parsed_args, lambda x: x.opt_inst.pass_through)
  613. options.pass_through_args.extend(accepted)
  614. if options.chrome_root:
  615. if options.chrome_rev != constants.CHROME_REV_LOCAL:
  616. cros_build_lib.Die('Chrome rev must be %s if chrome_root is set.' %
  617. constants.CHROME_REV_LOCAL)
  618. elif options.chrome_rev == constants.CHROME_REV_LOCAL:
  619. cros_build_lib.Die('Chrome root must be set if chrome_rev is %s.' %
  620. constants.CHROME_REV_LOCAL)
  621. if options.chrome_version:
  622. if options.chrome_rev != constants.CHROME_REV_SPEC:
  623. cros_build_lib.Die('Chrome rev must be %s if chrome_version is set.' %
  624. constants.CHROME_REV_SPEC)
  625. elif options.chrome_rev == constants.CHROME_REV_SPEC:
  626. cros_build_lib.Die(
  627. 'Chrome rev must not be %s if chrome_version is not set.'
  628. % constants.CHROME_REV_SPEC)
  629. patches = bool(options.gerrit_patches or options.local_patches or
  630. options.rietveld_patches)
  631. if options.remote:
  632. if options.local:
  633. cros_build_lib.Die('Cannot specify both --remote and --local')
  634. # options.channels is a convenient way to detect payloads builds.
  635. if (not options.list and not options.buildbot and not options.channels and
  636. not patches):
  637. prompt = ('No patches were provided; are you sure you want to just '
  638. 'run a remote build of %s?' % (
  639. options.branch if options.branch else 'ToT'))
  640. if not cros_build_lib.BooleanPrompt(prompt=prompt, default=False):
  641. cros_build_lib.Die('Must provide patches when running with --remote.')
  642. # --debug needs to be explicitly passed through for remote invocations.
  643. release_mode_with_patches = (options.buildbot and patches and
  644. '--debug' not in options.pass_through_args)
  645. else:
  646. if len(args) > 1:
  647. cros_build_lib.Die('Multiple configs not supported if not running with '
  648. '--remote. Got %r', args)
  649. if options.slaves:
  650. cros_build_lib.Die('Cannot use --slaves if not running with --remote.')
  651. release_mode_with_patches = (options.buildbot and patches and
  652. not options.debug)
  653. # When running in release mode, make sure we are running with checked-in code.
  654. # We want checked-in cbuildbot/scripts to prevent errors, and we want to build
  655. # a release image with checked-in code for CrOS packages.
  656. if release_mode_with_patches:
  657. cros_build_lib.Die(
  658. 'Cannot provide patches when running with --buildbot!')
  659. if options.buildbot and options.remote_trybot:
  660. cros_build_lib.Die(
  661. '--buildbot and --remote-trybot cannot be used together.')
  662. # Record whether --debug was set explicitly vs. it was inferred.
  663. options.debug_forced = False
  664. if options.debug:
  665. options.debug_forced = True
  666. if not options.debug:
  667. # We don't set debug by default for
  668. # 1. --buildbot invocations.
  669. # 2. --remote invocations, because it needs to push changes to the tryjob
  670. # repo.
  671. options.debug = not options.buildbot and not options.remote
  672. # Record the configs targeted.
  673. options.build_targets = args[:]
  674. if constants.BRANCH_UTIL_CONFIG in options.build_targets:
  675. if options.remote:
  676. cros_build_lib.Die(
  677. 'Running %s as a remote tryjob is not yet supported.',
  678. constants.BRANCH_UTIL_CONFIG)
  679. if len(options.build_targets) > 1:
  680. cros_build_lib.Die(
  681. 'Cannot run %s with any other configs.',
  682. constants.BRANCH_UTIL_CONFIG)
  683. if not options.branch_name:
  684. cros_build_lib.Die(
  685. 'Must specify --branch-name with the %s config.',
  686. constants.BRANCH_UTIL_CONFIG)
  687. if options.branch and options.branch != options.branch_name:
  688. cros_build_lib.Die(
  689. 'If --branch is specified with the %s config, it must'
  690. ' have the same value as --branch-name.',
  691. constants.BRANCH_UTIL_CONFIG)
  692. exclusive_opts = {'--version': options.force_version,
  693. '--delete-branch': options.delete_branch,
  694. '--rename-to': options.rename_to}
  695. if 1 != sum(1 for x in exclusive_opts.values() if x):
  696. cros_build_lib.Die('When using the %s config, you must'
  697. ' specifiy one and only one of the following'
  698. ' options: %s.', constants.BRANCH_UTIL_CONFIG,
  699. ', '.join(exclusive_opts.keys()))
  700. # When deleting or renaming a branch, the --branch and --nobootstrap
  701. # options are implied.
  702. if options.delete_branch or options.rename_to:
  703. if not options.branch:
  704. logging.info('Automatically enabling sync to branch %s for this %s '
  705. 'flow.', options.branch_name,
  706. constants.BRANCH_UTIL_CONFIG)
  707. options.branch = options.branch_name
  708. if options.bootstrap:
  709. logging.info('Automatically disabling bootstrap step for this %s flow.',
  710. constants.BRANCH_UTIL_CONFIG)
  711. options.bootstrap = False
  712. elif any([options.delete_branch, options.rename_to, options.branch_name]):
  713. cros_build_lib.Die(
  714. 'Cannot specify --delete-branch, --rename-to or --branch-name when not '
  715. 'running the %s config', constants.BRANCH_UTIL_CONFIG)
  716. # pylint: disable=W0613
  717. def _PostParseCheck(parser, options, args, site_config):
  718. """Perform some usage validation after we've parsed the arguments
  719. Args:
  720. parser: Option parser that was used to parse arguments.
  721. options: The options returned by optparse.
  722. args: The args returned by optparse.
  723. site_config: config_lib.SiteConfig containing all config info.
  724. """
  725. if not args:
  726. parser.error('Invalid usage: no configuration targets provided.'
  727. 'Use -h to see usage. Use -l to list supported configs.')
  728. if not options.branch:
  729. options.branch = git.GetChromiteTrackingBranch()
  730. if not repository.IsARepoRoot(options.sourceroot):
  731. if options.local_patches:
  732. raise Exception('Could not find repo checkout at %s!'
  733. % options.sourceroot)
  734. # Because the default cache dir depends on other options, FindCacheDir
  735. # always returns None, and we setup the default here.
  736. if options.cache_dir is None:
  737. # Note, options.sourceroot is set regardless of the path
  738. # actually existing.
  739. if options.buildroot is not None:
  740. options.cache_dir = os.path.join(options.buildroot, '.cache')
  741. elif os.path.exists(options.sourceroot):
  742. options.cache_dir = os.path.join(options.sourceroot, '.cache')
  743. else:
  744. options.cache_dir = parser.FindCacheDir(parser, options)
  745. options.cache_dir = os.path.abspath(options.cache_dir)
  746. parser.ConfigureCacheDir(options.cache_dir)
  747. osutils.SafeMakedirsNonRoot(options.cache_dir)
  748. if options.local_patches:
  749. options.local_patches = _CheckLocalPatches(
  750. options.sourceroot, options.local_patches)
  751. default = os.environ.get('CBUILDBOT_DEFAULT_MODE')
  752. if (default and not any([options.local, options.buildbot,
  753. options.remote, options.remote_trybot])):
  754. logging.info('CBUILDBOT_DEFAULT_MODE=%s env var detected, using it.'
  755. % default)
  756. default = default.lower()
  757. if default == 'local':
  758. options.local = True
  759. elif default == 'remote':
  760. options.remote = True
  761. elif default == 'buildbot':
  762. options.buildbot = True
  763. else:
  764. cros_build_lib.Die("CBUILDBOT_DEFAULT_MODE value %s isn't supported. "
  765. % default)
  766. # Ensure that all args are legitimate config targets.
  767. invalid_targets = []
  768. for arg in args:
  769. if arg not in site_config:
  770. invalid_targets.append(arg)
  771. logging.error('No such configuraton target: "%s".', arg)
  772. continue
  773. build_config = site_config[arg]
  774. is_payloads_build = build_config.build_type == constants.PAYLOADS_TYPE
  775. if options.channels and not is_payloads_build:
  776. cros_build_lib.Die('--channel must only be used with a payload config,'
  777. ' not target (%s).' % arg)
  778. if not options.channels and is_payloads_build:
  779. cros_build_lib.Die('payload configs (%s) require --channel to do anything'
  780. ' useful.' % arg)
  781. # The --version option is not compatible with an external target unless the
  782. # --buildbot option is specified. More correctly, only "paladin versions"
  783. # will work with external targets, and those are only used with --buildbot.
  784. # If --buildbot is specified, then user should know what they are doing and
  785. # only specify a version that will work. See crbug.com/311648.
  786. if (options.force_version and
  787. not (options.buildbot or build_config.internal)):
  788. cros_build_lib.Die('Cannot specify --version without --buildbot for an'
  789. ' external target (%s).' % arg)
  790. if invalid_targets:
  791. cros_build_lib.Die('One or more invalid configuration targets specified. '
  792. 'You can check the available configs by running '
  793. '`cbuildbot --list --all`')
  794. def _ParseCommandLine(parser, argv):
  795. """Completely parse the commandline arguments"""
  796. (options, args) = parser.parse_args(argv)
  797. # Strip out null arguments.
  798. # TODO(rcui): Remove when buildbot is fixed
  799. args = [arg for arg in args if arg]
  800. if options.output_api_version:
  801. print(constants.REEXEC_API_VERSION)
  802. sys.exit(0)
  803. _FinishParsing(options, args)
  804. return options, args
  805. _ENVIRONMENT_PROD = 'prod'
  806. _ENVIRONMENT_DEBUG = 'debug'
  807. _ENVIRONMENT_STANDALONE = 'standalone'
  808. def _GetRunEnvironment(options, build_config):
  809. """Determine whether this is a prod/debug/standalone run."""
  810. # TODO(akeshet): This is a temporary workaround to make sure that the cidb
  811. # is not used on waterfalls that the db schema does not support (in particular
  812. # the chromeos.chrome waterfall).
  813. # See crbug.com/406940
  814. waterfall = os.environ.get('BUILDBOT_MASTERNAME', '')
  815. if not waterfall in constants.CIDB_KNOWN_WATERFALLS:
  816. return _ENVIRONMENT_STANDALONE
  817. # TODO(akeshet): Clean up this code once we have better defined flags to
  818. # specify on-or-off waterfall and on-or-off production runs of cbuildbot.
  819. # See crbug.com/331417
  820. # --buildbot runs should use the production services, unless the --debug flag
  821. # is also present.
  822. if options.buildbot:
  823. if options.debug:
  824. return _ENVIRONMENT_DEBUG
  825. else:
  826. return _ENVIRONMENT_PROD
  827. # --remote-trybot runs should use the debug services, with the exception of
  828. # pre-cq builds, which should use the production services.
  829. if options.remote_trybot:
  830. if build_config['pre_cq']:
  831. return _ENVIRONMENT_PROD
  832. else:
  833. return _ENVIRONMENT_DEBUG
  834. # If neither --buildbot nor --remote-trybot flag was used, don't use external
  835. # services.
  836. return _ENVIRONMENT_STANDALONE
  837. def _SetupConnections(options, build_config):
  838. """Set up CIDB and graphite connections using the appropriate Setup call.
  839. Args:
  840. options: Command line options structure.
  841. build_config: Config object for this build.
  842. """
  843. # Outline:
  844. # 1) Based on options and build_config, decide whether we are a production
  845. # run, debug run, or standalone run.
  846. # 2) Set up cidb instance accordingly.
  847. # 3) Update topology info from cidb, so that any other service set up can use
  848. # topology.
  849. # 4) Set up any other services.
  850. run_type = _GetRunEnvironment(options, build_config)
  851. if run_type == _ENVIRONMENT_PROD:
  852. cidb.CIDBConnectionFactory.SetupProdCidb()
  853. elif run_type == _ENVIRONMENT_DEBUG:
  854. cidb.CIDBConnectionFactory.SetupDebugCidb()
  855. else:
  856. cidb.CIDBConnectionFactory.SetupNoCidb()
  857. db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
  858. topology.FetchTopologyFromCIDB(db)
  859. if run_type == _ENVIRONMENT_PROD:
  860. graphite.ESMetadataFactory.SetupProd()
  861. graphite.StatsFactory.SetupProd()
  862. elif run_type == _ENVIRONMENT_DEBUG:
  863. graphite.ESMetadataFactory.SetupReadOnly()
  864. graphite.StatsFactory.SetupDebug()
  865. else:
  866. graphite.ESMetadataFactory.SetupReadOnly()
  867. graphite.StatsFactory.SetupMock()
  868. def _FetchInitialBootstrapConfigRepo(repo_url, branch_name):
  869. """Fetch the TOT site config repo, if necessary to start bootstrap."""
  870. # If we are part of a repo checkout, the manifest stages control things.
  871. if git.FindRepoDir(constants.SOURCE_ROOT):
  872. return
  873. # We must be part of a bootstrap chromite checkout, probably on a buildbot.
  874. # If the config directory already exists, we have started the bootstrap
  875. # process. Assume the boostrap stage did the right thing, and leave the config
  876. # directory alone.
  877. if os.path.exists(constants.SITE_CONFIG_DIR):
  878. return
  879. # We are part of a clean chromite checkout (buildbot always cleans chromite
  880. # before launching us), so create the initial site config checkout.
  881. logging.info('Fetching Config Repo: %s', repo_url)
  882. git.Clone(constants.SITE_CONFIG_DIR, repo_url)
  883. if branch_name:
  884. git.RunGit(constants.SITE_CONFIG_DIR, ['checkout', branch_name])
  885. # TODO(build): This function is too damn long.
  886. def main(argv):
  887. # Turn on strict sudo checks.
  888. cros_build_lib.STRICT_SUDO = True
  889. # Set umask to 022 so files created by buildbot are readable.
  890. os.umask(0o22)
  891. parser = _CreateParser()
  892. options, args = _ParseCommandLine(parser, argv)
  893. if options.buildbot and options.config_repo:
  894. _FetchInitialBootstrapConfigRepo(options.config_repo, options.branch)
  895. if options.config_repo:
  896. # Ensure expected config file is present.
  897. if not os.path.exists(constants.SITE_CONFIG_FILE):
  898. cros_build_lib.Die('Unabled to find: %s', constants.SITE_CONFIG_FILE)
  899. # Fetch our site_config now, because we need it to do anything else.
  900. site_config = config_lib.GetConfig()
  901. if options.list:
  902. _PrintValidConfigs(site_config, options.print_all)
  903. sys.exit(0)
  904. _PostParseCheck(parser, options, args, site_config)
  905. cros_build_lib.AssertOutsideChroot()
  906. if options.enable_buildbot_tags:
  907. logging.EnableBuildbotMarkers()
  908. if options.remote:
  909. logging.getLogger().setLevel(logging.WARNING)
  910. # Verify configs are valid.
  911. # If hwtest flag is enabled, post a warning that HWTest step may fail if the
  912. # specified board is not a released platform or it is a generic overlay.
  913. for bot in args:
  914. build_config = site_config[bot]
  915. if options.hwtest:
  916. logging.warning(
  917. 'If %s is not a released platform or it is a generic overlay, '
  918. 'the HWTest step will most likely not run; please ask the lab '
  919. 'team for help if this is unexpected.' % build_config['boards'])
  920. # Verify gerrit patches are valid.
  921. print('Verifying patches...')
  922. patch_pool = trybot_patch_pool.TrybotPatchPool.FromOptions(
  923. gerrit_patches=options.gerrit_patches,
  924. local_patches=options.local_patches,
  925. sourceroot=options.sourceroot,
  926. remote_patches=options.remote_patches)
  927. # --debug need to be explicitly passed through for remote invocations.
  928. if options.buildbot and '--debug' not in options.pass_through_args:
  929. _ConfirmRemoteBuildbotRun()
  930. print('Submitting tryjob...')
  931. tryjob = remote_try.RemoteTryJob(options, args, patch_pool.local_patches)
  932. tryjob.Submit(testjob=options.test_tryjob, dryrun=False)
  933. print('Tryjob submitted!')
  934. print(('Go to %s to view the status of your job.'
  935. % tryjob.GetTrybotWaterfallLink()))
  936. sys.exit(0)
  937. elif (not options.buildbot and not options.remote_trybot
  938. and not options.resume and not options.local):
  939. cros_build_lib.Die('Please use --remote or --local to run trybots')
  940. # Only one config arg is allowed in this mode, which was confirmed earlier.
  941. bot_id = args[-1]
  942. build_config = site_config[bot_id]
  943. # TODO: Re-enable this block when reference_repo support handles this
  944. # properly. (see chromium:330775)
  945. # if options.reference_repo is None:
  946. # repo_path = os.path.join(options.sourceroot, '.repo')
  947. # # If we're being run from a repo checkout, reuse the repo's git pool to
  948. # # cut down on sync time.
  949. # if os.path.exists(repo_path):
  950. # options.reference_repo = options.sourceroot
  951. if options.reference_repo:
  952. if not os.path.exists(options.reference_repo):
  953. parser.error('Reference path %s does not exist'
  954. % (options.reference_repo,))
  955. elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
  956. parser.error('Reference path %s does not look to be the base of a '
  957. 'repo checkout; no .repo exists in the root.'
  958. % (options.reference_repo,))
  959. if (options.buildbot or options.remote_trybot) and not options.resume:
  960. if not options.cgroups:
  961. parser.error('Options --buildbot/--remote-trybot and --nocgroups cannot '
  962. 'be used together. Cgroup support is required for '
  963. 'buildbot/remote-trybot mode.')
  964. if not cgroups.Cgroup.IsSupported():
  965. parser.error('Option --buildbot/--remote-trybot was given, but this '
  966. 'system does not support cgroups. Failing.')
  967. missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
  968. if missing:
  969. parser.error('Option --buildbot/--remote-trybot requires the following '
  970. "binaries which couldn't be found in $PATH: %s"
  971. % (', '.join(missing)))
  972. if options.reference_repo:
  973. options.reference_repo = os.path.abspath(options.reference_repo)
  974. if not options.buildroot:
  975. if options.buildbot:
  976. parser.error('Please specify a buildroot with the --buildroot option.')
  977. options.buildroot = _DetermineDefaultBuildRoot(options.sourceroot,
  978. build_config['internal'])
  979. # We use a marker file in the buildroot to indicate the user has
  980. # consented to using this directory.
  981. if not os.path.exists(repository.GetTrybotMarkerPath(options.buildroot)):
  982. _ConfirmBuildRoot(options.buildroot)
  983. # Sanity check of buildroot- specifically that it's not pointing into the
  984. # midst of an existing repo since git-repo doesn't support nesting.
  985. if (not repository.IsARepoRoot(options.buildroot) and
  986. git.FindRepoDir(options.buildroot)):
  987. parser.error('Configured buildroot %s points into a repository checkout, '
  988. 'rather than the root of it. This is not supported.'
  989. % options.buildroot)
  990. if not options.log_dir:
  991. options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
  992. log_file = None
  993. if options.tee:
  994. log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
  995. osutils.SafeMakedirs(options.log_dir)
  996. _BackupPreviousLog(log_file)
  997. with cros_build_lib.ContextManagerStack() as stack:
  998. # TODO(ferringb): update this once
  999. # https://chromium-review.googlesource.com/25359
  1000. # is landed- it's sensitive to the manifest-versions cache path.
  1001. options.preserve_paths = set(['manifest-versions', '.cache',
  1002. 'manifest-versions-internal'])
  1003. if log_file is not None:
  1004. # We don't want the critical section to try to clean up the tee process,
  1005. # so we run Tee (forked off) outside of it. This prevents a deadlock
  1006. # because the Tee process only exits when its pipe is closed, and the
  1007. # critical section accidentally holds on to that file handle.
  1008. stack.Add(tee.Tee, log_file)
  1009. options.preserve_paths.add(_DEFAULT_LOG_DIR)
  1010. critical_section = stack.Add(cleanup.EnforcedCleanupSection)
  1011. stack.Add(sudo.SudoKeepAlive)
  1012. if not options.resume:
  1013. # If we're in resume mode, use our parents tempdir rather than
  1014. # nesting another layer.
  1015. stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
  1016. logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))
  1017. if options.cgroups:
  1018. stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
  1019. # Mark everything between EnforcedCleanupSection and here as having to
  1020. # be rolled back via the contextmanager cleanup handlers. This
  1021. # ensures that sudo bits cannot outlive cbuildbot, that anything
  1022. # cgroups would kill gets killed, etc.
  1023. stack.Add(critical_section.ForkWatchdog)
  1024. if not options.buildbot:
  1025. build_config = config_lib.OverrideConfigForTrybot(
  1026. build_config, options)
  1027. if options.mock_tree_status is not None:
  1028. stack.Add(mock.patch.object, tree_status, '_GetStatus',
  1029. return_value=options.mock_tree_status)
  1030. if options.mock_slave_status is not None:
  1031. with open(options.mock_slave_status, 'r') as f:
  1032. mock_statuses = pickle.load(f)
  1033. for key, value in mock_statuses.iteritems():
  1034. mock_statuses[key] = manifest_version.BuilderStatus(**value)
  1035. stack.Add(mock.patch.object,
  1036. completion_stages.MasterSlaveSyncCompletionStage,
  1037. '_FetchSlaveStatuses',
  1038. return_value=mock_statuses)
  1039. _SetupConnections(options, build_config)
  1040. retry_stats.SetupStats()
  1041. # For master-slave builds: Update slave's timeout using master's published
  1042. # deadline.
  1043. if options.buildbot and options.master_build_id is not None:
  1044. slave_timeout = None
  1045. if cidb.CIDBConnectionFactory.IsCIDBSetup():
  1046. cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
  1047. if cidb_handle:
  1048. slave_timeout = cidb_handle.GetTimeToDeadline(options.master_build_id)
  1049. if slave_timeout is not None:
  1050. # Cut me some slack. We artificially add a a small time here to the
  1051. # slave_timeout because '0' is handled specially, and because we don't
  1052. # want to timeout while trying to set things up.
  1053. slave_timeout = slave_timeout + 20
  1054. if options.timeout == 0 or slave_timeout < options.timeout:
  1055. logging.info('Updating slave build timeout to %d seconds enforced '
  1056. 'by the master', slave_timeout)
  1057. options.timeout = slave_timeout
  1058. else:
  1059. logging.warning('Could not get master deadline for master-slave build. '
  1060. 'Can not set slave timeout.')
  1061. if options.timeout > 0:
  1062. stack.Add(timeout_util.FatalTimeout, options.timeout)
  1063. _RunBuildStagesWrapper(options, site_config, build_config)