| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """A subclass of commands.SVN that allows more flexible error recovery. |
| |
| This code is only used on the slave but it is living in common/ because it is |
| directly imported from buildbot/slave/bot.py.""" |
| |
| import os |
| import re |
| |
| from twisted.python import log |
| from twisted.internet import defer |
| |
| from common import chromium_utils |
| |
| from buildslave.commands.base import SourceBaseCommand |
| from buildslave.commands.registry import commandRegistry |
| from buildslave import runprocess |
| |
| |
| PROJECTS_LOOKING_FOR = { |
| os.path.join('src'): 'got_chromium_revision', |
| os.path.join('src', 'native_client'): 'got_nacl_revision', |
| os.path.join('src', 'tools', 'swarming_client'): |
| 'got_swarming_client_revision', |
| os.path.join('src', 'v8'): 'got_v8_revision', |
| os.path.join('src', 'third_party', 'WebKit'): 'got_webkit_revision', |
| os.path.join('src', 'third_party', 'webrtc'): 'got_webrtc_revision', |
| } |
| |
| |
| # Local errors. |
| class InvalidPath(Exception): pass |
| |
| |
| def FixDiffLineEnding(diff): |
| """Fix patch files generated on windows and applied on mac/linux. |
| |
| For files with svn:eol-style=crlf, svn diff puts CRLF in the diff hunk header. |
| patch on linux and mac barfs on those hunks. As usual, blame svn.""" |
| output = '' |
| for line in diff.splitlines(True): |
| if (line.startswith('---') or line.startswith('+++') or |
| line.startswith('@@ ') or line.startswith('\\ No')): |
| # Strip any existing CRLF on the header lines |
| output += line.rstrip() + '\n' |
| else: |
| output += line |
| return output |
| |
| |
| def untangle(stdout_lines): |
| """Sort the lines of stdout by the job number prefix.""" |
| out = [] |
| tasks = {} |
| UNTANGLE_RE = re.compile(r'^(\d+)>(.*)$') |
| for line in stdout_lines: |
| m = UNTANGLE_RE.match(line) |
| if not m: |
| if line: # skip empty lines |
| # Output untangled lines so far. |
| for key in sorted(tasks.iterkeys()): |
| out.extend(tasks[key]) |
| tasks = {} |
| out.append(line) |
| else: |
| if m.group(2): # skip empty lines |
| tasks.setdefault(int(m.group(1)), []).append(m.group(2)) |
| for key in sorted(tasks.iterkeys()): |
| out.extend(tasks[key]) |
| return out |
| |
| |
| def extract_revisions(output): |
| """Extracts revision numbers for all the dependencies checked out. |
| |
| svn checkout operations finish with 'Checked out revision 16657.' |
| svn update operations finish the line 'At revision 16654.' when there is no |
| change. They finish with 'Updated to revision 16655.' otherwise. |
| |
| The first project checked out gets to be set to got_revision property. This is |
| important since this class is not only used for chromium, it's also used for |
| other projects. |
| """ |
| SCM_RE = { |
| 'git': r'^Checked out revision ([0-9a-fA-F]{40})$', |
| 'svn': r'^(?:Checked out|At|Updated to) revision ([0-9]+)\.', |
| } |
| GCLIENT_CMD = re.compile(r'^________ running \'(.+?)\' in \'(.+?)\'$') |
| GCLIENT_AT_REV = re.compile(r'_____ (.+?) at (\d+|[0-9a-fA-F]{40})$') |
| |
| untangled_stdout = untangle(output.splitlines(False)) |
| # We only care about the last sync which starts with "solutions=[...". |
| # Look backwards to find the last gclient sync call. It's parsing the whole |
| # sync step in one pass here, which could include gclient revert and two |
| # gclient sync calls. Only the last gclient sync call should be parsed. |
| for i, line in enumerate(reversed(untangled_stdout)): |
| if line.startswith('solutions=['): |
| # Only keep from "solutions" line to end. |
| untangled_stdout = untangled_stdout[-i-1:] |
| break |
| |
| revisions_found = {} |
| current_scm = None |
| current_project = None |
| reldir = None |
| for line in untangled_stdout: |
| match = GCLIENT_AT_REV.match(line) |
| if match: |
| current_project = None |
| current_scm = None |
| log.msg('gclient: %s == %s' % match.groups()) |
| project = PROJECTS_LOOKING_FOR.get(match.group(1)) |
| if not revisions_found: |
| # Set it to got_revision, independent if it's a chromium-specific |
| # thing. |
| revisions_found['got_revision'] = match.group(2) |
| if project: |
| revisions_found[project] = match.group(2) |
| continue |
| |
| if current_scm: |
| # Look for revision. |
| match = re.match(SCM_RE[current_scm], line) |
| if match: |
| # Override any previous value, since update can happen multiple times. |
| log.msg( |
| 'scm: %s (%s) == %s' % (reldir, current_project, match.group(1))) |
| if not revisions_found: |
| revisions_found['got_revision'] = match.group(1) |
| if current_project: |
| revisions_found[current_project] = match.group(1) |
| current_project = None |
| current_scm = None |
| continue |
| |
| match = GCLIENT_CMD.match(line) |
| if match: |
| command, directory = match.groups() |
| parts = command.split(' ') |
| directory = directory.rstrip(os.path.sep) + os.path.sep |
| if parts[0] in SCM_RE: |
| current_scm = parts[0] |
| for part in parts: |
| # This code assumes absolute paths, which are easy to find in the |
| # argument list. |
| if part.startswith(directory): |
| reldir = part[len(directory):] |
| if reldir: |
| current_project = PROJECTS_LOOKING_FOR.get(reldir) |
| break |
| |
| return revisions_found |
| |
| |
| def _RenameDirectoryCommand(src_dir, dest_dir): |
| """Returns a command list to rename a directory (or file) using Python.""" |
| # Use / instead of \ in paths to avoid issues with escaping. |
| return ['python', '-c', |
| 'import os; ' |
| 'os.rename("%s", "%s")' % |
| (src_dir.replace('\\', '/'), dest_dir.replace('\\', '/'))] |
| |
| |
| def _RemoveFileCommand(filename): |
| """Returns a command list to remove a directory (or file) using Python.""" |
| # Use / instead of \ in paths to avoid issues with escaping. |
| return ['python', '-c', |
| 'from common import chromium_utils; ' |
| 'chromium_utils.RemoveFile("%s")' % filename.replace('\\', '/')] |
| |
| |
| class GClient(SourceBaseCommand): |
| """Source class that handles gclient checkouts. |
| |
| Buildbot 8.3 changed from commands.SourceBase to SourceBaseCommand, |
| so this inherits from a variable set to the right item. Docs below |
| assume 8.3 (and this hack should be removed when pre-8.3 clients are |
| turned down. |
| |
| In addition to the arguments handled by SourceBaseCommand, this command |
| reads the following keys: |
| |
| ['gclient_spec']: |
| if not None, then this specifies the text of the .gclient file to use. |
| this overrides any 'svnurl' argument that may also be specified. |
| |
| ['rm_timeout']: |
| if not None, a different timeout used only for the 'rm -rf' operation in |
| doClobber. Otherwise the svn timeout will be used for that operation too. |
| |
| ['svnurl']: |
| if not None, then this specifies the svn url to pass to 'gclient config' |
| to create a .gclient file. |
| |
| ['branch']: |
| if not None, then this specifies the module name to pass to 'gclient sync' |
| in --revision argument. |
| |
| ['project']: |
| if not None, then this specifies the module name to pass to 'gclient sync' |
| in --revision argument. This value overloads 'branch', and is mostly useful |
| for git checkouts. See also 'no_gclient_branch'. |
| |
| ['env']: |
| Augment os.environ. |
| |
| ['no_gclient_branch']: |
| If --revision is specified, don't prepend it with <branch>@. This is |
| necessary for git, where the solution name is 'src' and the branch name |
| is 'master'. Use the project attribute if there are several solution in |
| the .gclient file. |
| |
| ['no_gclient_revision']: |
| Do not specify the --revision argument to gclient at all. |
| """ |
| |
| # pylint complains that __init__ in GClient's parent isn't called. This is |
| # because it doesn't realize we call it through GetParentClass(). Disable |
| # that warning next. |
| # pylint: disable=W0231 |
| |
| header = 'gclient' |
| |
| def __init__(self, *args, **kwargs): |
| # TODO(maruel): Mainly to keep pylint happy, remove once buildbot fixed |
| # their style. |
| self.branch = None |
| self.srcdir = None |
| self.revision = None |
| self.env = None |
| self.sourcedatafile = None |
| self.patch = None |
| self.command = None |
| self.vcexe = None |
| self.svnurl = None |
| self.sudo_for_remove = None |
| self.gclient_spec = None |
| self.gclient_deps = None |
| self.rm_timeout = None |
| self.gclient_nohooks = False |
| self.was_patched = False |
| self.no_gclient_branch = False |
| self.no_gclient_revision = False |
| self.delete_unversioned_trees_when_updating = True |
| self.gclient_jobs = None |
| self.project = None |
| # TODO(maruel): Remove once buildbot 0.8.4p1 conversion is complete. |
| self.sourcedata = None |
| self.do_nothing = None |
| chromium_utils.GetParentClass(GClient).__init__(self, *args, **kwargs) |
| |
| def setup(self, args): |
| """Our implementation of command.Commands.setup() method. |
| The method will get all the arguments that are passed to remote command |
| and is invoked before start() method (that will in turn call doVCUpdate()). |
| """ |
| SourceBaseCommand.setup(self, args) |
| self.vcexe = self.getCommand('gclient') |
| self.svnurl = args['svnurl'] |
| self.branch = args.get('branch') |
| self.revision = args.get('revision') |
| self.patch = args.get('patch') |
| self.sudo_for_remove = args.get('sudo_for_remove') |
| self.gclient_spec = args['gclient_spec'] |
| self.gclient_deps = args.get('gclient_deps') |
| self.sourcedata = '%s\n' % self.svnurl |
| self.rm_timeout = args.get('rm_timeout', self.timeout) |
| self.env = args.get('env') |
| self.gclient_nohooks = args.get('gclient_nohooks', False) |
| self.env['CHROMIUM_GYP_SYNTAX_CHECK'] = '1' |
| self.no_gclient_branch = args.get('no_gclient_branch') |
| self.no_gclient_revision = args.get('no_gclient_revision', False) |
| self.gclient_jobs = args.get('gclient_jobs') |
| self.project = args.get('project', None) |
| |
| def start(self): |
| """Start the update process. |
| |
| start() is cut-and-paste from the base class, the block calling |
| self.sourcedirIsPatched() and the revert support is the only functional |
| difference from base.""" |
| self.sendStatus({'header': "starting " + self.header + "\n"}) |
| self.command = None |
| |
| # self.srcdir is where the VC system should put the sources |
| if self.mode == "copy": |
| self.srcdir = "source" # hardwired directory name, sorry |
| else: |
| self.srcdir = self.workdir |
| self.sourcedatafile = os.path.join(self.builder.basedir, |
| self.srcdir, |
| ".buildbot-sourcedata") |
| self.do_nothing = os.path.isfile(os.path.join(self.builder.basedir, |
| self.srcdir, |
| 'update.flag')) |
| |
| d = defer.succeed(0) |
| |
| if self.do_nothing: |
| # If bot update is run, we don't need to run the traditional update step. |
| msg = 'update.flag file found: bot_update has run and checkout is \n' |
| msg += 'already in a consistent state.\n' |
| msg += 'No actions will be performed in this step.' |
| self.sendStatus({'header': msg}) |
| d.addCallback(self._sendRC) |
| return d |
| |
| # Do we need to clobber anything? |
| if self.mode in ("copy", "clobber", "export"): |
| d.addCallback(self.doClobber, self.workdir) |
| if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()): |
| # the directory cannot be updated, so we have to clobber it. |
| # Perhaps the master just changed modes from 'export' to |
| # 'update'. |
| d.addCallback(self.doClobber, self.srcdir) |
| elif self.sourcedirIsPatched(): |
| # The directory is patched. Revert the sources. |
| d.addCallback(self.doRevert) |
| self.was_patched = True |
| |
| d.addCallback(self.doVC) |
| |
| if self.mode == "copy": |
| d.addCallback(self.doCopy) |
| if self.patch: |
| d.addCallback(self.doPatch) |
| |
| # Only after the patch call the actual code to get the revision numbers. |
| d.addCallback(self._handleGotRevision) |
| |
| if (self.patch or self.was_patched) and not self.gclient_nohooks: |
| # Always run doRunHooks if there *is* or there *was* a patch because |
| # revert is run with --nohooks and `gclient sync` will not regenerate the |
| # output files if the input files weren't updated.. |
| d.addCallback(self.doRunHooks) |
| d.addCallbacks(self._sendRC, self._checkAbandoned) |
| return d |
| |
| def sourcedirIsPatched(self): |
| return os.path.exists(os.path.join(self.builder.basedir, |
| self.srcdir, '.buildbot-patched')) |
| |
| def sourcedirIsUpdateable(self): |
| # Patched directories are updatable. |
| return os.path.exists(os.path.join(self.builder.basedir, |
| self.srcdir, '.gclient')) |
| |
| # TODO(pamg): consolidate these with the copies above. |
| def _RemoveDirectoryCommand(self, rm_dir): |
| """Returns a command list to delete a directory using Python.""" |
| # Use / instead of \ in paths to avoid issues with escaping. |
| cmd = ['python', '-c', |
| 'from common import chromium_utils; ' |
| 'chromium_utils.RemoveDirectory("%s")' % rm_dir.replace('\\', '/')] |
| if self.sudo_for_remove: |
| cmd = ['sudo'] + cmd |
| return cmd |
| |
| def doGclientUpdate(self): |
| """Sync the client |
| """ |
| dirname = os.path.join(self.builder.basedir, self.srcdir) |
| command = [chromium_utils.GetGClientCommand(), |
| 'sync', '--verbose', '--reset', '--manually_grab_svn_rev', |
| '--force', '--with_branch_heads'] |
| if self.delete_unversioned_trees_when_updating: |
| command.append('--delete_unversioned_trees') |
| if self.gclient_jobs: |
| command.append('-j%d' % self.gclient_jobs) |
| # Don't run hooks if it was patched or there is a patch since runhooks will |
| # be run after. |
| if self.gclient_nohooks or self.patch or self.was_patched: |
| command.append('--nohooks') |
| # GClient accepts --revision argument of two types 'module@rev' and 'rev'. |
| if self.revision and not self.no_gclient_revision: |
| command.append('--revision') |
| # Ignore non-svn part of compound revisions. |
| if ':' in self.revision: |
| command.append(self.revision.split(':')[0]) |
| elif (not self.branch or |
| self.no_gclient_branch or |
| '@' in str(self.revision)): |
| command.append(str(self.revision)) |
| else: |
| # Make the revision look like branch@revision. |
| prefix = self.project if self.project else self.branch |
| command.append('%s@%s' % (prefix, self.revision)) |
| |
| if self.gclient_deps: |
| command.append('--deps=' + self.gclient_deps) |
| |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| keepStdout=True, environ=self.env) |
| self.command = c |
| return c.start() |
| |
| def getGclientConfigCommand(self): |
| """Return the command to run the gclient config step. |
| """ |
| dirname = os.path.join(self.builder.basedir, self.srcdir) |
| command = [chromium_utils.GetGClientCommand(), 'config'] |
| |
| if self.gclient_spec: |
| command.append('--spec=%s' % self.gclient_spec) |
| else: |
| command.append(self.svnurl) |
| |
| git_cache_dir = os.path.abspath( |
| os.path.join(self.builder.basedir, os.pardir, os.pardir, os.pardir, |
| 'git_cache')) |
| command.append('--cache-dir=' + git_cache_dir) |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| keepStdout=True, environ=self.env) |
| return c |
| |
| def doVCUpdate(self): |
| """Sync the client |
| """ |
| # Make sure the .gclient is updated. |
| os.remove(os.path.join(self.builder.basedir, self.srcdir, '.gclient')) |
| c = self.getGclientConfigCommand() |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| d.addCallback(lambda _: self.doGclientUpdate()) |
| return d |
| |
| def doVCFull(self): |
| """Setup the .gclient file and then sync |
| """ |
| chromium_utils.MaybeMakeDirectory(self.builder.basedir, self.srcdir) |
| |
| c = self.getGclientConfigCommand() |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| d.addCallback(lambda _: self.doGclientUpdate()) |
| return d |
| |
| def doClobber(self, dummy, dirname, _=False): |
| """Move the old directory aside, or delete it if that's already been done. |
| |
| This function is designed to be used with a source dir. If it's called |
| with anything else, the caller will need to be sure to clean up the |
| <dirname>.dead directory once it's no longer needed. |
| |
| If this is the first time we're clobbering since we last finished a |
| successful update or checkout, move the old directory aside so a human |
| can try to recover from it if desired. Otherwise -- if such a backup |
| directory already exists, because this isn't the first retry -- just |
| remove the old directory. |
| |
| Args: |
| dummy: unused |
| dirname: the directory within self.builder.basedir to be clobbered |
| """ |
| old_dir = os.path.join(self.builder.basedir, dirname) |
| dead_dir = old_dir + '.dead' |
| if os.path.isdir(old_dir): |
| if os.path.isdir(dead_dir): |
| command = self._RemoveDirectoryCommand(old_dir) |
| else: |
| command = _RenameDirectoryCommand(old_dir, dead_dir) |
| c = runprocess.RunProcess( |
| self.builder, command, self.builder.basedir, |
| sendRC=0, timeout=self.rm_timeout, |
| environ=self.env) |
| self.command = c |
| # See commands.SVN.doClobber for notes about sendRC. |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| return d |
| return None |
| |
| def doRevert(self, dummy): |
| """Revert any modification done by a previous patch. |
| |
| This is done in 2 parts to trap potential errors at each step. Note that |
| it is assumed that .orig and .rej files will be reverted, e.g. deleted by |
| the 'gclient revert' command. If the try bot is configured with |
| 'global-ignores=*.orig', patch failure will occur.""" |
| dirname = os.path.join(self.builder.basedir, self.srcdir) |
| command = [chromium_utils.GetGClientCommand(), 'revert', '--nohooks'] |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| keepStdout=True, environ=self.env) |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| # Remove patch residues. |
| d.addCallback(lambda _: self._doRevertRemoveSignalFile()) |
| return d |
| |
| def _doRevertRemoveSignalFile(self): |
| """Removes the file that signals that the checkout is patched. |
| |
| Must be called after a revert has been done and the patch residues have |
| been removed.""" |
| command = _RemoveFileCommand(os.path.join(self.builder.basedir, |
| self.srcdir, '.buildbot-patched')) |
| dirname = os.path.join(self.builder.basedir, self.srcdir) |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| keepStdout=True, environ=self.env) |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| return d |
| |
| def doPatch(self, res): |
| patchlevel = self.patch[0] |
| diff = FixDiffLineEnding(self.patch[1]) |
| |
| # Allow overwriting the root with an environment variable. |
| root = self.env.get("GCLIENT_PATCH_ROOT", None) |
| |
| if len(self.patch) >= 3 and root is None: |
| root = self.patch[2] |
| command = [ |
| self.getCommand("patch"), |
| '-p%d' % patchlevel, |
| '--remove-empty-files', |
| '--force', |
| '--forward', |
| ] |
| dirname = os.path.join(self.builder.basedir, self.workdir) |
| # Mark the directory so we don't try to update it later. |
| open(os.path.join(dirname, ".buildbot-patched"), "w").write("patched\n") |
| |
| # Update 'dirname' with the 'root' option. Make sure it is a subdirectory |
| # of dirname. |
| if (root and |
| os.path.abspath(os.path.join(dirname, root) |
| ).startswith(os.path.abspath(dirname))): |
| dirname = os.path.join(dirname, root) |
| |
| # Now apply the patch. |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| initialStdin=diff, environ=self.env) |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| if diff.find('DEPS') != -1: |
| d.addCallback(self.doVCUpdateOnPatch) |
| d.addCallback(self._abandonOnFailure) |
| return d |
| |
| def doVCUpdateOnPatch(self, res): |
| if self.revision and not self.branch and '@' not in str(self.revision): |
| self.branch = 'src' |
| self.delete_unversioned_trees_when_updating = False |
| return self.doVCUpdate() |
| |
| def doRunHooks(self, dummy): |
| """Runs "gclient runhooks" after patching.""" |
| dirname = os.path.join(self.builder.basedir, self.srcdir) |
| command = [chromium_utils.GetGClientCommand(), 'runhooks'] |
| c = runprocess.RunProcess( |
| self.builder, command, dirname, |
| sendRC=False, timeout=self.timeout, |
| keepStdout=True, environ=self.env) |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| return d |
| |
| def writeSourcedata(self, res): |
| """Write the sourcedata file and remove any dead source directory.""" |
| d = None |
| dead_dir = os.path.join(self.builder.basedir, self.srcdir + '.dead') |
| if os.path.isdir(dead_dir): |
| msg = 'Removing dead source dir' |
| self.sendStatus({'header': msg + '\n'}) |
| log.msg(msg) |
| command = self._RemoveDirectoryCommand(dead_dir) |
| c = runprocess.RunProcess( |
| self.builder, command, self.builder.basedir, |
| sendRC=0, timeout=self.rm_timeout, |
| environ=self.env) |
| self.command = c |
| d = c.start() |
| d.addCallback(self._abandonOnFailure) |
| open(self.sourcedatafile, 'w').write(self.sourcedata) |
| return d |
| |
| def parseGotRevision(self): |
| if not hasattr(self.command, 'stdout'): |
| # self.command may or may not have a .stdout property. The problem is when |
| # buildslave.runprocess.RunProcess(keepStdout=False) is used, it doesn't |
| # set the property .stdout at all. |
| # |
| # It may happen depending on the order of execution as self.command only |
| # tracks the last run command. |
| return {} |
| return extract_revisions(self.command.stdout) |
| |
| |
| def _handleGotRevision(self, res): |
| """Sends parseGotRevision() return values as status updates to the master. |
| """ |
| d = defer.maybeDeferred(self.parseGotRevision) |
| # parseGotRevision returns the revision dict, which is passed as the first |
| # argument to sendStatus. |
| d.addCallback(self.sendStatus) |
| return d |
| |
| def maybeDoVCFallback(self, rc): |
| """Called after doVCUpdate.""" |
| if isinstance(rc, int) and rc == 2: |
| # Non-VC failure, return 2 to turn the step red. |
| return rc |
| |
| # super |
| return SourceBaseCommand.maybeDoVCFallback(self, rc) |
| |
| def maybeDoVCRetry(self, res): |
| """Called after doVCFull.""" |
| if isinstance(res, int) and res == 2: |
| # Non-VC failure, return 2 to turn the step red. |
| return res |
| |
| # super |
| return SourceBaseCommand.maybeDoVCRetry(self, res) |
| |
| |
| def RegisterCommands(): |
| """Registers all command objects defined in this file.""" |
| try: |
| # We run this code in a try because it fails with an assertion if |
| # the module is loaded twice. |
| commandRegistry['gclient'] = 'slave.chromium_commands.GClient' |
| return |
| except (AssertionError, NameError): |
| pass |
| |
| |
| RegisterCommands() |
