| #!/usr/bin/env python |
| # Copyright 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| import collections |
| import logging |
| import os |
| import re |
| import subprocess |
| import sys |
| import time |
| |
| |
| SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir)) |
| sys.path.insert(0, os.path.join(SRC_DIR, 'build')) |
| import find_depot_tools |
| find_depot_tools.add_depot_tools_to_path() |
| import roll_dep_svn |
| from third_party import upload |
| |
| # Avoid depot_tools/third_party/upload.py print verbose messages. |
| upload.verbosity = 0 # Errors only. |
| |
| CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git' |
| COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') |
| CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$') |
| REVIEW_URL_RE = re.compile('^https?://(.*)/(.*)') |
| ROLL_BRANCH_NAME = 'special_webrtc_roll_branch' |
| TRYJOB_STATUS_SLEEP_SECONDS = 30 |
| |
| # Use a shell for subcommands on Windows to get a PATH search. |
| IS_WIN = sys.platform.startswith('win') |
| WEBRTC_PATH = os.path.join('third_party', 'webrtc') |
| # Run these CQ trybots in addition to the default ones in infra/config/cq.cfg. |
| EXTRA_TRYBOTS = ( |
| 'master.tryserver.chromium.linux:linux_chromium_archive_rel_ng;' |
| 'master.tryserver.chromium.mac:mac_chromium_archive_rel_ng' |
| ) |
| |
| # Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py |
| # plus the -1 code which is used when there's no result yet. |
| TRYJOB_STATUS = { |
| -1: 'RUNNING', |
| 0: 'SUCCESS', |
| 1: 'WARNINGS', |
| 2: 'FAILURE', |
| 3: 'SKIPPED', |
| 4: 'EXCEPTION', |
| 5: 'RETRY', |
| } |
| SUCCESS_STATUS = (0, 1, 3) |
| FAILURE_STATUS = (2, 4, 5) |
| |
| CommitInfo = collections.namedtuple('CommitInfo', ['commit_position', |
| 'git_commit', |
| 'git_repo_url']) |
| CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'review_server']) |
| |
| |
| def _VarLookup(local_scope): |
| return lambda var_name: local_scope['vars'][var_name] |
| |
| |
| def _PosixPath(path): |
| """Convert a possibly-Windows path to a posix-style path.""" |
| (_, path) = os.path.splitdrive(path) |
| return path.replace(os.sep, '/') |
| |
| |
| def _ParseGitCommitPosition(description): |
| for line in reversed(description.splitlines()): |
| m = COMMIT_POSITION_RE.match(line.strip()) |
| if m: |
| return m.group(1) |
| logging.error('Failed to parse svn revision id from:\n%s\n', description) |
| sys.exit(-1) |
| |
| |
| def _ParseGitCommitHash(description): |
| for line in description.splitlines(): |
| if line.startswith('commit '): |
| return line.split()[1] |
| logging.error('Failed to parse git commit id from:\n%s\n', description) |
| sys.exit(-1) |
| return None |
| |
| |
| def _ParseDepsFile(filename): |
| with open(filename, 'rb') as f: |
| deps_content = f.read() |
| return _ParseDepsDict(deps_content) |
| |
| |
| def _ParseDepsDict(deps_content): |
| local_scope = {} |
| global_scope = { |
| 'Var': _VarLookup(local_scope), |
| 'deps_os': {}, |
| } |
| exec(deps_content, global_scope, local_scope) |
| return local_scope |
| |
| |
| def _PrintTrybotsStatus(tryjob_results): |
| status_to_name = {} |
| for trybot_result in tryjob_results: |
| status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN') |
| status_to_name.setdefault(status, []) |
| status_to_name[status].append(trybot_result['builder']) |
| |
| print '\n========== TRYJOBS STATUS ==========' |
| for status,name_list in status_to_name.iteritems(): |
| print '%s: %s' % (status, ','.join(sorted(name_list))) |
| |
| |
| class AutoRoller(object): |
| def __init__(self, chromium_src): |
| self._chromium_src = chromium_src |
| |
| def _RunCommand(self, command, working_dir=None, ignore_exit_code=False, |
| extra_env=None): |
| """Runs a command and returns the stdout from that command. |
| |
| If the command fails (exit code != 0), the function will exit the process. |
| """ |
| working_dir = working_dir or self._chromium_src |
| logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir) |
| env = os.environ.copy() |
| if extra_env: |
| logging.debug('extra env: %s', extra_env) |
| env.update(extra_env) |
| p = subprocess.Popen(command, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, shell=IS_WIN, env=env, |
| cwd=working_dir, universal_newlines=True) |
| output = p.stdout.read() |
| p.wait() |
| p.stdout.close() |
| p.stderr.close() |
| |
| if not ignore_exit_code and p.returncode != 0: |
| logging.error('Command failed: %s\n%s', str(command), output) |
| sys.exit(p.returncode) |
| return output |
| |
| def _GenerateCLDescriptionCommand(self, webrtc_current, webrtc_new): |
| commit_range = '%s..%s' % (webrtc_current.git_commit[:7], |
| webrtc_new.git_commit[:7]) |
| |
| webrtc_changelog_url = '%s/+log/%s' % (webrtc_current.git_repo_url, |
| commit_range) |
| |
| git_log_cmd = ['git', 'log', commit_range, '--date=short', '--no-merges', |
| '--format=%ad %ae %s'] |
| |
| working_dir = os.path.join(self._chromium_src, WEBRTC_PATH) |
| git_log = self._RunCommand(git_log_cmd, working_dir=working_dir) |
| |
| nb_commits = git_log.count('\n') |
| webrtc_header = 'Roll WebRTC %s:%s (%d commit%s)' % ( |
| webrtc_current.commit_position, webrtc_new.commit_position, |
| nb_commits, 's' if nb_commits > 1 else '') |
| git_author = self._RunCommand( |
| ['git', 'config', 'user.email'], |
| working_dir=self._chromium_src).splitlines()[0] |
| description = ('%s\n\n' |
| 'Changes: %s\n\n' |
| '$ %s\n' |
| '%s\n' |
| 'TBR=%s\n' |
| 'CQ_INCLUDE_TRYBOTS=%s\n') % ( |
| webrtc_header, |
| webrtc_changelog_url, |
| ' '.join(git_log_cmd), |
| git_log, |
| git_author, |
| EXTRA_TRYBOTS) |
| |
| return description |
| |
| def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None): |
| working_dir = os.path.join(self._chromium_src, path_below_src) |
| self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir) |
| revision_range = git_hash or 'origin' |
| ret = self._RunCommand( |
| ['git', '--no-pager', 'log', revision_range, |
| '--no-abbrev-commit', '--pretty=full', '-1'], |
| working_dir=working_dir) |
| return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret), |
| git_repo_url) |
| |
| def _GetDepsCommitInfo(self, deps_dict, path_below_src): |
| entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)] |
| at_index = entry.find('@') |
| git_repo_url = entry[:at_index] |
| git_hash = entry[at_index + 1:] |
| return self._GetCommitInfo(path_below_src, git_hash, git_repo_url) |
| |
| def _GetCLInfo(self): |
| cl_output = self._RunCommand(['git', 'cl', 'issue']) |
| m = CL_ISSUE_RE.match(cl_output.strip()) |
| if not m: |
| logging.error('Cannot find any CL info. Output was:\n%s', cl_output) |
| sys.exit(-1) |
| issue_number = int(m.group(1)) |
| url = m.group(2) |
| |
| # Parse the codereview host from the URL. |
| m = REVIEW_URL_RE.match(url) |
| if not m: |
| logging.error('Cannot parse codereview host from URL: %s', url) |
| sys.exit(-1) |
| review_server = m.group(1) |
| return CLInfo(issue_number, url, review_server) |
| |
| def _GetCurrentBranchName(self): |
| return self._RunCommand( |
| ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0] |
| |
| def _IsTreeClean(self): |
| lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines() |
| if len(lines) == 0: |
| return True |
| |
| logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines)) |
| return False |
| |
| def _UpdateReadmeFile(self, readme_path, new_revision): |
| readme = open(os.path.join(self._chromium_src, readme_path), 'r+') |
| txt = readme.read() |
| m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE), |
| ('Revision: %s' % new_revision), txt) |
| readme.seek(0) |
| readme.write(m) |
| readme.truncate() |
| |
| def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll, |
| revision): |
| # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for |
| # cross platform compatibility. |
| |
| if not ignore_checks: |
| if self._GetCurrentBranchName() != 'master': |
| logging.error('Please checkout the master branch.') |
| return -1 |
| if not self._IsTreeClean(): |
| logging.error('Please make sure you don\'t have any modified files.') |
| return -1 |
| |
| logging.debug('Checking for a previous roll branch.') |
| if close_previous_roll: |
| self.Abort() |
| |
| logging.debug('Pulling latest changes') |
| if not ignore_checks: |
| self._RunCommand(['git', 'pull']) |
| |
| self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) |
| |
| # Modify Chromium's DEPS file. |
| |
| # Parse current hashes. |
| deps_filename = os.path.join(self._chromium_src, 'DEPS') |
| deps = _ParseDepsFile(deps_filename) |
| webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH) |
| |
| # Get the commit info for the given revision. If it's None, get the commit |
| # info for ToT. |
| revision_info = self._GetCommitInfo(WEBRTC_PATH, revision) |
| |
| if IS_WIN: |
| # Make sure the roll script doesn't use Windows line endings. |
| self._RunCommand(['git', 'config', 'core.autocrlf', 'true']) |
| |
| self._UpdateDep(deps_filename, WEBRTC_PATH, revision_info) |
| |
| if self._IsTreeClean(): |
| print 'The latest revision is already rolled for WebRTC.' |
| self._DeleteRollBranch() |
| else: |
| description = self._GenerateCLDescriptionCommand( |
| webrtc_current, revision_info) |
| logging.debug('Committing changes locally.') |
| self._RunCommand(['git', 'add', '--update', '.']) |
| self._RunCommand(['git', 'commit', '-m', description]) |
| logging.debug('Uploading changes...') |
| upload_cmd = ['git', 'cl', 'upload'] |
| if not dry_run and not no_commit: |
| logging.debug('Sending the CL to the CQ...') |
| upload_cmd.extend(['--use-commit-queue', '--send-mail']) |
| self._RunCommand(upload_cmd, extra_env={'EDITOR': 'true'}) |
| cl_info = self._GetCLInfo() |
| logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url) |
| |
| # TODO(kjellander): Checkout masters/previous branches again. |
| return 0 |
| |
| def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info): |
| dep_name = os.path.join('src', dep_relative_to_src) |
| comment = 'commit position %s' % commit_info.commit_position |
| |
| # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's |
| # temporarily change the working directory and then change back. |
| cwd = os.getcwd() |
| os.chdir(os.path.dirname(deps_filename)) |
| roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name, |
| commit_info.git_commit, comment) |
| os.chdir(cwd) |
| |
| def _DeleteRollBranch(self): |
| self._RunCommand(['git', 'checkout', 'master']) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME) |
| |
| |
| def _GetBranches(self): |
| """Returns a tuple of active,branches. |
| |
| The 'active' is the name of the currently active branch and 'branches' is a |
| list of all branches. |
| """ |
| lines = self._RunCommand(['git', 'branch']).split('\n') |
| branches = [] |
| active = '' |
| for l in lines: |
| if '*' in l: |
| # The assumption is that the first char will always be the '*'. |
| active = l[1:].strip() |
| branches.append(active) |
| else: |
| b = l.strip() |
| if b: |
| branches.append(b) |
| return (active, branches) |
| |
| def Abort(self): |
| active_branch, branches = self._GetBranches() |
| if active_branch == ROLL_BRANCH_NAME: |
| active_branch = 'master' |
| if ROLL_BRANCH_NAME in branches: |
| print 'Aborting pending roll.' |
| self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) |
| # Ignore an error here in case an issue wasn't created for some reason. |
| self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True) |
| self._RunCommand(['git', 'checkout', active_branch]) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| return 0 |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser( |
| description='Find webrtc revisions for roll.') |
| parser.add_argument('--abort', |
| help=('Aborts a previously prepared roll. ' |
| 'Closes any associated issues and deletes the roll branches'), |
| action='store_true') |
| parser.add_argument('--no-commit', |
| help=('Don\'t send the CL to the CQ. This is useful if additional changes ' |
| 'are needed to the CL (like for API changes).'), |
| action='store_true') |
| parser.add_argument('--close-previous-roll', action='store_true', |
| help='Abort a previous roll if one exists.') |
| parser.add_argument('--dry-run', action='store_true', default=False, |
| help='Create branches and CLs but doesn\'t send tryjobs or commit.') |
| parser.add_argument('--ignore-checks', action='store_true', default=False, |
| help=('Skips checks for being on the master branch, dirty workspaces and ' |
| 'the updating of the checkout. Will still delete and create local ' |
| 'Git branches.')) |
| parser.add_argument('-r', '--revision', default=None, |
| help='WebRTC revision to roll. If not specified,' |
| 'the latest version will be used') |
| parser.add_argument('-v', '--verbose', action='store_true', default=False, |
| help='Be extra verbose in printing of log messages.') |
| args = parser.parse_args() |
| |
| if args.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| else: |
| logging.basicConfig(level=logging.ERROR) |
| |
| autoroller = AutoRoller(SRC_DIR) |
| if args.abort: |
| return autoroller.Abort() |
| else: |
| return autoroller.PrepareRoll(args.dry_run, args.ignore_checks, |
| args.no_commit, args.close_previous_roll, |
| args.revision) |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
