| #!/usr/bin/env python |
| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """An auto-roller for GN binaries into Chromium. |
| |
| This script is used to update the GN binaries that a Chromium |
| checkout uses. In order to update the binaries, one must follow |
| four steps in order: |
| |
| 1. Trigger try jobs to build a new GN binary at tip-of-tree and upload |
| the newly-built binaries into the right Google CloudStorage bucket. |
| 2. Wait for the try jobs to complete. |
| 3. Update the buildtools repo with the .sha1 hashes of the newly built |
| binaries. |
| 4. Update Chromium's DEPS file to the new version of the buildtools repo. |
| |
| The script has four commands that correspond to the four steps above: |
| 'build', 'wait', 'roll_buildtools', and 'roll_deps'. |
| |
| The script has a fifth command, 'roll', that runs the four in order. |
| |
| If given no arguments, the script will run the 'roll' command. |
| |
| It can only be run on linux in a clean Chromium checkout; it should |
| error out in most cases if something bad happens, but the error checking |
| isn't yet foolproof. |
| |
| """ |
| |
| from __future__ import print_function |
| |
| import argparse |
| import json |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| import urllib2 |
| |
| depot_tools_path = None |
| for p in os.environ['PATH'].split(os.pathsep): |
| if (p.rstrip(os.sep).endswith('depot_tools') and |
| os.path.isfile(os.path.join(p, 'gclient.py'))): |
| depot_tools_path = p |
| |
| assert depot_tools_path |
| if not depot_tools_path in sys.path: |
| sys.path.insert(0, depot_tools_path) |
| |
| third_party_path = os.path.join(depot_tools_path, 'third_party') |
| if not third_party_path in sys.path: |
| sys.path.insert(0, third_party_path) |
| |
| import upload |
| |
| |
| CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git' |
| |
| CODE_REVIEW_SERVER = 'https://codereview.chromium.org' |
| |
| COMMITISH_DIGITS = 10 |
| |
| class GNRoller(object): |
| def __init__(self): |
| self.chromium_src_dir = None |
| self.buildtools_dir = None |
| self.old_gn_commitish = None |
| self.new_gn_commitish = None |
| self.old_gn_version = None |
| self.new_gn_version = None |
| self.reviewer = 'dpranke@chromium.org' |
| if os.getenv('USER') == 'dpranke': |
| self.reviewer = 'brettw@chromium.org' |
| |
| def Roll(self): |
| parser = argparse.ArgumentParser() |
| parser.usage = __doc__ |
| parser.add_argument('command', nargs='?', default='roll', |
| help='build|roll|roll_buildtools|roll_deps|wait' |
| ' (%(default)s is the default)') |
| |
| args = parser.parse_args() |
| command = args.command |
| ret = self.SetUp() |
| if not ret and command in ('roll', 'build'): |
| ret = self.TriggerBuild() |
| if not ret and command in ('roll', 'wait'): |
| ret = self.WaitForBuildToFinish() |
| if not ret and command in ('roll', 'roll_buildtools'): |
| ret = self.RollBuildtools() |
| if not ret and command in ('roll', 'roll_deps'): |
| ret = self.RollDEPS() |
| |
| return ret |
| |
| def SetUp(self): |
| if sys.platform != 'linux2': |
| print('roll_gn is only tested and working on Linux for now.') |
| return 1 |
| |
| ret, out, _ = self.Call('git config --get remote.origin.url') |
| origin = out.strip() |
| if ret or origin != CHROMIUM_REPO: |
| print('Not in a Chromium repo? git config --get remote.origin.url ' |
| 'returned %d: %s' % (ret, origin)) |
| return 1 |
| |
| ret, _, _ = self.Call('git diff -q') |
| if ret: |
| print("Checkout is dirty, exiting") |
| return 1 |
| |
| _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd()) |
| self.chromium_src_dir = out.strip() |
| self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools') |
| |
| self.new_gn_commitish, self.new_gn_version = self.GetNewVersions() |
| |
| _, out, _ = self.Call('gn --version') |
| self.old_gn_version = out.strip() |
| |
| _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version) |
| self.old_gn_commitish = out.strip() |
| return 0 |
| |
| def GetNewVersions(self): |
| _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position') |
| commit_msg = out.splitlines() |
| first_line = commit_msg[0] |
| new_gn_commitish = first_line.split()[1] |
| |
| last_line = commit_msg[-1] |
| new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line) |
| |
| return new_gn_commitish, new_gn_version |
| |
| def TriggerBuild(self): |
| ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version) |
| if ret: |
| print('Failed to create a new branch for build_gn_%s' % |
| self.new_gn_version) |
| return 1 |
| |
| self.MakeDummyDepsChange() |
| |
| ret, out, err = self.Call('git commit -a -m "Build gn at %s"' % |
| self.new_gn_version) |
| if ret: |
| print('git commit failed: %s' % out + err) |
| return 1 |
| |
| print('Uploading CL to build GN at {#%s} - %s' % |
| (self.new_gn_version, self.new_gn_commitish)) |
| ret, out, err = self.Call('git cl upload -f') |
| if ret: |
| print('git-cl upload failed: %s' % out + err) |
| return 1 |
| |
| print('Starting try jobs') |
| self.Call('git-cl try -m tryserver.chromium.linux ' |
| '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish) |
| self.Call('git-cl try -m tryserver.chromium.mac ' |
| '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish) |
| self.Call('git-cl try -m tryserver.chromium.win ' |
| '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish) |
| |
| return 0 |
| |
| def MakeDummyDepsChange(self): |
| with open('DEPS') as fp: |
| deps_content = fp.read() |
| new_deps = deps_content.replace("'buildtools_revision':", |
| "'buildtools_revision': ") |
| |
| with open('DEPS', 'w') as fp: |
| fp.write(new_deps) |
| |
| def WaitForBuildToFinish(self): |
| ret = self.CheckoutBuildBranch() |
| if ret: |
| return ret |
| |
| print('Checking build') |
| results = self.CheckBuild() |
| while (any(r['state'] in ('pending', 'started') |
| for r in results.values())): |
| print() |
| print('Sleeping for 30 seconds') |
| time.sleep(30) |
| print('Checking build') |
| results = self.CheckBuild() |
| |
| ret = 0 if all(r['state'] == 'success' for r in results.values()) else 1 |
| if ret: |
| print('Build failed.') |
| else: |
| print('Builds ready.') |
| |
| # Close the build CL and move off of the build branch back to whatever |
| # we were on before. |
| self.Call('git-cl set-close') |
| self.MoveToLastHead() |
| |
| return ret |
| |
| def CheckoutBuildBranch(self): |
| ret, out, err = self.Call('git checkout build_gn_%s' % self.new_gn_version) |
| if ret: |
| print('Failed to check out build_gn_%s' % self.new_gn_version) |
| if out: |
| print(out) |
| if err: |
| print(err, file=sys.stderr) |
| return ret |
| |
| def CheckBuild(self): |
| _, out, _ = self.Call('git-cl issue') |
| |
| issue = int(out.split()[2]) |
| |
| _, out, _ = self.Call('git config user.email') |
| email = '' |
| rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email) |
| try: |
| props = json.loads(rpc_server.Send('/api/%d' % issue)) |
| except Exception as e: |
| print('Failed to load patch data: %s' % e) |
| return {} |
| |
| patchset = int(props['patchsets'][-1]) |
| |
| try: |
| try_job_results = json.loads(rpc_server.Send( |
| '/api/%d/%d/try_job_results' % (issue, patchset))) |
| except Exception as e: |
| print('Failed to load try job results: %s' % e) |
| return {} |
| |
| if not try_job_results: |
| print('No try jobs found on most recent patchset') |
| return {} |
| |
| results = {} |
| for job in try_job_results: |
| builder = job['builder'] |
| if builder == 'linux_chromium_gn_upload': |
| platform = 'linux64' |
| elif builder == 'mac_chromium_gn_upload': |
| platform = 'mac' |
| elif builder == 'win8_chromium_gn_upload': |
| platform = 'win' |
| else: |
| print('Unexpected builder: %s') |
| continue |
| |
| TRY_JOB_RESULT_STATES = ('started', 'success', 'warnings', 'failure', |
| 'skipped', 'exception', 'retry', 'pending') |
| state = TRY_JOB_RESULT_STATES[int(job['result']) + 1] |
| url_str = ' %s' % job['url'] |
| build = url_str.split('/')[-1] |
| |
| sha1 = '-' |
| results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str}) |
| |
| if state == 'success': |
| jsurl = url_str.replace('http://build.chromium.org/', |
| 'http://chrome-build-extract.appspot.com/') |
| jsurl = jsurl + '?json=1' |
| try: |
| fp = urllib2.urlopen(jsurl) |
| except urllib2.HTTPError as e: |
| print('Failed to open %s: %s' % (jsurl, e)) |
| return {} |
| |
| js = json.loads(fp.read()) |
| fp.close() |
| sha1_step_name = 'gn sha1' |
| for step in js['steps']: |
| if step['name'] == sha1_step_name: |
| # TODO: At some point infra changed the step text to |
| # contain the step name; once all of the masters have been |
| # restarted we can probably assert that the step text |
| # with the step_name. |
| sha1_step_text_prefix = sha1_step_name + '<br>' |
| if step['text'][-1].startswith(sha1_step_text_prefix): |
| sha1 = step['text'][-1][len(sha1_step_text_prefix):] |
| else: |
| sha1 = step['text'][-1] |
| |
| if results[platform]['build'] < build: |
| results[platform]['build'] = build |
| results[platform]['sha1'] = sha1 |
| results[platform]['state'] = state |
| results[platform]['url'] = url_str |
| |
| for platform, r in results.items(): |
| print(platform) |
| print(' sha1: %s' % r['sha1']) |
| print(' state: %s' % r['state']) |
| print(' build: %s' % r['build']) |
| print(' url: %s' % r['url']) |
| print() |
| |
| return results |
| |
| def RollBuildtools(self): |
| ret = self.CheckoutBuildBranch() |
| if ret: |
| return ret |
| |
| results = self.CheckBuild() |
| if (len(results) < 3 or |
| not all(r['state'] == 'success' for r in results.values())): |
| print("Roll isn't done or didn't succeed, exiting:") |
| return 1 |
| |
| desc = self.GetBuildtoolsDesc() |
| |
| self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version, |
| cwd=self.buildtools_dir) |
| |
| for platform in results: |
| fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1' |
| path = os.path.join(self.buildtools_dir, platform, fname) |
| with open(path, 'w') as fp: |
| fp.write('%s\n' % results[platform]['sha1']) |
| |
| desc_file = tempfile.NamedTemporaryFile(delete=False) |
| try: |
| desc_file.write(desc) |
| desc_file.close() |
| self.Call('git commit -a -F %s' % desc_file.name, |
| cwd=self.buildtools_dir) |
| self.Call('git-cl upload -f --send-mail', |
| cwd=self.buildtools_dir) |
| finally: |
| os.remove(desc_file.name) |
| |
| ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir) |
| if ret: |
| print("buildtools git cl land failed: %d" % ret) |
| if out: |
| print(out) |
| if err: |
| print(err) |
| return ret |
| |
| # Fetch the revision we just committed so that RollDEPS will find it. |
| self.Call('git fetch', cwd=self.buildtools_dir) |
| |
| # Reset buildtools to the new commit so that we're not still on the |
| # merged branch. |
| self.Call('git checkout origin/master', cwd=self.buildtools_dir) |
| |
| _, out, _ = self.Call('git rev-parse origin/master', |
| cwd=self.buildtools_dir) |
| new_buildtools_commitish = out.strip() |
| print('Ready to roll buildtools to %s in DEPS' % new_buildtools_commitish) |
| |
| return 0 |
| |
| def RollDEPS(self): |
| ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version) |
| if ret: |
| print('Failed to create a new branch for roll_gn_%s' % |
| self.new_gn_version) |
| return 1 |
| |
| _, out, _ = self.Call('git rev-parse origin/master', |
| cwd=self.buildtools_dir) |
| new_buildtools_commitish = out.strip() |
| |
| new_deps_lines = [] |
| old_buildtools_commitish = '' |
| with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp: |
| for l in fp.readlines(): |
| m = re.match(".*'buildtools_revision':.*'(.+)',", l) |
| if m: |
| old_buildtools_commitish = m.group(1) |
| new_deps_lines.append(" 'buildtools_revision': '%s',\n" % |
| new_buildtools_commitish) |
| else: |
| new_deps_lines.append(l) |
| |
| if not old_buildtools_commitish: |
| print('Could not update DEPS properly, exiting') |
| return 1 |
| |
| with open('DEPS', 'w') as fp: |
| fp.write(''.join(new_deps_lines)) |
| |
| desc = self.GetDEPSRollDesc(old_buildtools_commitish, |
| new_buildtools_commitish) |
| desc_file = tempfile.NamedTemporaryFile(delete=False) |
| try: |
| desc_file.write(desc) |
| desc_file.close() |
| self.Call('git commit -a -F %s' % desc_file.name) |
| self.Call('git-cl upload -f --send-mail --use-commit-queue') |
| finally: |
| os.remove(desc_file.name) |
| |
| # Move off of the roll branch onto whatever we were on before. |
| # Do not explicitly close the roll CL issue, however; the CQ |
| # will close it when the roll lands, assuming it does so. |
| self.MoveToLastHead() |
| |
| return 0 |
| |
| def MoveToLastHead(self): |
| # When this is called, there will be a commit + a checkout as |
| # the two most recent entries in the reflog, assuming nothing as |
| # modified the repo while this script has been running. |
| _, out, _ = self.Call('git reflog -2') |
| m = re.search('moving from ([^\s]+)', out) |
| last_head = m.group(1) |
| self.Call('git checkout %s' % last_head) |
| |
| def GetBuildtoolsDesc(self): |
| gn_changes = self.GetGNChanges() |
| return ( |
| 'Roll gn %s..%s (r%s:r%s)\n' |
| '\n' |
| '%s' |
| '\n' |
| 'TBR=%s\n' % ( |
| self.old_gn_commitish[:COMMITISH_DIGITS], |
| self.new_gn_commitish[:COMMITISH_DIGITS], |
| self.old_gn_version, |
| self.new_gn_version, |
| gn_changes, |
| self.reviewer, |
| )) |
| |
| def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish): |
| gn_changes = self.GetGNChanges() |
| |
| return ( |
| 'Roll buildtools %s..%s\n' |
| '\n' |
| ' In order to roll GN %s..%s (r%s:r%s) and pick up\n' |
| ' the following changes:\n' |
| '\n' |
| '%s' |
| '\n' |
| 'TBR=%s\n' % ( |
| old_buildtools_commitish[:COMMITISH_DIGITS], |
| new_buildtools_commitish[:COMMITISH_DIGITS], |
| self.old_gn_commitish[:COMMITISH_DIGITS], |
| self.new_gn_commitish[:COMMITISH_DIGITS], |
| self.old_gn_version, |
| self.new_gn_version, |
| gn_changes, |
| self.reviewer, |
| )) |
| |
| def GetGNChanges(self): |
| _, out, _ = self.Call( |
| "git log --pretty=' %h %s' " + |
| "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish)) |
| return out |
| |
| def Call(self, cmd, cwd=None): |
| proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, |
| cwd=(cwd or self.chromium_src_dir)) |
| out, err = proc.communicate() |
| return proc.returncode, out or '', err or '' |
| |
| |
| if __name__ == '__main__': |
| roller = GNRoller() |
| sys.exit(roller.Roll()) |