Add ensure_gn_version.py and DEPS hook.
In r641353, I switched from using download_from_google_storage to download
GN binaries to using CIPD. Unfortunately, I chose to install the binary
into the same location we were using previously, which, while convenient,
meant that we might hit cases where someone would sync back to a pre-cipd
version of the checkout, run the old hook to download an old version of
GN, and then sync back to tip-of-tree, and CIPD wouldn't know that the
binary got clobbered. This could lead to really weird errors.
This CL adds a hook to DEPS to check that have the right version of GN,
and, if not, forcibly download and install it again. In the common case,
the hook should be very fast. Ultimately we want to modify CIPD to
actually validate that the files it thinks are installed are in fact right.
Bug: 944367
Change-Id: I86d3eeaf9ef232e4d472d36db54a40458702078e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1534554
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#643568}diff --git a/DEPS b/DEPS
index 07101ca..5609759 100644
--- a/DEPS
+++ b/DEPS
@@ -2259,6 +2259,19 @@
],
},
{
+ # Verify that we have the right GN binary and force-install it if we
+ # don't, in order to work around crbug.com/944367.
+ # TODO(crbug.com/944667) Get rid of this when cipd is ensuring we
+ # have the right binary more carefully and we no longer need this.
+ 'name': 'ensure_gn_version',
+ 'pattern': '.',
+ 'action': [
+ 'python',
+ 'src/buildtools/ensure_gn_version.py',
+ Var('gn_version')
+ ],
+ },
+ {
# This downloads binaries for Native Client's newlib toolchain.
# Done in lieu of building the toolchain from scratch as it can take
# anywhere from 30 minutes to 4 hours depending on platform to build.
diff --git a/buildtools/ensure_gn_version.py b/buildtools/ensure_gn_version.py
new file mode 100755
index 0000000..bbcf104
--- /dev/null
+++ b/buildtools/ensure_gn_version.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Ensure that CIPD fetched the right GN version.
+
+Due to crbug.com/944367, using cipd in gclient to fetch GN binaries
+may not always work right. This is a script that can be used as
+a backup method to force-install GN at the right revision, just in case.
+
+It should be used as a gclient hook alongside fetching GN via CIPD
+until we have a proper fix in place.
+
+TODO(crbug.com/944667): remove this script when it is no longer needed.
+"""
+
+from __future__ import print_function
+
+import argparse
+import io
+import os
+import re
+import stat
+import subprocess
+import sys
+import urllib2
+import zipfile
+
+
+BUILDTOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
+SRC_DIR = os.path.dirname(BUILDTOOLS_DIR)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('version',
+ help='CIPD "git_revision:XYZ" label for GN to sync to')
+
+ args = parser.parse_args()
+
+ if not args.version.startswith('git_revision:'):
+ print('Unknown version format: %s' % args.version)
+ return 2
+
+ desired_revision = args.version[len('git_revision:'):]
+
+ if sys.platform == 'darwin':
+ platform, member, dest_dir = ('mac-amd64', 'gn', 'mac')
+ elif sys.platform == 'win32':
+ platform, member, dest_dir = ('windows-amd64', 'gn.exe', 'win')
+ else:
+ platform, member, dest_dir = ('linux-amd64', 'gn', 'linux64')
+
+ path_to_exe = os.path.join(BUILDTOOLS_DIR, dest_dir, member)
+ cmd = [path_to_exe, '--version']
+ cmd_str = ' '.join(cmd)
+ try:
+ out = subprocess.check_output(cmd,
+ stderr=subprocess.STDOUT,
+ cwd=SRC_DIR).decode(errors='replace')
+ except subprocess.CalledProcessError as e:
+ print('`%s` returned %d:\n%s' % (cmd_str, e.returncode, e.output))
+ return 1
+ except Exception as e:
+ print('`%s` failed:\n%s' % (cmd_str, e.message))
+ return 1
+
+ current_revision = re.findall(r'\((.*)\)', out)[0]
+ if desired_revision.startswith(current_revision):
+ # We're on the right version, so we're done.
+ return 0
+
+ print("`%s` returned '%s', which wasn't what we were expecting."
+ % (cmd_str, out.strip()))
+ print("Force-installing %s to update it." % desired_revision)
+
+ url = 'https://chrome-infra-packages.appspot.com/dl/gn/gn/%s/+/%s' % (
+ platform, args.version)
+ try:
+ zipdata = urllib2.urlopen(url).read()
+ except urllib2.HTTPError as e:
+ print('Failed to download the package from %s: %d %s' % (
+ url, e.code, e.reason))
+ return 1
+ except Exception as e:
+ print('Failed to download the package from %s:\n%s' % (url, e.message))
+ return 1
+
+ try:
+ zf = zipfile.ZipFile(io.BytesIO(zipdata))
+ zf.extract(member, os.path.join(BUILDTOOLS_DIR, dest_dir))
+ except Exception as e:
+ print('Failed to extract the binary:\n%s\n' % e.msg)
+ return 1
+
+ try:
+ os.chmod(path_to_exe,
+ stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | # This is 0o755.
+ stat.S_IRGRP | stat.S_IXGRP |
+ stat.S_IROTH | stat.S_IXOTH)
+ except Exception as e:
+ print('Failed to make the binary executable:\n%s\n' %
+ e.message)
+ return 1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
