|
#!/usr/bin/env python |
|
# Copyright 2014 The Chromium Authors. All rights reserved. |
|
# Use of this source code is governed by a BSD-style license that can be |
|
# found in the LICENSE file. |
|
|
|
"""Script that attempts to push to a special git repository to verify that git |
|
credentials are configured correctly. It also verifies that gclient solution is |
|
configured to use git checkout. |
|
|
|
It will be added as gclient hook shortly before Chromium switches to git and |
|
removed after the switch. |
|
|
|
When running as hook in *.corp.google.com network it will also report status |
|
of the push attempt to the server (on appengine), so that chrome-infra team can |
|
collect information about misconfigured Git accounts. |
|
""" |
|
|
|
import contextlib |
|
import datetime |
|
import errno |
|
import getpass |
|
import json |
|
import logging |
|
import netrc |
|
import optparse |
|
import os |
|
import pprint |
|
import shutil |
|
import socket |
|
import ssl |
|
import subprocess |
|
import sys |
|
import tempfile |
|
import time |
|
import urllib2 |
|
import urlparse |
|
|
|
|
|
# Absolute path to src/ directory. |
|
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
|
|
|
# Absolute path to a file with gclient solutions. |
|
GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient') |
|
|
|
# Incremented whenever some changes to scrip logic are made. Change in version |
|
# will cause the check to be rerun on next gclient runhooks invocation. |
|
CHECKER_VERSION = 1 |
|
|
|
# Do not attempt to upload a report after this date. |
|
UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1) |
|
|
|
# URL to POST json with results to. |
|
MOTHERSHIP_URL = ( |
|
'https://chromium-git-access.appspot.com/' |
|
'git_access/api/v1/reports/access_check') |
|
|
|
# Repository to push test commits to. |
|
TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test' |
|
|
|
# Git-compatible gclient solution. |
|
GOOD_GCLIENT_SOLUTION = { |
|
'name': 'src', |
|
'deps_file': 'DEPS', |
|
'managed': False, |
|
'url': 'https://chromium.googlesource.com/chromium/src.git', |
|
} |
|
|
|
# Possible chunks of git push response in case .netrc is misconfigured. |
|
BAD_ACL_ERRORS = ( |
|
'(prohibited by Gerrit)', |
|
'does not match your user account', |
|
'Git repository not found', |
|
'Invalid user name or password', |
|
'Please make sure you have the correct access rights', |
|
) |
|
|
|
# Git executable to call. |
|
GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git' |
|
|
|
|
|
def is_on_bot(): |
|
"""True when running under buildbot.""" |
|
return os.environ.get('CHROME_HEADLESS') == '1' |
|
|
|
|
|
def is_in_google_corp(): |
|
"""True when running in google corp network.""" |
|
try: |
|
return socket.getfqdn().endswith('.corp.google.com') |
|
except socket.error: |
|
logging.exception('Failed to get FQDN') |
|
return False |
|
|
|
|
|
def is_using_git(): |
|
"""True if git checkout is used.""" |
|
return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects')) |
|
|
|
|
|
def is_using_svn(): |
|
"""True if svn checkout is used.""" |
|
return os.path.exists(os.path.join(REPO_ROOT, '.svn')) |
|
|
|
|
|
def read_git_config(prop): |
|
"""Reads git config property of src.git repo. |
|
|
|
Returns empty string in case of errors. |
|
""" |
|
try: |
|
proc = subprocess.Popen( |
|
[GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT) |
|
out, _ = proc.communicate() |
|
return out.strip() |
|
except OSError as exc: |
|
if exc.errno != errno.ENOENT: |
|
logging.exception('Unexpected error when calling git') |
|
return '' |
|
|
|
|
|
def read_netrc_user(netrc_obj, host): |
|
"""Reads 'user' field of a host entry in netrc. |
|
|
|
Returns empty string if netrc is missing, or host is not there. |
|
""" |
|
if not netrc_obj: |
|
return '' |
|
entry = netrc_obj.authenticators(host) |
|
if not entry: |
|
return '' |
|
return entry[0] |
|
|
|
|
|
def get_git_version(): |
|
"""Returns version of git or None if git is not available.""" |
|
try: |
|
proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE) |
|
out, _ = proc.communicate() |
|
return out.strip() if proc.returncode == 0 else '' |
|
except OSError as exc: |
|
if exc.errno != errno.ENOENT: |
|
logging.exception('Unexpected error when calling git') |
|
return '' |
|
|
|
|
|
def read_gclient_solution(): |
|
"""Read information about 'src' gclient solution from .gclient file. |
|
|
|
Returns tuple: |
|
(url, deps_file, managed) |
|
or |
|
(None, None, None) if no such solution. |
|
""" |
|
try: |
|
env = {} |
|
execfile(GCLIENT_CONFIG, env, env) |
|
for sol in (env.get('solutions') or []): |
|
if sol.get('name') == 'src': |
|
return sol.get('url'), sol.get('deps_file'), sol.get('managed') |
|
return None, None, None |
|
except Exception: |
|
logging.exception('Failed to read .gclient solution') |
|
return None, None, None |
|
|
|
|
|
def read_git_insteadof(host): |
|
"""Reads relevant insteadOf config entries.""" |
|
try: |
|
proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE) |
|
out, _ = proc.communicate() |
|
lines = [] |
|
for line in out.strip().split('\n'): |
|
line = line.lower() |
|
if 'insteadof=' in line and host in line: |
|
lines.append(line) |
|
return '\n'.join(lines) |
|
except OSError as exc: |
|
if exc.errno != errno.ENOENT: |
|
logging.exception('Unexpected error when calling git') |
|
return '' |
|
|
|
|
|
def scan_configuration(): |
|
"""Scans local environment for git related configuration values.""" |
|
# Git checkout? |
|
is_git = is_using_git() |
|
|
|
# On Windows HOME should be set. |
|
if 'HOME' in os.environ: |
|
netrc_path = os.path.join( |
|
os.environ['HOME'], |
|
'_netrc' if sys.platform.startswith('win') else '.netrc') |
|
else: |
|
netrc_path = None |
|
|
|
# Netrc exists? |
|
is_using_netrc = netrc_path and os.path.exists(netrc_path) |
|
|
|
# Read it. |
|
netrc_obj = None |
|
if is_using_netrc: |
|
try: |
|
netrc_obj = netrc.netrc(netrc_path) |
|
except Exception: |
|
logging.exception('Failed to read netrc from %s', netrc_path) |
|
netrc_obj = None |
|
|
|
# Read gclient 'src' solution. |
|
gclient_url, gclient_deps, gclient_managed = read_gclient_solution() |
|
|
|
return { |
|
'checker_version': CHECKER_VERSION, |
|
'is_git': is_git, |
|
'is_home_set': 'HOME' in os.environ, |
|
'is_using_netrc': is_using_netrc, |
|
'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0, |
|
'git_version': get_git_version(), |
|
'platform': sys.platform, |
|
'username': getpass.getuser(), |
|
'git_user_email': read_git_config('user.email') if is_git else '', |
|
'git_user_name': read_git_config('user.name') if is_git else '', |
|
'git_insteadof': read_git_insteadof('chromium.googlesource.com'), |
|
'chromium_netrc_email': |
|
read_netrc_user(netrc_obj, 'chromium.googlesource.com'), |
|
'chrome_internal_netrc_email': |
|
read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'), |
|
'gclient_deps': gclient_deps, |
|
'gclient_managed': gclient_managed, |
|
'gclient_url': gclient_url, |
|
} |
|
|
|
|
|
def last_configuration_path(): |
|
"""Path to store last checked configuration.""" |
|
if is_using_git(): |
|
return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json') |
|
elif is_using_svn(): |
|
return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json') |
|
else: |
|
return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json') |
|
|
|
|
|
def read_last_configuration(): |
|
"""Reads last checked configuration if it exists.""" |
|
try: |
|
with open(last_configuration_path(), 'r') as f: |
|
return json.load(f) |
|
except (IOError, ValueError): |
|
return None |
|
|
|
|
|
def write_last_configuration(conf): |
|
"""Writes last checked configuration to a file.""" |
|
try: |
|
with open(last_configuration_path(), 'w') as f: |
|
json.dump(conf, f, indent=2, sort_keys=True) |
|
except IOError: |
|
logging.exception('Failed to write JSON to %s', path) |
|
|
|
|
|
@contextlib.contextmanager |
|
def temp_directory(): |
|
"""Creates a temp directory, then nukes it.""" |
|
tmp = tempfile.mkdtemp() |
|
try: |
|
yield tmp |
|
finally: |
|
try: |
|
shutil.rmtree(tmp) |
|
except (OSError, IOError): |
|
logging.exception('Failed to remove temp directory %s', tmp) |
|
|
|
|
|
class Runner(object): |
|
"""Runs a bunch of commands in some directory, collects logs from them.""" |
|
|
|
def __init__(self, cwd, verbose): |
|
self.cwd = cwd |
|
self.verbose = verbose |
|
self.log = [] |
|
|
|
def run(self, cmd): |
|
self.append_to_log('> ' + ' '.join(cmd)) |
|
retcode = -1 |
|
try: |
|
proc = subprocess.Popen( |
|
cmd, |
|
stdout=subprocess.PIPE, |
|
stderr=subprocess.STDOUT, |
|
cwd=self.cwd) |
|
out, _ = proc.communicate() |
|
out = out.strip() |
|
retcode = proc.returncode |
|
except OSError as exc: |
|
out = str(exc) |
|
if retcode: |
|
out += '\n(exit code: %d)' % retcode |
|
self.append_to_log(out) |
|
return retcode |
|
|
|
def append_to_log(self, text): |
|
if text: |
|
self.log.append(text) |
|
if self.verbose: |
|
logging.warning(text) |
|
|
|
|
|
def check_git_config(conf, report_url, verbose): |
|
"""Attempts to push to a git repository, reports results to a server. |
|
|
|
Returns True if the check finished without incidents (push itself may |
|
have failed) and should NOT be retried on next invocation of the hook. |
|
""" |
|
# Don't even try to push if netrc is not configured. |
|
if not conf['chromium_netrc_email']: |
|
return upload_report( |
|
conf, |
|
report_url, |
|
verbose, |
|
push_works=False, |
|
push_log='', |
|
push_duration_ms=0) |
|
|
|
# Ref to push to, each user has its own ref. |
|
ref = 'refs/push-test/%s' % conf['chromium_netrc_email'] |
|
|
|
push_works = False |
|
flake = False |
|
started = time.time() |
|
try: |
|
logging.warning('Checking push access to the git repository...') |
|
with temp_directory() as tmp: |
|
# Prepare a simple commit on a new timeline. |
|
runner = Runner(tmp, verbose) |
|
runner.run([GIT_EXE, 'init', '.']) |
|
if conf['git_user_name']: |
|
runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']]) |
|
if conf['git_user_email']: |
|
runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']]) |
|
with open(os.path.join(tmp, 'timestamp'), 'w') as f: |
|
f.write(str(int(time.time() * 1000))) |
|
runner.run([GIT_EXE, 'add', 'timestamp']) |
|
runner.run([GIT_EXE, 'commit', '-m', 'Push test.']) |
|
# Try to push multiple times if it fails due to issues other than ACLs. |
|
attempt = 0 |
|
while attempt < 5: |
|
attempt += 1 |
|
logging.info('Pushing to %s %s', TEST_REPO_URL, ref) |
|
ret = runner.run( |
|
[GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f']) |
|
if not ret: |
|
push_works = True |
|
break |
|
if any(x in runner.log[-1] for x in BAD_ACL_ERRORS): |
|
push_works = False |
|
break |
|
except Exception: |
|
logging.exception('Unexpected exception when pushing') |
|
flake = True |
|
|
|
if push_works: |
|
logging.warning('Git push works!') |
|
else: |
|
logging.warning( |
|
'Git push doesn\'t work, which is fine if you are not a committer.') |
|
|
|
uploaded = upload_report( |
|
conf, |
|
report_url, |
|
verbose, |
|
push_works=push_works, |
|
push_log='\n'.join(runner.log), |
|
push_duration_ms=int((time.time() - started) * 1000)) |
|
return uploaded and not flake |
|
|
|
|
|
def check_gclient_config(conf): |
|
"""Shows warning if gclient solution is not properly configured for git.""" |
|
# Ignore configs that do not have 'src' solution at all. |
|
if not conf['gclient_url']: |
|
return |
|
current = { |
|
'name': 'src', |
|
'deps_file': conf['gclient_deps'] or 'DEPS', |
|
'managed': conf['gclient_managed'] or False, |
|
'url': conf['gclient_url'], |
|
} |
|
# After depot_tools r291592 both DEPS and .DEPS.git are valid. |
|
good = GOOD_GCLIENT_SOLUTION.copy() |
|
good['deps_file'] = current['deps_file'] |
|
if current == good: |
|
return |
|
# Show big warning if url or deps_file is wrong. |
|
if current['url'] != good['url'] or current['deps_file'] != good['deps_file']: |
|
print '-' * 80 |
|
print 'Your gclient solution is not set to use supported git workflow!' |
|
print |
|
print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG |
|
print pprint.pformat(current, indent=2) |
|
print |
|
print 'Correct \'src\' solution to use git:' |
|
print pprint.pformat(good, indent=2) |
|
print |
|
print 'Please update your .gclient file ASAP.' |
|
print '-' * 80 |
|
# Show smaller (additional) warning about managed workflow. |
|
if current['managed']: |
|
print '-' * 80 |
|
print ( |
|
'You are using managed gclient mode with git, which was deprecated ' |
|
'on 8/22/13:') |
|
print ( |
|
'https://groups.google.com/a/chromium.org/' |
|
'forum/#!topic/chromium-dev/n9N5N3JL2_U') |
|
print |
|
print ( |
|
'It is strongly advised to switch to unmanaged mode. For more ' |
|
'information about managed mode and reasons for its deprecation see:') |
|
print 'http://www.chromium.org/developers/how-tos/get-the-code/gclient-managed-mode' |
|
print |
|
print ( |
|
'There\'s also a large suite of tools to assist managing git ' |
|
'checkouts.\nSee \'man depot_tools\' (or read ' |
|
'depot_tools/man/html/depot_tools.html).') |
|
print '-' * 80 |
|
|
|
|
|
def upload_report( |
|
conf, report_url, verbose, push_works, push_log, push_duration_ms): |
|
"""Posts report to the server, returns True if server accepted it. |
|
|
|
Uploads the report only if script is running in Google corp network. Otherwise |
|
just prints the report. |
|
""" |
|
report = conf.copy() |
|
report.update( |
|
push_works=push_works, |
|
push_log=push_log, |
|
push_duration_ms=push_duration_ms) |
|
|
|
as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True) |
|
if verbose: |
|
print 'Status of git push attempt:' |
|
print as_bytes |
|
|
|
# Do not upload it outside of corp or if server side is already disabled. |
|
if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS: |
|
if verbose: |
|
print ( |
|
'You can send the above report to chrome-git-migration@google.com ' |
|
'if you need help to set up you committer git account.') |
|
return True |
|
|
|
req = urllib2.Request( |
|
url=report_url, |
|
data=as_bytes, |
|
headers={'Content-Type': 'application/json; charset=utf-8'}) |
|
|
|
attempt = 0 |
|
success = False |
|
while not success and attempt < 10: |
|
attempt += 1 |
|
try: |
|
logging.warning( |
|
'Attempting to upload the report to %s...', |
|
urlparse.urlparse(report_url).netloc) |
|
resp = urllib2.urlopen(req, timeout=5) |
|
report_id = None |
|
try: |
|
report_id = json.load(resp)['report_id'] |
|
except (ValueError, TypeError, KeyError): |
|
pass |
|
logging.warning('Report uploaded: %s', report_id) |
|
success = True |
|
except (urllib2.URLError, socket.error, ssl.SSLError) as exc: |
|
logging.warning('Failed to upload the report: %s', exc) |
|
return success |
|
|
|
|
|
def main(args): |
|
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) |
|
parser.add_option( |
|
'--running-as-hook', |
|
action='store_true', |
|
help='Set when invoked from gclient hook') |
|
parser.add_option( |
|
'--report-url', |
|
default=MOTHERSHIP_URL, |
|
help='URL to submit the report to') |
|
parser.add_option( |
|
'--verbose', |
|
action='store_true', |
|
help='More logging') |
|
options, args = parser.parse_args() |
|
if args: |
|
parser.error('Unknown argument %s' % args) |
|
logging.basicConfig( |
|
format='%(message)s', |
|
level=logging.INFO if options.verbose else logging.WARN) |
|
|
|
# When invoked not as a hook, always run the check. |
|
if not options.running_as_hook: |
|
config = scan_configuration() |
|
check_gclient_config(config) |
|
check_git_config(config, options.report_url, True) |
|
return 0 |
|
|
|
# Always do nothing on bots. |
|
if is_on_bot(): |
|
return 0 |
|
|
|
# Read current config, verify gclient solution looks correct. |
|
config = scan_configuration() |
|
check_gclient_config(config) |
|
|
|
# Do not attempt to push from non-google owned machines. |
|
if not is_in_google_corp(): |
|
logging.info('Skipping git push check: non *.corp.google.com machine.') |
|
return 0 |
|
|
|
# Skip git push check if current configuration was already checked. |
|
if config == read_last_configuration(): |
|
logging.info('Check already performed, skipping.') |
|
return 0 |
|
|
|
# Run the check. Mark configuration as checked only on success. Ignore any |
|
# exceptions or errors. This check must not break gclient runhooks. |
|
try: |
|
ok = check_git_config(config, options.report_url, False) |
|
if ok: |
|
write_last_configuration(config) |
|
else: |
|
logging.warning('Check failed and will be retried on the next run') |
|
except Exception: |
|
logging.exception('Unexpected exception when performing git access check') |
|
return 0 |
|
|
|
|
|
if __name__ == '__main__': |
|
sys.exit(main(sys.argv[1:])) |