blob: 3d4f476c153cbf08535f6e08e65973ea01f90a97 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Copyright (C) 2008 Evan Martin <martine@danga.com>
"""A git-command for integrating reviews on Rietveld and Gerrit."""
from __future__ import print_function
from distutils.version import LooseVersion
from multiprocessing.pool import ThreadPool
import base64
import collections
import contextlib
import datetime
import fnmatch
import httplib
import itertools
import json
import logging
import multiprocessing
import optparse
import os
import re
import shutil
import stat
import sys
import tempfile
import textwrap
import urllib
import urllib2
import urlparse
import uuid
import webbrowser
import zlib
try:
import readline # pylint: disable=import-error,W0611
except ImportError:
pass
from third_party import colorama
from third_party import httplib2
from third_party import upload
import auth
import checkout
import clang_format
import dart_format
import setup_color
import fix_encoding
import gclient_utils
import gerrit_util
import git_cache
import git_common
import git_footers
import owners
import owners_finder
import presubmit_support
import rietveld
import scm
import split_cl
import subcommand
import subprocess2
import watchlists
__version__ = '2.0'
COMMIT_BOT_EMAIL = 'commit-bot@chromium.org'
DEFAULT_SERVER = 'https://codereview.chromium.org'
POSTUPSTREAM_HOOK = '.git/hooks/post-cl-land'
DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup'
REFS_THAT_ALIAS_TO_OTHER_REFS = {
'refs/remotes/origin/lkgr': 'refs/remotes/origin/master',
'refs/remotes/origin/lkcr': 'refs/remotes/origin/master',
}
# Valid extensions for files we want to lint.
DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)"
DEFAULT_LINT_IGNORE_REGEX = r"$^"
# Buildbucket master name prefix.
MASTER_PREFIX = 'master.'
# Shortcut since it quickly becomes redundant.
Fore = colorama.Fore
# Initialized in main()
settings = None
# Used by tests/git_cl_test.py to add extra logging.
# Inside the weirdly failing test, add this:
# >>> self.mock(git_cl, '_IS_BEING_TESTED', True)
# And scroll up to see the stack trace printed.
_IS_BEING_TESTED = False
def DieWithError(message, change_desc=None):
if change_desc:
SaveDescriptionBackup(change_desc)
print(message, file=sys.stderr)
sys.exit(1)
def SaveDescriptionBackup(change_desc):
backup_path = os.path.expanduser(DESCRIPTION_BACKUP_FILE)
print('\nError after CL description prompt -- saving description to %s\n' %
backup_path)
backup_file = open(backup_path, 'w')
backup_file.write(change_desc.description)
backup_file.close()
def GetNoGitPagerEnv():
env = os.environ.copy()
# 'cat' is a magical git string that disables pagers on all platforms.
env['GIT_PAGER'] = 'cat'
return env
def RunCommand(args, error_ok=False, error_message=None, shell=False, **kwargs):
try:
return subprocess2.check_output(args, shell=shell, **kwargs)
except subprocess2.CalledProcessError as e:
logging.debug('Failed running %s', args)
if not error_ok:
DieWithError(
'Command "%s" failed.\n%s' % (
' '.join(args), error_message or e.stdout or ''))
return e.stdout
def RunGit(args, **kwargs):
"""Returns stdout."""
return RunCommand(['git'] + args, **kwargs)
def RunGitWithCode(args, suppress_stderr=False):
"""Returns return code and stdout."""
if suppress_stderr:
stderr = subprocess2.VOID
else:
stderr = sys.stderr
try:
(out, _), code = subprocess2.communicate(['git'] + args,
env=GetNoGitPagerEnv(),
stdout=subprocess2.PIPE,
stderr=stderr)
return code, out
except subprocess2.CalledProcessError as e:
logging.debug('Failed running %s', ['git'] + args)
return e.returncode, e.stdout
def RunGitSilent(args):
"""Returns stdout, suppresses stderr and ignores the return code."""
return RunGitWithCode(args, suppress_stderr=True)[1]
def IsGitVersionAtLeast(min_version):
prefix = 'git version '
version = RunGit(['--version']).strip()
return (version.startswith(prefix) and
LooseVersion(version[len(prefix):]) >= LooseVersion(min_version))
def BranchExists(branch):
"""Return True if specified branch exists."""
code, _ = RunGitWithCode(['rev-parse', '--verify', branch],
suppress_stderr=True)
return not code
def time_sleep(seconds):
# Use this so that it can be mocked in tests without interfering with python
# system machinery.
import time # Local import to discourage others from importing time globally.
return time.sleep(seconds)
def ask_for_data(prompt):
try:
return raw_input(prompt)
except KeyboardInterrupt:
# Hide the exception.
sys.exit(1)
def confirm_or_exit(prefix='', action='confirm'):
"""Asks user to press enter to continue or press Ctrl+C to abort."""
if not prefix or prefix.endswith('\n'):
mid = 'Press'
elif prefix.endswith('.') or prefix.endswith('?'):
mid = ' Press'
elif prefix.endswith(' '):
mid = 'press'
else:
mid = ' press'
ask_for_data('%s%s Enter to %s, or Ctrl+C to abort' % (prefix, mid, action))
def ask_for_explicit_yes(prompt):
"""Returns whether user typed 'y' or 'yes' to confirm the given prompt"""
result = ask_for_data(prompt + ' [Yes/No]: ').lower()
while True:
if 'yes'.startswith(result):
return True
if 'no'.startswith(result):
return False
result = ask_for_data('Please, type yes or no: ').lower()
def _git_branch_config_key(branch, key):
"""Helper method to return Git config key for a branch."""
assert branch, 'branch name is required to set git config for it'
return 'branch.%s.%s' % (branch, key)
def _git_get_branch_config_value(key, default=None, value_type=str,
branch=False):
"""Returns git config value of given or current branch if any.
Returns default in all other cases.
"""
assert value_type in (int, str, bool)
if branch is False: # Distinguishing default arg value from None.
branch = GetCurrentBranch()
if not branch:
return default
args = ['config']
if value_type == bool:
args.append('--bool')
# git config also has --int, but apparently git config suffers from integer
# overflows (http://crbug.com/640115), so don't use it.
args.append(_git_branch_config_key(branch, key))
code, out = RunGitWithCode(args)
if code == 0:
value = out.strip()
if value_type == int:
return int(value)
if value_type == bool:
return bool(value.lower() == 'true')
return value
return default
def _git_set_branch_config_value(key, value, branch=None, **kwargs):
"""Sets the value or unsets if it's None of a git branch config.
Valid, though not necessarily existing, branch must be provided,
otherwise currently checked out branch is used.
"""
if not branch:
branch = GetCurrentBranch()
assert branch, 'a branch name OR currently checked out branch is required'
args = ['config']
# Check for boolean first, because bool is int, but int is not bool.
if value is None:
args.append('--unset')
elif isinstance(value, bool):
args.append('--bool')
value = str(value).lower()
else:
# git config also has --int, but apparently git config suffers from integer
# overflows (http://crbug.com/640115), so don't use it.
value = str(value)
args.append(_git_branch_config_key(branch, key))
if value is not None:
args.append(value)
RunGit(args, **kwargs)
def _get_committer_timestamp(commit):
"""Returns Unix timestamp as integer of a committer in a commit.
Commit can be whatever git show would recognize, such as HEAD, sha1 or ref.
"""
# Git also stores timezone offset, but it only affects visual display,
# actual point in time is defined by this timestamp only.
return int(RunGit(['show', '-s', '--format=%ct', commit]).strip())
def _git_amend_head(message, committer_timestamp):
"""Amends commit with new message and desired committer_timestamp.
Sets committer timezone to UTC.
"""
env = os.environ.copy()
env['GIT_COMMITTER_DATE'] = '%d+0000' % committer_timestamp
return RunGit(['commit', '--amend', '-m', message], env=env)
def _get_properties_from_options(options):
properties = dict(x.split('=', 1) for x in options.properties)
for key, val in properties.iteritems():
try:
properties[key] = json.loads(val)
except ValueError:
pass # If a value couldn't be evaluated, treat it as a string.
return properties
def _prefix_master(master):
"""Convert user-specified master name to full master name.
Buildbucket uses full master name(master.tryserver.chromium.linux) as bucket
name, while the developers always use shortened master name
(tryserver.chromium.linux) by stripping off the prefix 'master.'. This
function does the conversion for buildbucket migration.
"""
if master.startswith(MASTER_PREFIX):
return master
return '%s%s' % (MASTER_PREFIX, master)
def _unprefix_master(bucket):
"""Convert bucket name to shortened master name.
Buildbucket uses full master name(master.tryserver.chromium.linux) as bucket
name, while the developers always use shortened master name
(tryserver.chromium.linux) by stripping off the prefix 'master.'. This
function does the conversion for buildbucket migration.
"""
if bucket.startswith(MASTER_PREFIX):
return bucket[len(MASTER_PREFIX):]
return bucket
def _buildbucket_retry(operation_name, http, *args, **kwargs):
"""Retries requests to buildbucket service and returns parsed json content."""
try_count = 0
while True:
response, content = http.request(*args, **kwargs)
try:
content_json = json.loads(content)
except ValueError:
content_json = None
# Buildbucket could return an error even if status==200.
if content_json and content_json.get('error'):
error = content_json.get('error')
if error.get('code') == 403:
raise BuildbucketResponseException(
'Access denied: %s' % error.get('message', ''))
msg = 'Error in response. Reason: %s. Message: %s.' % (
error.get('reason', ''), error.get('message', ''))
raise BuildbucketResponseException(msg)
if response.status == 200:
if not content_json:
raise BuildbucketResponseException(
'Buildbucket returns invalid json content: %s.\n'
'Please file bugs at http://crbug.com, label "Infra-BuildBucket".' %
content)
return content_json
if response.status < 500 or try_count >= 2:
raise httplib2.HttpLib2Error(content)
# status >= 500 means transient failures.
logging.debug('Transient errors when %s. Will retry.', operation_name)
time_sleep(0.5 + 1.5*try_count)
try_count += 1
assert False, 'unreachable'
def _get_bucket_map(changelist, options, option_parser):
"""Returns a dict mapping bucket names to builders and tests,
for triggering try jobs.
"""
# If no bots are listed, we try to get a set of builders and tests based
# on GetPreferredTryMasters functions in PRESUBMIT.py files.
if not options.bot:
change = changelist.GetChange(
changelist.GetCommonAncestorWithUpstream(), None)
# Get try masters from PRESUBMIT.py files.
masters = presubmit_support.DoGetTryMasters(
change=change,
changed_files=change.LocalPaths(),
repository_root=settings.GetRoot(),
default_presubmit=None,
project=None,
verbose=options.verbose,
output_stream=sys.stdout)
if masters is None:
return None
return {_prefix_master(m): b for m, b in masters.iteritems()}
if options.bucket:
return {options.bucket: {b: [] for b in options.bot}}
if options.master:
return {_prefix_master(options.master): {b: [] for b in options.bot}}
# If bots are listed but no master or bucket, then we need to find out
# the corresponding master for each bot.
bucket_map, error_message = _get_bucket_map_for_builders(options.bot)
if error_message:
option_parser.error(
'Tryserver master cannot be found because: %s\n'
'Please manually specify the tryserver master, e.g. '
'"-m tryserver.chromium.linux".' % error_message)
return bucket_map
def _get_bucket_map_for_builders(builders):
"""Returns a map of buckets to builders for the given builders."""
map_url = 'https://builders-map.appspot.com/'
try:
builders_map = json.load(urllib2.urlopen(map_url))
except urllib2.URLError as e:
return None, ('Failed to fetch builder-to-master map from %s. Error: %s.' %
(map_url, e))
except ValueError as e:
return None, ('Invalid json string from %s. Error: %s.' % (map_url, e))
if not builders_map:
return None, 'Failed to build master map.'
bucket_map = {}
for builder in builders:
bucket = builders_map.get(builder, {}).get('bucket')
if bucket:
bucket_map.setdefault(bucket, {})[builder] = []
return bucket_map, None
def _trigger_try_jobs(auth_config, changelist, buckets, options, patchset):
"""Sends a request to Buildbucket to trigger try jobs for a changelist.
Args:
auth_config: AuthConfig for Buildbucket.
changelist: Changelist that the try jobs are associated with.
buckets: A nested dict mapping bucket names to builders to tests.
options: Command-line options.
"""
assert changelist.GetIssue(), 'CL must be uploaded first'
codereview_url = changelist.GetCodereviewServer()
assert codereview_url, 'CL must be uploaded first'
patchset = patchset or changelist.GetMostRecentPatchset()
assert patchset, 'CL must be uploaded first'
codereview_host = urlparse.urlparse(codereview_url).hostname
# Cache the buildbucket credentials under the codereview host key, so that
# users can use different credentials for different buckets.
authenticator = auth.get_authenticator_for_host(codereview_host, auth_config)
http = authenticator.authorize(httplib2.Http())
http.force_exception_to_status_code = True
buildbucket_put_url = (
'https://{hostname}/_ah/api/buildbucket/v1/builds/batch'.format(
hostname=options.buildbucket_host))
buildset = 'patch/{codereview}/{hostname}/{issue}/{patch}'.format(
codereview='gerrit' if changelist.IsGerrit() else 'rietveld',
hostname=codereview_host,
issue=changelist.GetIssue(),
patch=patchset)
shared_parameters_properties = changelist.GetTryJobProperties(patchset)
shared_parameters_properties['category'] = options.category
if options.clobber:
shared_parameters_properties['clobber'] = True
extra_properties = _get_properties_from_options(options)
if extra_properties:
shared_parameters_properties.update(extra_properties)
batch_req_body = {'builds': []}
print_text = []
print_text.append('Tried jobs on:')
for bucket, builders_and_tests in sorted(buckets.iteritems()):
print_text.append('Bucket: %s' % bucket)
master = None
if bucket.startswith(MASTER_PREFIX):
master = _unprefix_master(bucket)
for builder, tests in sorted(builders_and_tests.iteritems()):
print_text.append(' %s: %s' % (builder, tests))
parameters = {
'builder_name': builder,
'changes': [{
'author': {'email': changelist.GetIssueOwner()},
'revision': options.revision,
}],
'properties': shared_parameters_properties.copy(),
}
if 'presubmit' in builder.lower():
parameters['properties']['dry_run'] = 'true'
if tests:
parameters['properties']['testfilter'] = tests
tags = [
'builder:%s' % builder,
'buildset:%s' % buildset,
'user_agent:git_cl_try',
]
if master:
parameters['properties']['master'] = master
tags.append('master:%s' % master)
batch_req_body['builds'].append(
{
'bucket': bucket,
'parameters_json': json.dumps(parameters),
'client_operation_id': str(uuid.uuid4()),
'tags': tags,
}
)
_buildbucket_retry(
'triggering try jobs',
http,
buildbucket_put_url,
'PUT',
body=json.dumps(batch_req_body),
headers={'Content-Type': 'application/json'}
)
print_text.append('To see results here, run: git cl try-results')
print_text.append('To see results in browser, run: git cl web')
print('\n'.join(print_text))
def fetch_try_jobs(auth_config, changelist, buildbucket_host,
patchset=None):
"""Fetches try jobs from buildbucket.
Returns a map from build id to build info as a dictionary.
"""
assert buildbucket_host
assert changelist.GetIssue(), 'CL must be uploaded first'
assert changelist.GetCodereviewServer(), 'CL must be uploaded first'
patchset = patchset or changelist.GetMostRecentPatchset()
assert patchset, 'CL must be uploaded first'
codereview_url = changelist.GetCodereviewServer()
codereview_host = urlparse.urlparse(codereview_url).hostname
authenticator = auth.get_authenticator_for_host(codereview_host, auth_config)
if authenticator.has_cached_credentials():
http = authenticator.authorize(httplib2.Http())
else:
print('Warning: Some results might be missing because %s' %
# Get the message on how to login.
(auth.LoginRequiredError(codereview_host).message,))
http = httplib2.Http()
http.force_exception_to_status_code = True
buildset = 'patch/{codereview}/{hostname}/{issue}/{patch}'.format(
codereview='gerrit' if changelist.IsGerrit() else 'rietveld',
hostname=codereview_host,
issue=changelist.GetIssue(),
patch=patchset)
params = {'tag': 'buildset:%s' % buildset}
builds = {}
while True:
url = 'https://{hostname}/_ah/api/buildbucket/v1/search?{params}'.format(
hostname=buildbucket_host,
params=urllib.urlencode(params))
content = _buildbucket_retry('fetching try jobs', http, url, 'GET')
for build in content.get('builds', []):
builds[build['id']] = build
if 'next_cursor' in content:
params['start_cursor'] = content['next_cursor']
else:
break
return builds
def print_try_jobs(options, builds):
"""Prints nicely result of fetch_try_jobs."""
if not builds:
print('No try jobs scheduled.')
return
# Make a copy, because we'll be modifying builds dictionary.
builds = builds.copy()
builder_names_cache = {}
def get_builder(b):
try:
return builder_names_cache[b['id']]
except KeyError:
try:
parameters = json.loads(b['parameters_json'])
name = parameters['builder_name']
except (ValueError, KeyError) as error:
print('WARNING: Failed to get builder name for build %s: %s' % (
b['id'], error))
name = None
builder_names_cache[b['id']] = name
return name
def get_bucket(b):
bucket = b['bucket']
if bucket.startswith('master.'):
return bucket[len('master.'):]
return bucket
if options.print_master:
name_fmt = '%%-%ds %%-%ds' % (
max(len(str(get_bucket(b))) for b in builds.itervalues()),
max(len(str(get_builder(b))) for b in builds.itervalues()))
def get_name(b):
return name_fmt % (get_bucket(b), get_builder(b))
else:
name_fmt = '%%-%ds' % (
max(len(str(get_builder(b))) for b in builds.itervalues()))
def get_name(b):
return name_fmt % get_builder(b)
def sort_key(b):
return b['status'], b.get('result'), get_name(b), b.get('url')
def pop(title, f, color=None, **kwargs):
"""Pop matching builds from `builds` dict and print them."""
if not options.color or color is None:
colorize = str
else:
colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET)
result = []
for b in builds.values():
if all(b.get(k) == v for k, v in kwargs.iteritems()):
builds.pop(b['id'])
result.append(b)
if result:
print(colorize(title))
for b in sorted(result, key=sort_key):
print(' ', colorize('\t'.join(map(str, f(b)))))
total = len(builds)
pop(status='COMPLETED', result='SUCCESS',
title='Successes:', color=Fore.GREEN,
f=lambda b: (get_name(b), b.get('url')))
pop(status='COMPLETED', result='FAILURE', failure_reason='INFRA_FAILURE',
title='Infra Failures:', color=Fore.MAGENTA,
f=lambda b: (get_name(b), b.get('url')))
pop(status='COMPLETED', result='FAILURE', failure_reason='BUILD_FAILURE',
title='Failures:', color=Fore.RED,
f=lambda b: (get_name(b), b.get('url')))
pop(status='COMPLETED', result='CANCELED',
title='Canceled:', color=Fore.MAGENTA,
f=lambda b: (get_name(b),))
pop(status='COMPLETED', result='FAILURE',
failure_reason='INVALID_BUILD_DEFINITION',
title='Wrong master/builder name:', color=Fore.MAGENTA,
f=lambda b: (get_name(b),))
pop(status='COMPLETED', result='FAILURE',
title='Other failures:',
f=lambda b: (get_name(b), b.get('failure_reason'), b.get('url')))
pop(status='COMPLETED',
title='Other finished:',
f=lambda b: (get_name(b), b.get('result'), b.get('url')))
pop(status='STARTED',
title='Started:', color=Fore.YELLOW,
f=lambda b: (get_name(b), b.get('url')))
pop(status='SCHEDULED',
title='Scheduled:',
f=lambda b: (get_name(b), 'id=%s' % b['id']))
# The last section is just in case buildbucket API changes OR there is a bug.
pop(title='Other:',
f=lambda b: (get_name(b), 'id=%s' % b['id']))
assert len(builds) == 0
print('Total: %d try jobs' % total)
def write_try_results_json(output_file, builds):
"""Writes a subset of the data from fetch_try_jobs to a file as JSON.
The input |builds| dict is assumed to be generated by Buildbucket.
Buildbucket documentation: http://goo.gl/G0s101
"""
def convert_build_dict(build):
"""Extracts some of the information from one build dict."""
parameters = json.loads(build.get('parameters_json', '{}')) or {}
return {
'buildbucket_id': build.get('id'),
'bucket': build.get('bucket'),
'builder_name': parameters.get('builder_name'),
'created_ts': build.get('created_ts'),
'experimental': build.get('experimental'),
'failure_reason': build.get('failure_reason'),
'result': build.get('result'),
'status': build.get('status'),
'tags': build.get('tags'),
'url': build.get('url'),
}
converted = []
for _, build in sorted(builds.items()):
converted.append(convert_build_dict(build))
write_json(output_file, converted)
def print_stats(args):
"""Prints statistics about the change to the user."""
# --no-ext-diff is broken in some versions of Git, so try to work around
# this by overriding the environment (but there is still a problem if the
# git config key "diff.external" is used).
env = GetNoGitPagerEnv()
if 'GIT_EXTERNAL_DIFF' in env:
del env['GIT_EXTERNAL_DIFF']
try:
stdout = sys.stdout.fileno()
except AttributeError:
stdout = None
return subprocess2.call(
['git', 'diff', '--no-ext-diff', '--stat', '-l100000', '-C50'] + args,
stdout=stdout, env=env)
class BuildbucketResponseException(Exception):
pass
class Settings(object):
def __init__(self):
self.default_server = None
self.cc = None
self.root = None
self.tree_status_url = None
self.viewvc_url = None
self.updated = False
self.is_gerrit = None
self.squash_gerrit_uploads = None
self.gerrit_skip_ensure_authenticated = None
self.git_editor = None
self.project = None
self.force_https_commit_url = None
def LazyUpdateIfNeeded(self):
"""Updates the settings from a codereview.settings file, if available."""
if not self.updated:
# The only value that actually changes the behavior is
# autoupdate = "false". Everything else means "true".
autoupdate = RunGit(['config', 'rietveld.autoupdate'],
error_ok=True
).strip().lower()
cr_settings_file = FindCodereviewSettingsFile()
if autoupdate != 'false' and cr_settings_file:
LoadCodereviewSettingsFromFile(cr_settings_file)
self.updated = True
def GetDefaultServerUrl(self, error_ok=False):
if not self.default_server:
self.LazyUpdateIfNeeded()
self.default_server = gclient_utils.UpgradeToHttps(
self._GetRietveldConfig('server', error_ok=True))
if error_ok:
return self.default_server
if not self.default_server:
error_message = ('Could not find settings file. You must configure '
'your review setup by running "git cl config".')
self.default_server = gclient_utils.UpgradeToHttps(
self._GetRietveldConfig('server', error_message=error_message))
return self.default_server
@staticmethod
def GetRelativeRoot():
return RunGit(['rev-parse', '--show-cdup']).strip()
def GetRoot(self):
if self.root is None:
self.root = os.path.abspath(self.GetRelativeRoot())
return self.root
def GetGitMirror(self, remote='origin'):
"""If this checkout is from a local git mirror, return a Mirror object."""
local_url = RunGit(['config', '--get', 'remote.%s.url' % remote]).strip()
if not os.path.isdir(local_url):
return None
git_cache.Mirror.SetCachePath(os.path.dirname(local_url))
remote_url = git_cache.Mirror.CacheDirToUrl(local_url)
# Use the /dev/null print_func to avoid terminal spew.
mirror = git_cache.Mirror(remote_url, print_func=lambda *args: None)
if mirror.exists():
return mirror
return None
def GetTreeStatusUrl(self, error_ok=False):
if not self.tree_status_url:
error_message = ('You must configure your tree status URL by running '
'"git cl config".')
self.tree_status_url = self._GetRietveldConfig(
'tree-status-url', error_ok=error_ok, error_message=error_message)
return self.tree_status_url
def GetViewVCUrl(self):
if not self.viewvc_url:
self.viewvc_url = self._GetRietveldConfig('viewvc-url', error_ok=True)
return self.viewvc_url
def GetBugPrefix(self):
return self._GetRietveldConfig('bug-prefix', error_ok=True)
def GetIsSkipDependencyUpload(self, branch_name):
"""Returns true if specified branch should skip dep uploads."""
return self._GetBranchConfig(branch_name, 'skip-deps-uploads',
error_ok=True)
def GetRunPostUploadHook(self):
run_post_upload_hook = self._GetRietveldConfig(
'run-post-upload-hook', error_ok=True)
return run_post_upload_hook == "True"
def GetDefaultCCList(self):
return self._GetRietveldConfig('cc', error_ok=True)
def GetDefaultPrivateFlag(self):
return self._GetRietveldConfig('private', error_ok=True)
def GetIsGerrit(self):
"""Return true if this repo is associated with gerrit code review system."""
if self.is_gerrit is None:
self.is_gerrit = (
self._GetConfig('gerrit.host', error_ok=True).lower() == 'true')
return self.is_gerrit
def GetSquashGerritUploads(self):
"""Return true if uploads to Gerrit should be squashed by default."""
if self.squash_gerrit_uploads is None:
self.squash_gerrit_uploads = self.GetSquashGerritUploadsOverride()
if self.squash_gerrit_uploads is None:
# Default is squash now (http://crbug.com/611892#c23).
self.squash_gerrit_uploads = not (
RunGit(['config', '--bool', 'gerrit.squash-uploads'],
error_ok=True).strip() == 'false')
return self.squash_gerrit_uploads
def GetSquashGerritUploadsOverride(self):
"""Return True or False if codereview.settings should be overridden.
Returns None if no override has been defined.
"""
# See also http://crbug.com/611892#c23
result = RunGit(['config', '--bool', 'gerrit.override-squash-uploads'],
error_ok=True).strip()
if result == 'true':
return True
if result == 'false':
return False
return None
def GetGerritSkipEnsureAuthenticated(self):
"""Return True if EnsureAuthenticated should not be done for Gerrit
uploads."""
if self.gerrit_skip_ensure_authenticated is None:
self.gerrit_skip_ensure_authenticated = (
RunGit(['config', '--bool', 'gerrit.skip-ensure-authenticated'],
error_ok=True).strip() == 'true')
return self.gerrit_skip_ensure_authenticated
def GetGitEditor(self):
"""Return the editor specified in the git config, or None if none is."""
if self.git_editor is None:
self.git_editor = self._GetConfig('core.editor', error_ok=True)
return self.git_editor or None
def GetLintRegex(self):
return (self._GetRietveldConfig('cpplint-regex', error_ok=True) or
DEFAULT_LINT_REGEX)
def GetLintIgnoreRegex(self):
return (self._GetRietveldConfig('cpplint-ignore-regex', error_ok=True) or
DEFAULT_LINT_IGNORE_REGEX)
def GetProject(self):
if not self.project:
self.project = self._GetRietveldConfig('project', error_ok=True)
return self.project
def _GetRietveldConfig(self, param, **kwargs):
return self._GetConfig('rietveld.' + param, **kwargs)
def _GetBranchConfig(self, branch_name, param, **kwargs):
return self._GetConfig('branch.' + branch_name + '.' + param, **kwargs)
def _GetConfig(self, param, **kwargs):
self.LazyUpdateIfNeeded()
return RunGit(['config', param], **kwargs).strip()
@contextlib.contextmanager
def _get_gerrit_project_config_file(remote_url):
"""Context manager to fetch and store Gerrit's project.config from
refs/meta/config branch and store it in temp file.
Provides a temporary filename or None if there was error.
"""
error, _ = RunGitWithCode([
'fetch', remote_url,
'+refs/meta/config:refs/git_cl/meta/config'])
if error:
# Ref doesn't exist or isn't accessible to current user.
print('WARNING: Failed to fetch project config for %s: %s' %
(remote_url, error))
yield None
return
error, project_config_data = RunGitWithCode(
['show', 'refs/git_cl/meta/config:project.config'])
if error:
print('WARNING: project.config file not found')
yield None
return
with gclient_utils.temporary_directory() as tempdir:
project_config_file = os.path.join(tempdir, 'project.config')
gclient_utils.FileWrite(project_config_file, project_config_data)
yield project_config_file
def _is_git_numberer_enabled(remote_url, remote_ref):
"""Returns True if Git Numberer is enabled on this ref."""
# TODO(tandrii): this should be deleted once repos below are 100% on Gerrit.
KNOWN_PROJECTS_WHITELIST = [
'chromium/src',
'external/webrtc',
'v8/v8',
'infra/experimental',
# For webrtc.googlesource.com/src.
'src',
]
assert remote_ref and remote_ref.startswith('refs/'), remote_ref
url_parts = urlparse.urlparse(remote_url)
project_name = url_parts.path.lstrip('/').rstrip('git./')
for known in KNOWN_PROJECTS_WHITELIST:
if project_name.endswith(known):
break
else:
# Early exit to avoid extra fetches for repos that aren't using Git
# Numberer.
return False
with _get_gerrit_project_config_file(remote_url) as project_config_file:
if project_config_file is None:
# Failed to fetch project.config, which shouldn't happen on open source
# repos KNOWN_PROJECTS_WHITELIST.
return False
def get_opts(x):
code, out = RunGitWithCode(
['config', '-f', project_config_file, '--get-all',
'plugin.git-numberer.validate-%s-refglob' % x])
if code == 0:
return out.strip().splitlines()
return []
enabled, disabled = map(get_opts, ['enabled', 'disabled'])
logging.info('validator config enabled %s disabled %s refglobs for '
'(this ref: %s)', enabled, disabled, remote_ref)
def match_refglobs(refglobs):
for refglob in refglobs:
if remote_ref == refglob or fnmatch.fnmatch(remote_ref, refglob):
return True
return False
if match_refglobs(disabled):
return False
return match_refglobs(enabled)
def ShortBranchName(branch):
"""Convert a name like 'refs/heads/foo' to just 'foo'."""
return branch.replace('refs/heads/', '', 1)
def GetCurrentBranchRef():
"""Returns branch ref (e.g., refs/heads/master) or None."""
return RunGit(['symbolic-ref', 'HEAD'],
stderr=subprocess2.VOID, error_ok=True).strip() or None
def GetCurrentBranch():
"""Returns current branch or None.
For refs/heads/* branches, returns just last part. For others, full ref.
"""
branchref = GetCurrentBranchRef()
if branchref:
return ShortBranchName(branchref)
return None
class _CQState(object):
"""Enum for states of CL with respect to Commit Queue."""
NONE = 'none'
DRY_RUN = 'dry_run'
COMMIT = 'commit'
ALL_STATES = [NONE, DRY_RUN, COMMIT]
class _ParsedIssueNumberArgument(object):
def __init__(self, issue=None, patchset=None, hostname=None, codereview=None):
self.issue = issue
self.patchset = patchset
self.hostname = hostname
assert codereview in (None, 'rietveld', 'gerrit')
self.codereview = codereview
@property
def valid(self):
return self.issue is not None
def ParseIssueNumberArgument(arg, codereview=None):
"""Parses the issue argument and returns _ParsedIssueNumberArgument."""
fail_result = _ParsedIssueNumberArgument()
if arg.isdigit():
return _ParsedIssueNumberArgument(issue=int(arg), codereview=codereview)
if not arg.startswith('http'):
return fail_result
url = gclient_utils.UpgradeToHttps(arg)
try:
parsed_url = urlparse.urlparse(url)
except ValueError:
return fail_result
if codereview is not None:
parsed = _CODEREVIEW_IMPLEMENTATIONS[codereview].ParseIssueURL(parsed_url)
return parsed or fail_result
results = {}
for name, cls in _CODEREVIEW_IMPLEMENTATIONS.iteritems():
parsed = cls.ParseIssueURL(parsed_url)
if parsed is not None:
results[name] = parsed
if not results:
return fail_result
if len(results) == 1:
return results.values()[0]
if parsed_url.netloc and parsed_url.netloc.split('.')[0].endswith('-review'):
# This is likely Gerrit.
return results['gerrit']
# Choose Rietveld as before if URL can parsed by either.
return results['rietveld']
class GerritChangeNotExists(Exception):
def __init__(self, issue, url):
self.issue = issue
self.url = url
super(GerritChangeNotExists, self).__init__()
def __str__(self):
return 'change %s at %s does not exist or you have no access to it' % (
self.issue, self.url)
_CommentSummary = collections.namedtuple(
'_CommentSummary', ['date', 'message', 'sender',
# TODO(tandrii): these two aren't known in Gerrit.
'approval', 'disapproval'])
class Changelist(object):
"""Changelist works with one changelist in local branch.
Supports two codereview backends: Rietveld or Gerrit, selected at object
creation.
Notes:
* Not safe for concurrent multi-{thread,process} use.
* Caches values from current branch. Therefore, re-use after branch change
with great care.
"""
def __init__(self, branchref=None, issue=None, codereview=None, **kwargs):
"""Create a new ChangeList instance.
If issue is given, the codereview must be given too.
If `codereview` is given, it must be 'rietveld' or 'gerrit'.
Otherwise, it's decided based on current configuration of the local branch,
with default being 'rietveld' for backwards compatibility.
See _load_codereview_impl for more details.
**kwargs will be passed directly to codereview implementation.
"""
# Poke settings so we get the "configure your server" message if necessary.
global settings
if not settings:
# Happens when git_cl.py is used as a utility library.
settings = Settings()
if issue:
assert codereview, 'codereview must be known, if issue is known'
self.branchref = branchref
if self.branchref:
assert branchref.startswith('refs/heads/')
self.branch = ShortBranchName(self.branchref)
else:
self.branch = None
self.upstream_branch = None
self.lookedup_issue = False
self.issue = issue or None
self.has_description = False
self.description = None
self.lookedup_patchset = False
self.patchset = None
self.cc = None
self.more_cc = []
self._remote = None
self._codereview_impl = None
self._codereview = None
self._load_codereview_impl(codereview, **kwargs)
assert self._codereview_impl
assert self._codereview in _CODEREVIEW_IMPLEMENTATIONS
def _load_codereview_impl(self, codereview=None, **kwargs):
if codereview:
assert codereview in _CODEREVIEW_IMPLEMENTATIONS
cls = _CODEREVIEW_IMPLEMENTATIONS[codereview]
self._codereview = codereview
self._codereview_impl = cls(self, **kwargs)
return
# Automatic selection based on issue number set for a current branch.
# Rietveld takes precedence over Gerrit.
assert not self.issue
# Whether we find issue or not, we are doing the lookup.
self.lookedup_issue = True
if self.GetBranch():
for codereview, cls in _CODEREVIEW_IMPLEMENTATIONS.iteritems():
issue = _git_get_branch_config_value(
cls.IssueConfigKey(), value_type=int, branch=self.GetBranch())
if issue:
self._codereview = codereview
self._codereview_impl = cls(self, **kwargs)
self.issue = int(issue)
return
# No issue is set for this branch, so decide based on repo-wide settings.
return self._load_codereview_impl(
codereview='gerrit' if settings.GetIsGerrit() else 'rietveld',
**kwargs)
def IsGerrit(self):
return self._codereview == 'gerrit'
def GetCCList(self):
"""Returns the users cc'd on this CL.
The return value is a string suitable for passing to git cl with the --cc
flag.
"""
if self.cc is None:
base_cc = settings.GetDefaultCCList()
more_cc = ','.join(self.more_cc)
self.cc = ','.join(filter(None, (base_cc, more_cc))) or ''
return self.cc
def GetCCListWithoutDefault(self):
"""Return the users cc'd on this CL excluding default ones."""
if self.cc is None:
self.cc = ','.join(self.more_cc)
return self.cc
def ExtendCC(self, more_cc):
"""Extends the list of users to cc on this CL based on the changed files."""
self.more_cc.extend(more_cc)
def GetBranch(self):
"""Returns the short branch name, e.g. 'master'."""
if not self.branch:
branchref = GetCurrentBranchRef()
if not branchref:
return None
self.branchref = branchref
self.branch = ShortBranchName(self.branchref)
return self.branch
def GetBranchRef(self):
"""Returns the full branch name, e.g. 'refs/heads/master'."""
self.GetBranch() # Poke the lazy loader.
return self.branchref
def ClearBranch(self):
"""Clears cached branch data of this object."""
self.branch = self.branchref = None
def _GitGetBranchConfigValue(self, key, default=None, **kwargs):
assert 'branch' not in kwargs, 'this CL branch is used automatically'
kwargs['branch'] = self.GetBranch()
return _git_get_branch_config_value(key, default, **kwargs)
def _GitSetBranchConfigValue(self, key, value, **kwargs):
assert 'branch' not in kwargs, 'this CL branch is used automatically'
assert self.GetBranch(), (
'this CL must have an associated branch to %sset %s%s' %
('un' if value is None else '',
key,
'' if value is None else ' to %r' % value))
kwargs['branch'] = self.GetBranch()
return _git_set_branch_config_value(key, value, **kwargs)
@staticmethod
def FetchUpstreamTuple(branch):
"""Returns a tuple containing remote and remote ref,
e.g. 'origin', 'refs/heads/master'
"""
remote = '.'
upstream_branch = _git_get_branch_config_value('merge', branch=branch)
if upstream_branch:
remote = _git_get_branch_config_value('remote', branch=branch)
else:
upstream_branch = RunGit(['config', 'rietveld.upstream-branch'],
error_ok=True).strip()
if upstream_branch:
remote = RunGit(['config', 'rietveld.upstream-remote']).strip()
else:
# Else, try to guess the origin remote.
remote_branches = RunGit(['branch', '-r']).split()
if 'origin/master' in remote_branches:
# Fall back on origin/master if it exits.
remote = 'origin'
upstream_branch = 'refs/heads/master'
else:
DieWithError(
'Unable to determine default branch to diff against.\n'
'Either pass complete "git diff"-style arguments, like\n'
' git cl upload origin/master\n'
'or verify this branch is set up to track another \n'
'(via the --track argument to "git checkout -b ...").')
return remote, upstream_branch
def GetCommonAncestorWithUpstream(self):
upstream_branch = self.GetUpstreamBranch()
if not BranchExists(upstream_branch):
DieWithError('The upstream for the current branch (%s) does not exist '
'anymore.\nPlease fix it and try again.' % self.GetBranch())
return git_common.get_or_create_merge_base(self.GetBranch(),
upstream_branch)
def GetUpstreamBranch(self):
if self.upstream_branch is None:
remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch())
if remote is not '.':
upstream_branch = upstream_branch.replace('refs/heads/',
'refs/remotes/%s/' % remote)
upstream_branch = upstream_branch.replace('refs/branch-heads/',
'refs/remotes/branch-heads/')
self.upstream_branch = upstream_branch
return self.upstream_branch
def GetRemoteBranch(self):
if not self._remote:
remote, branch = None, self.GetBranch()
seen_branches = set()
while branch not in seen_branches:
seen_branches.add(branch)
remote, branch = self.FetchUpstreamTuple(branch)
branch = ShortBranchName(branch)
if remote != '.' or branch.startswith('refs/remotes'):
break
else:
remotes = RunGit(['remote'], error_ok=True).split()
if len(remotes) == 1:
remote, = remotes
elif 'origin' in remotes:
remote = 'origin'
logging.warn('Could not determine which remote this change is '
'associated with, so defaulting to "%s".' % self._remote)
else:
logging.warn('Could not determine which remote this change is '
'associated with.')
branch = 'HEAD'
if branch.startswith('refs/remotes'):
self._remote = (remote, branch)
elif branch.startswith('refs/branch-heads/'):
self._remote = (remote, branch.replace('refs/', 'refs/remotes/'))
else:
self._remote = (remote, 'refs/remotes/%s/%s' % (remote, branch))
return self._remote
def GitSanityChecks(self, upstream_git_obj):
"""Checks git repo status and ensures diff is from local commits."""
if upstream_git_obj is None:
if self.GetBranch() is None:
print('ERROR: Unable to determine current branch (detached HEAD?)',
file=sys.stderr)
else:
print('ERROR: No upstream branch.', file=sys.stderr)
return False
# Verify the commit we're diffing against is in our current branch.
upstream_sha = RunGit(['rev-parse', '--verify', upstream_git_obj]).strip()
common_ancestor = RunGit(['merge-base', upstream_sha, 'HEAD']).strip()
if upstream_sha != common_ancestor:
print('ERROR: %s is not in the current branch. You may need to rebase '
'your tracking branch' % upstream_sha, file=sys.stderr)
return False
# List the commits inside the diff, and verify they are all local.
commits_in_diff = RunGit(
['rev-list', '^%s' % upstream_sha, 'HEAD']).splitlines()
code, remote_branch = RunGitWithCode(['config', 'gitcl.remotebranch'])
remote_branch = remote_branch.strip()
if code != 0:
_, remote_branch = self.GetRemoteBranch()
commits_in_remote = RunGit(
['rev-list', '^%s' % upstream_sha, remote_branch]).splitlines()
common_commits = set(commits_in_diff) & set(commits_in_remote)
if common_commits:
print('ERROR: Your diff contains %d commits already in %s.\n'
'Run "git log --oneline %s..HEAD" to get a list of commits in '
'the diff. If you are using a custom git flow, you can override'
' the reference used for this check with "git config '
'gitcl.remotebranch <git-ref>".' % (
len(common_commits), remote_branch, upstream_git_obj),
file=sys.stderr)
return False
return True
def GetGitBaseUrlFromConfig(self):
"""Return the configured base URL from branch.<branchname>.baseurl.
Returns None if it is not set.
"""
return self._GitGetBranchConfigValue('base-url')
def GetRemoteUrl(self):
"""Return the configured remote URL, e.g. 'git://example.org/foo.git/'.
Returns None if there is no remote.
"""
remote, _ = self.GetRemoteBranch()
url = RunGit(['config', 'remote.%s.url' % remote], error_ok=True).strip()
# If URL is pointing to a local directory, it is probably a git cache.
if os.path.isdir(url):
url = RunGit(['config', 'remote.%s.url' % remote],
error_ok=True,
cwd=url).strip()
return url
def GetIssue(self):
"""Returns the issue number as a int or None if not set."""
if self.issue is None and not self.lookedup_issue:
self.issue = self._GitGetBranchConfigValue(
self._codereview_impl.IssueConfigKey(), value_type=int)
self.lookedup_issue = True
return self.issue
def GetIssueURL(self):
"""Get the URL for a particular issue."""
issue = self.GetIssue()
if not issue:
return None
return '%s/%s' % (self._codereview_impl.GetCodereviewServer(), issue)
def GetDescription(self, pretty=False, force=False):
if not self.has_description or force:
if self.GetIssue():
self.description = self._codereview_impl.FetchDescription(force=force)
self.has_description = True
if pretty:
# Set width to 72 columns + 2 space indent.
wrapper = textwrap.TextWrapper(width=74, replace_whitespace=True)
wrapper.initial_indent = wrapper.subsequent_indent = ' '
lines = self.description.splitlines()
return '\n'.join([wrapper.fill(line) for line in lines])
return self.description
def GetDescriptionFooters(self):
"""Returns (non_footer_lines, footers) for the commit message.
Returns:
non_footer_lines (list(str)) - Simple list of description lines without
any footer. The lines do not contain newlines, nor does the list contain
the empty line between the message and the footers.
footers (list(tuple(KEY, VALUE))) - List of parsed footers, e.g.
[("Change-Id", "Ideadbeef...."), ...]
"""
raw_description = self.GetDescription()
msg_lines, _, footers = git_footers.split_footers(raw_description)
if footers:
msg_lines = msg_lines[:len(msg_lines)-1]
return msg_lines, footers
def GetPatchset(self):
"""Returns the patchset number as a int or None if not set."""
if self.patchset is None and not self.lookedup_patchset:
self.patchset = self._GitGetBranchConfigValue(
self._codereview_impl.PatchsetConfigKey(), value_type=int)
self.lookedup_patchset = True
return self.patchset
def SetPatchset(self, patchset):
"""Set this branch's patchset. If patchset=0, clears the patchset."""
assert self.GetBranch()
if not patchset:
self.patchset = None
else:
self.patchset = int(patchset)
self._GitSetBranchConfigValue(
self._codereview_impl.PatchsetConfigKey(), self.patchset)
def SetIssue(self, issue=None):
"""Set this branch's issue. If issue isn't given, clears the issue."""
assert self.GetBranch()
if issue:
issue = int(issue)
self._GitSetBranchConfigValue(
self._codereview_impl.IssueConfigKey(), issue)
self.issue = issue
codereview_server = self._codereview_impl.GetCodereviewServer()
if codereview_server:
self._GitSetBranchConfigValue(
self._codereview_impl.CodereviewServerConfigKey(),
codereview_server)
else:
# Reset all of these just to be clean.
reset_suffixes = [
'last-upload-hash',
self._codereview_impl.IssueConfigKey(),
self._codereview_impl.PatchsetConfigKey(),
self._codereview_impl.CodereviewServerConfigKey(),
] + self._PostUnsetIssueProperties()
for prop in reset_suffixes:
self._GitSetBranchConfigValue(prop, None, error_ok=True)
msg = RunGit(['log', '-1', '--format=%B']).strip()
if msg and git_footers.get_footer_change_id(msg):
print('WARNING: The change patched into this branch has a Change-Id. '
'Removing it.')
RunGit(['commit', '--amend', '-m',
git_footers.remove_footer(msg, 'Change-Id')])
self.issue = None
self.patchset = None
def GetChange(self, upstream_branch, author, local_description=False):
if not self.GitSanityChecks(upstream_branch):
DieWithError('\nGit sanity check failure')
root = settings.GetRelativeRoot()
if not root:
root = '.'
absroot = os.path.abspath(root)
# We use the sha1 of HEAD as a name of this change.
name = RunGitWithCode(['rev-parse', 'HEAD'])[1].strip()
# Need to pass a relative path for msysgit.
try:
files = scm.GIT.CaptureStatus([root], '.', upstream_branch)
except subprocess2.CalledProcessError:
DieWithError(
('\nFailed to diff against upstream branch %s\n\n'
'This branch probably doesn\'t exist anymore. To reset the\n'
'tracking branch, please run\n'
' git branch --set-upstream-to origin/master %s\n'
'or replace origin/master with the relevant branch') %
(upstream_branch, self.GetBranch()))
issue = self.GetIssue()
patchset = self.GetPatchset()
if issue and not local_description:
description = self.GetDescription()
else:
# If the change was never uploaded, use the log messages of all commits
# up to the branch point, as git cl upload will prefill the description
# with these log messages.
args = ['log', '--pretty=format:%s%n%n%b', '%s...' % (upstream_branch)]
description = RunGitWithCode(args)[1].strip()
if not author:
author = RunGit(['config', 'user.email']).strip() or None
return presubmit_support.GitChange(
name,
description,
absroot,
files,
issue,
patchset,
author,
upstream=upstream_branch)
def UpdateDescription(self, description, force=False):
self._codereview_impl.UpdateDescriptionRemote(description, force=force)
self.description = description
self.has_description = True
def UpdateDescriptionFooters(self, description_lines, footers, force=False):
"""Sets the description for this CL remotely.
You can get description_lines and footers with GetDescriptionFooters.
Args:
description_lines (list(str)) - List of CL description lines without
newline characters.
footers (list(tuple(KEY, VALUE))) - List of footers, as returned by
GetDescriptionFooters. Key must conform to the git footers format (i.e.
`List-Of-Tokens`). It will be case-normalized so that each token is
title-cased.
"""
new_description = '\n'.join(description_lines)
if footers:
new_description += '\n'
for k, v in footers:
foot = '%s: %s' % (git_footers.normalize_name(k), v)
if not git_footers.FOOTER_PATTERN.match(foot):
raise ValueError('Invalid footer %r' % foot)
new_description += foot + '\n'
self.UpdateDescription(new_description, force)
def RunHook(self, committing, may_prompt, verbose, change, parallel):
"""Calls sys.exit() if the hook fails; returns a HookResults otherwise."""
try:
return presubmit_support.DoPresubmitChecks(change, committing,
verbose=verbose, output_stream=sys.stdout, input_stream=sys.stdin,
default_presubmit=None, may_prompt=may_prompt,
gerrit_obj=self._codereview_impl.GetGerritObjForPresubmit(),
parallel=parallel)
except presubmit_support.PresubmitFailure as e:
DieWithError('%s\nMaybe your depot_tools is out of date?' % e)
def CMDPatchIssue(self, issue_arg, reject, nocommit, directory):
"""Fetches and applies the issue patch from codereview to local branch."""
if isinstance(issue_arg, (int, long)) or issue_arg.isdigit():
parsed_issue_arg = _ParsedIssueNumberArgument(int(issue_arg))
else:
# Assume url.
parsed_issue_arg = self._codereview_impl.ParseIssueURL(
urlparse.urlparse(issue_arg))
if not parsed_issue_arg or not parsed_issue_arg.valid:
DieWithError('Failed to parse issue argument "%s". '
'Must be an issue number or a valid URL.' % issue_arg)
return self._codereview_impl.CMDPatchWithParsedIssue(
parsed_issue_arg, reject, nocommit, directory, False)
def CMDUpload(self, options, git_diff_args, orig_args):
"""Uploads a change to codereview."""
custom_cl_base = None
if git_diff_args:
custom_cl_base = base_branch = git_diff_args[0]
else:
if self.GetBranch() is None:
DieWithError('Can\'t upload from detached HEAD state. Get on a branch!')
# Default to diffing against common ancestor of upstream branch
base_branch = self.GetCommonAncestorWithUpstream()
git_diff_args = [base_branch, 'HEAD']
# Warn about Rietveld deprecation for initial uploads to Rietveld.
if not self.IsGerrit() and not self.GetIssue():
print('=====================================')
print('NOTICE: Rietveld is being deprecated. '
'You can upload changes to Gerrit with')
print(' git cl upload --gerrit')
print('or set Gerrit to be your default code review tool with')
print(' git config gerrit.host true')
print('=====================================')
# Fast best-effort checks to abort before running potentially
# expensive hooks if uploading is likely to fail anyway. Passing these
# checks does not guarantee that uploading will not fail.
self._codereview_impl.EnsureAuthenticated(force=options.force)
self._codereview_impl.EnsureCanUploadPatchset(force=options.force)
# Apply watchlists on upload.
change = self.GetChange(base_branch, None)
watchlist = watchlists.Watchlists(change.RepositoryRoot())
files = [f.LocalPath() for f in change.AffectedFiles()]
if not options.bypass_watchlists:
self.ExtendCC(watchlist.GetWatchersForPaths(files))
if not options.bypass_hooks:
if options.reviewers or options.tbrs or options.add_owners_to:
# Set the reviewer list now so that presubmit checks can access it.
change_description = ChangeDescription(change.FullDescriptionText())
change_description.update_reviewers(options.reviewers,
options.tbrs,
options.add_owners_to,
change)
change.SetDescriptionText(change_description.description)
hook_results = self.RunHook(committing=False,
may_prompt=not options.force,
verbose=options.verbose,
change=change, parallel=options.parallel)
if not hook_results.should_continue():
return 1
if not options.reviewers and hook_results.reviewers:
options.reviewers = hook_results.reviewers.split(',')
self.ExtendCC(hook_results.more_cc)
# TODO(tandrii): Checking local patchset against remote patchset is only
# supported for Rietveld. Extend it to Gerrit or remove it completely.
if self.GetIssue() and not self.IsGerrit():
latest_patchset = self.GetMostRecentPatchset()
local_patchset = self.GetPatchset()
if (latest_patchset and local_patchset and
local_patchset != latest_patchset):
print('The last upload made from this repository was patchset #%d but '
'the most recent patchset on the server is #%d.'
% (local_patchset, latest_patchset))
print('Uploading will still work, but if you\'ve uploaded to this '
'issue from another machine or branch the patch you\'re '
'uploading now might not include those changes.')
confirm_or_exit(action='upload')
print_stats(git_diff_args)
ret = self.CMDUploadChange(options, git_diff_args, custom_cl_base, change)
if not ret:
if self.IsGerrit():
self.SetLabels(options.enable_auto_submit, options.use_commit_queue,
options.cq_dry_run);
else:
if options.use_commit_queue:
self.SetCQState(_CQState.COMMIT)
elif options.cq_dry_run:
self.SetCQState(_CQState.DRY_RUN)
_git_set_branch_config_value('last-upload-hash',
RunGit(['rev-parse', 'HEAD']).strip())
# Run post upload hooks, if specified.
if settings.GetRunPostUploadHook():
presubmit_support.DoPostUploadExecuter(
change,
self,
settings.GetRoot(),
options.verbose,
sys.stdout)
# Upload all dependencies if specified.
if options.dependencies:
print()
print('--dependencies has been specified.')
print('All dependent local branches will be re-uploaded.')
print()
# Remove the dependencies flag from args so that we do not end up in a
# loop.
orig_args.remove('--dependencies')
ret = upload_branch_deps(self, orig_args)
return ret
def SetLabels(self, enable_auto_submit, use_commit_queue, cq_dry_run):
"""Sets labels on the change based on the provided flags.
Sets labels if issue is already uploaded and known, else returns without
doing anything.
Args:
enable_auto_submit: Sets Auto-Submit+1 on the change.
use_commit_queue: Sets Commit-Queue+2 on the change.
cq_dry_run: Sets Commit-Queue+1 on the change. Overrides Commit-Queue+2 if
both use_commit_queue and cq_dry_run are true.
"""
if not self.GetIssue():
return
try:
self._codereview_impl.SetLabels(enable_auto_submit, use_commit_queue,
cq_dry_run)
return 0
except KeyboardInterrupt:
raise
except:
labels = []
if enable_auto_submit:
labels.append('Auto-Submit')
if use_commit_queue or cq_dry_run:
labels.append('Commit-Queue')
print('WARNING: Failed to set label(s) on your change: %s\n'
'Either:\n'
' * Your project does not have the above label(s),\n'
' * You don\'t have permission to set the above label(s),\n'
' * There\'s a bug in this code (see stack trace below).\n' %
(', '.join(labels)))
# Still raise exception so that stack trace is printed.
raise
def SetCQState(self, new_state):
"""Updates the CQ state for the latest patchset.
Issue must have been already uploaded and known.
"""
assert new_state in _CQState.ALL_STATES
assert self.GetIssue()
try:
self._codereview_impl.SetCQState(new_state)
return 0
except KeyboardInterrupt:
raise
except:
print('WARNING: Failed to %s.\n'
'Either:\n'
' * Your project has no CQ,\n'
' * You don\'t have permission to change the CQ state,\n'
' * There\'s a bug in this code (see stack trace below).\n'
'Consider specifying which bots to trigger manually or asking your '
'project owners for permissions or contacting Chrome Infra at:\n'
'https://www.chromium.org/infra\n\n' %
('cancel CQ' if new_state == _CQState.NONE else 'trigger CQ'))
# Still raise exception so that stack trace is printed.
raise
# Forward methods to codereview specific implementation.
def AddComment(self, message, publish=None):
return self._codereview_impl.AddComment(message, publish=publish)
def GetCommentsSummary(self, readable=True):
"""Returns list of _CommentSummary for each comment.
args:
readable: determines whether the output is designed for a human or a machine
"""
return self._codereview_impl.GetCommentsSummary(readable)
def CloseIssue(self):
return self._codereview_impl.CloseIssue()
def GetStatus(self):
return self._codereview_impl.GetStatus()
def GetCodereviewServer(self):
return self._codereview_impl.GetCodereviewServer()
def GetIssueOwner(self):
"""Get owner from codereview, which may differ from this checkout."""
return self._codereview_impl.GetIssueOwner()
def GetReviewers(self):
return self._codereview_impl.GetReviewers()
def GetMostRecentPatchset(self):
return self._codereview_impl.GetMostRecentPatchset()
def CannotTriggerTryJobReason(self):
"""Returns reason (str) if unable trigger try jobs on this CL or None."""
return self._codereview_impl.CannotTriggerTryJobReason()
def GetTryJobProperties(self, patchset=None):
"""Returns dictionary of properties to launch try job."""
return self._codereview_impl.GetTryJobProperties(patchset=patchset)
def __getattr__(self, attr):
# This is because lots of untested code accesses Rietveld-specific stuff
# directly, and it's hard to fix for sure. So, just let it work, and fix
# on a case by case basis.
# Note that child method defines __getattr__ as well, and forwards it here,
# because _RietveldChangelistImpl is not cleaned up yet, and given
# deprecation of Rietveld, it should probably be just removed.
# Until that time, avoid infinite recursion by bypassing __getattr__
# of implementation class.
return self._codereview_impl.__getattribute__(attr)
class _ChangelistCodereviewBase(object):
"""Abstract base class encapsulating codereview specifics of a changelist."""
def __init__(self, changelist):
self._changelist = changelist # instance of Changelist
def __getattr__(self, attr):
# Forward methods to changelist.
# TODO(tandrii): maybe clean up _GerritChangelistImpl and
# _RietveldChangelistImpl to avoid this hack?
return getattr(self._changelist, attr)
def GetStatus(self):
"""Apply a rough heuristic to give a simple summary of an issue's review
or CQ status, assuming adherence to a common workflow.
Returns None if no issue for this branch, or specific string keywords.
"""
raise NotImplementedError()
def GetCodereviewServer(self):
"""Returns server URL without end slash, like "https://codereview.com"."""
raise NotImplementedError()
def FetchDescription(self, force=False):
"""Fetches and returns description from the codereview server."""
raise NotImplementedError()
@classmethod
def IssueConfigKey(cls):
"""Returns branch setting storing issue number."""
raise NotImplementedError()
@classmethod
def PatchsetConfigKey(cls):
"""Returns branch setting storing patchset number."""
raise NotImplementedError()
@classmethod
def CodereviewServerConfigKey(cls):
"""Returns branch setting storing codereview server."""
raise NotImplementedError()
def _PostUnsetIssueProperties(self):
"""Which branch-specific properties to erase when unsetting issue."""
return []
def GetGerritObjForPresubmit(self):
# None is valid return value, otherwise presubmit_support.GerritAccessor.
return None
def UpdateDescriptionRemote(self, description, force=False):
"""Update the description on codereview site."""
raise NotImplementedError()
def AddComment(self, message, publish=None):
"""Posts a comment to the codereview site."""
raise NotImplementedError()
def GetCommentsSummary(self, readable=True):
raise NotImplementedError()
def CloseIssue(self):
"""Closes the issue."""
raise NotImplementedError()
def GetMostRecentPatchset(self):
"""Returns the most recent patchset number from the codereview site."""
raise NotImplementedError()
def CMDPatchWithParsedIssue(self, parsed_issue_arg, reject, nocommit,
directory, force):
"""Fetches and applies the issue.
Arguments:
parsed_issue_arg: instance of _ParsedIssueNumberArgument.
reject: if True, reject the failed patch instead of switching to 3-way
merge. Rietveld only.
nocommit: do not commit the patch, thus leave the tree dirty. Rietveld
only.
directory: switch to directory before applying the patch. Rietveld only.
force: if true, overwrites existing local state.
"""
raise NotImplementedError()
@staticmethod
def ParseIssueURL(parsed_url):
"""Parses url and returns instance of _ParsedIssueNumberArgument or None if
failed."""
raise NotImplementedError()
def EnsureAuthenticated(self, force, refresh=False):
"""Best effort check that user is authenticated with codereview server.
Arguments:
force: whether to skip confirmation questions.
refresh: whether to attempt to refresh credentials. Ignored if not
applicable.
"""
raise NotImplementedError()
def EnsureCanUploadPatchset(self, force):
"""Best effort check that uploading isn't supposed to fail for predictable
reasons.
This method should raise informative exception if uploading shouldn't
proceed.
Arguments:
force: whether to skip confirmation questions.
"""
raise NotImplementedError()
def CMDUploadChange(self, options, git_diff_args, custom_cl_base, change):
"""Uploads a change to codereview."""
raise NotImplementedError()
def SetLabels(self, enable_auto_submit, use_commit_queue, cq_dry_run):
"""Sets labels on the change based on the provided flags.
Issue must have been already uploaded and known.
"""
raise NotImplementedError()
def SetCQState(self, new_state):
"""Updates the CQ state for the latest patchset.
Issue must have been already uploaded and known.
"""
raise NotImplementedError()
def CannotTriggerTryJobReason(self):
"""Returns reason (str) if unable trigger try jobs on this CL or None."""
raise NotImplementedError()
def GetIssueOwner(self):
raise NotImplementedError()
def GetReviewers(self):
raise NotImplementedError()
def GetTryJobProperties(self, patchset=None):
raise NotImplementedError()
class _RietveldChangelistImpl(_ChangelistCodereviewBase):
def __init__(self, changelist, auth_config=None, codereview_host=None):
super(_RietveldChangelistImpl, self).__init__(changelist)
assert settings, 'must be initialized in _ChangelistCodereviewBase'
if not codereview_host:
settings.GetDefaultServerUrl()
self._rietveld_server = codereview_host
self._auth_config = auth_config or auth.make_auth_config()
self._props = None
self._rpc_server = None
def GetCodereviewServer(self):
if not self._rietveld_server:
# If we're on a branch then get the server potentially associated
# with that branch.
if self.GetIssue():
self._rietveld_server = gclient_utils.UpgradeToHttps(
self._GitGetBranchConfigValue(self.CodereviewServerConfigKey()))
if not self._rietveld_server:
self._rietveld_server = settings.GetDefaultServerUrl()
return self._rietveld_server
def EnsureAuthenticated(self, force, refresh=False):
"""Best effort check that user is authenticated with Rietveld server."""
if self._auth_config.use_oauth2:
authenticator = auth.get_authenticator_for_host(
self.GetCodereviewServer(), self._auth_config)
if not authenticator.has_cached_credentials():
raise auth.LoginRequiredError(self.GetCodereviewServer())
if refresh:
authenticator.get_access_token()
def EnsureCanUploadPatchset(self, force):
# No checks for Rietveld because we are deprecating Rietveld.
pass
def FetchDescription(self, force=False):
issue = self.GetIssue()
assert issue
try:
return self.RpcServer().get_description(issue, force=force).strip()
except urllib2.HTTPError as e:
if e.code == 404:
DieWithError(
('\nWhile fetching the description for issue %d, received a '
'404 (not found)\n'
'error. It is likely that you deleted this '
'issue on the server. If this is the\n'
'case, please run\n\n'
' git cl issue 0\n\n'
'to clear the association with the deleted issue. Then run '
'this command again.') % issue)
else:
DieWithError(
'\nFailed to fetch issue description. HTTP error %d' % e.code)
except urllib2.URLError as e:
print('Warning: Failed to retrieve CL description due to network '
'failure.', file=sys.stderr)
return ''
def GetMostRecentPatchset(self):
return self.GetIssueProperties()['patchsets'][-1]
def GetIssueProperties(self):
if self._props is None:
issue = self.GetIssue()
if not issue:
self._props = {}
else:
self._props = self.RpcServer().get_issue_properties(issue, True)
return self._props
def CannotTriggerTryJobReason(self):
props = self.GetIssueProperties()
if not props:
return 'Rietveld doesn\'t know about your issue %s' % self.GetIssue()
if props.get('closed'):
return 'CL %s is closed' % self.GetIssue()
if props.get('private'):
return 'CL %s is private' % self.GetIssue()
return None
def GetTryJobProperties(self, patchset=None):
"""Returns dictionary of properties to launch try job."""
project = (self.GetIssueProperties() or {}).get('project')
return {
'issue': self.GetIssue(),
'patch_project': project,
'patch_storage': 'rietveld',
'patchset': patchset or self.GetPatchset(),
'rietveld': self.GetCodereviewServer(),
}
def GetIssueOwner(self):
return (self.GetIssueProperties() or {}).get('owner_email')
def GetReviewers(self):
return (self.GetIssueProperties() or {}).get('reviewers')
def AddComment(self, message, publish=None):
return self.RpcServer().add_comment(self.GetIssue(), message)
def GetCommentsSummary(self, _readable=True):
summary = []
for message in self.GetIssueProperties().get('messages', []):
date = datetime.datetime.strptime(message['date'], '%Y-%m-%d %H:%M:%S.%f')
summary.append(_CommentSummary(
date=date,
disapproval=bool(message['disapproval']),
approval=bool(message['approval']),
sender=message['sender'],
message=message['text'],
))
return summary
def GetStatus(self):
"""Applies a rough heuristic to give a simple summary of an issue's review
or CQ status, assuming adherence to a common workflow.
Returns None if no issue for this branch, or one of the following keywords:
* 'error' - error from review tool (including deleted issues)
* 'unsent' - not sent for review
* 'waiting' - waiting for review
* 'reply' - waiting for owner to reply to review
* 'not lgtm' - Code-Review label has been set negatively
* 'lgtm' - LGTM from at least one approved reviewer
* 'commit' - in the commit queue
* 'closed' - closed
"""
if not self.GetIssue():
return None
try:
props = self.GetIssueProperties()
except urllib2.HTTPError:
return 'error'
if props.get('closed'):
# Issue is closed.
return 'closed'
if props.get('commit') and not props.get('cq_dry_run', False):
# Issue is in the commit queue.
return 'commit'
messages = props.get('messages') or []
if not messages:
# No message was sent.
return 'unsent'
if get_approving_reviewers(props):
return 'lgtm'
elif get_approving_reviewers(props, disapproval=True):
return 'not lgtm'
# Skip CQ messages that don't require owner's action.
while messages and messages[-1]['sender'] == COMMIT_BOT_EMAIL:
if 'Dry run:' in messages[-1]['text']:
messages.pop()
elif 'The CQ bit was unchecked' in messages[-1]['text']:
# This message always follows prior messages from CQ,
# so skip this too.
messages.pop()
else:
# This is probably a CQ messages warranting user attention.
break
if messages[-1]['sender'] != props.get('owner_email'):
# Non-LGTM reply from non-owner and not CQ bot.
return 'reply'
return 'waiting'
def UpdateDescriptionRemote(self, description, force=False):
self.RpcServer().update_description(self.GetIssue(), description)
def CloseIssue(self):
return self.RpcServer().close_issue(self.GetIssue())
def SetFlag(self, flag, value):
return self.SetFlags({flag: value})
def SetFlags(self, flags):
"""Sets flags on this CL/patchset in Rietveld.
"""
patchset = self.GetPatchset() or self.GetMostRecentPatchset()
try:
return self.RpcServer().set_flags(
self.GetIssue(), patchset, flags)
except urllib2.HTTPError as e:
if e.code == 404:
DieWithError('The issue %s doesn\'t exist.' % self.GetIssue())
if e.code == 403:
DieWithError(
('Access denied to issue %s. Maybe the patchset %s doesn\'t '
'match?') % (self.GetIssue(), patchset))
raise
def RpcServer(self):
"""Returns an upload.RpcServer() to access this review's rietveld instance.
"""
if not self._rpc_server:
self._rpc_server = rietveld.CachingRietveld(
self.GetCodereviewServer(),
self._auth_config)
return self._rpc_server
@classmethod
def IssueConfigKey(cls):
return 'rietveldissue'
@classmethod
def PatchsetConfigKey(cls):
return 'rietveldpatchset'
@classmethod
def CodereviewServerConfigKey(cls):
return 'rietveldserver'
def SetLabels(self, enable_auto_submit, use_commit_queue, cq_dry_run):
raise NotImplementedError()
def SetCQState(self, new_state):
props = self.GetIssueProperties()
if props.get('private'):
DieWithError('Cannot set-commit on private issue')
if new_state == _CQState.COMMIT:
self.SetFlags({'commit': '1', 'cq_dry_run': '0'})
elif new_state == _CQState.NONE:
self.SetFlags({'commit': '0', 'cq_dry_run': '0'})
else:
assert new_state == _CQState.DRY_RUN
self.SetFlags({'commit': '1', 'cq_dry_run': '1'})
def CMDPatchWithParsedIssue(self, parsed_issue_arg, reject, nocommit,
directory, force):
# PatchIssue should never be called with a dirty tree. It is up to the
# caller to check this, but just in case we assert here since the
# consequences of the caller not checking this could be dire.
assert(not git_common.is_dirty_git_tree('apply'))
assert(parsed_issue_arg.valid)
self._changelist.issue = parsed_issue_arg.issue
if parsed_issue_arg.hostname:
self._rietveld_server = 'https://%s' % parsed_issue_arg.hostname
patchset = parsed_issue_arg.patchset or self.GetMostRecentPatchset()
patchset_object = self.RpcServer().get_patch(self.GetIssue(), patchset)
scm_obj = checkout.GitCheckout(settings.GetRoot(), None, None, None, None)
try:
scm_obj.apply_patch(patchset_object)
except Exception as e:
print(str(e))
return 1
# If we had an issue, commit the current state and register the issue.
if not nocommit:
self.SetIssue(self.GetIssue())
self.SetPatchset(patchset)
RunGit(['commit', '-m', (self.GetDescription() + '\n\n' +
'patch from issue %(i)s at patchset '
'%(p)s (http://crrev.com/%(i)s#ps%(p)s)'
% {'i': self.GetIssue(), 'p': patchset})])
print('Committed patch locally.')
else:
print('Patch applied to index.')
return 0
@staticmethod
def ParseIssueURL(parsed_url):
if not parsed_url.scheme or not parsed_url.scheme.startswith('http'):
return None
# Rietveld patch: https://domain/<number>/#ps<patchset>
match = re.match(r'/(\d+)/$', parsed_url.path)
match2 = re.match(r'ps(\d+)$', parsed_url.fragment)
if match and match2:
return _ParsedIssueNumberArgument(
issue=int(match.group(1)),
patchset=int(match2.group(1)),
hostname=parsed_url.netloc,
codereview='rietveld')
# Typical url: https://domain/<issue_number>[/[other]]
match = re.match('/(\d+)(/.*)?$', parsed_url.path)
if match:
return _ParsedIssueNumberArgument(
issue=int(match.group(1)),
hostname=parsed_url.netloc,
codereview='rietveld')
# Rietveld patch: https://domain/download/issue<number>_<patchset>.diff
match = re.match(r'/download/issue(\d+)_(\d+).diff$', parsed_url.path)
if match:
return _ParsedIssueNumberArgument(
issue=int(match.group(1)),
patchset=int(match.group(2)),
hostname=parsed_url.netloc,
codereview='rietveld')
return None
def CMDUploadChange(self, options, args, custom_cl_base, change):
"""Upload the patch to Rietveld."""
upload_args = ['--assume_yes'] # Don't ask about untracked files.
upload_args.extend(['--server', self.GetCodereviewServer()])
upload_args.extend(auth.auth_config_to_command_options(self._auth_config))
if options.emulate_svn_auto_props:
upload_args.append('--emulate_svn_auto_props')
change_desc = None
if options.email is not None:
upload_args.extend(['--email', options.email])
if self.GetIssue():
if options.title is not None:
upload_args.extend(['--title', options.title])
if options.message:
upload_args.extend(['--message', options.message])
upload_args.extend(['--issue', str(self.GetIssue())])
print('This branch is associated with issue %s. '
'Adding patch to that issue.' % self.GetIssue())
else:
if options.title is not None:
upload_args.extend(['--title', options.title])
if options.message:
message = options.message
else:
message = CreateDescriptionFromLog(args)
if options.title:
message = options.title + '\n\n' + message
change_desc = ChangeDescription(message)
if options.reviewers or options.add_owners_to:
change_desc.update_reviewers(options.reviewers, options.tbrs,
options.add_owners_to, change)
if not options.force:
change_desc.prompt(bug=options.bug, git_footer=False)
if not change_desc.description:
print('Description is empty; aborting.')
return 1
upload_args.extend(['--message', change_desc.description])
if change_desc.get_reviewers():
upload_args.append('--reviewers=%s' % ','.join(
change_desc.get_reviewers()))
if options.send_mail:
if not change_desc.get_reviewers():
DieWithError("Must specify reviewers to send email.", change_desc)
upload_args.append('--send_mail')
# We check this before applying rietveld.private assuming that in
# rietveld.cc only addresses which we can send private CLs to are listed
# if rietveld.private is set, and so we should ignore rietveld.cc only
# when --private is specified explicitly on the command line.
if options.private:
logging.warn('rietveld.cc is ignored since private flag is specified. '
'You need to review and add them manually if necessary.')
cc = self.GetCCListWithoutDefault()
else:
cc = self.GetCCList()
cc = ','.join(filter(None, (cc, ','.join(options.cc))))
if change_desc.get_cced():
cc = ','.join(filter(None, (cc, ','.join(change_desc.get_cced()))))
if cc:
upload_args.extend(['--cc', cc])
if options.private or settings.GetDefaultPrivateFlag() == "True":
upload_args.append('--private')
# Include the upstream repo's URL in the change -- this is useful for
# projects that have their source spread across multiple repos.
remote_url = self.GetGitBaseUrlFromConfig()
if not remote_url:
if self.GetRemoteUrl() and '/' in self.GetUpstreamBranch():
remote_url = '%s@%s' % (self.GetRemoteUrl(),
self.GetUpstreamBranch().split('/')[-1])
if remote_url:
remote, remote_branch = self.GetRemoteBranch()
target_ref = GetTargetRef(remote, remote_branch, options.target_branch)
if target_ref:
upload_args.extend(['--target_ref', target_ref])
# Look for dependent patchsets. See crbug.com/480453 for more details.
remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch())
upstream_branch = ShortBranchName(upstream_branch)
if remote is '.':
# A local branch is being tracked.
local_branch = upstream_branch
if settings.GetIsSkipDependencyUpload(local_branch):
print()
print('Skipping dependency patchset upload because git config '
'branch.%s.skip-deps-uploads is set to True.' % local_branch)
print()
else:
auth_config = auth.extract_auth_config_from_options(options)
branch_cl = Changelist(branchref='refs/heads/'+local_branch,
auth_config=auth_config)
branch_cl_issue_url = branch_cl.GetIssueURL()
branch_cl_issue = branch_cl.GetIssue()
branch_cl_patchset = branch_cl.GetPatchset()
if branch_cl_issue_url and branch_cl_issue and branch_cl_patchset:
upload_args.extend(
['--depends_on_patchset', '%s:%s' % (
branch_cl_issue, branch_cl_patchset)])
print(
'\n'
'The current branch (%s) is tracking a local branch (%s) with '
'an associated CL.\n'
'Adding %s/#ps%s as a dependency patchset.\n'
'\n' % (self.GetBranch(), local_branch, branch_cl_issue_url,
branch_cl_patchset))
project = settings.GetProject()
if project:
upload_args.extend(['--project', project])
else:
print()
print('WARNING: Uploading without a project specified. Please ensure '
'your repo\'s codereview.settings has a "PROJECT: foo" line.')
print()
try:
upload_args = ['upload'] + upload_args + args
logging.info('upload.RealMain(%s)', upload_args)
issue, patchset = upload.RealMain(upload_args)
issue = int(issue)
patchset = int(patchset)
except KeyboardInterrupt:
sys.exit(1)
except:
# If we got an exception after the user typed a description for their
# change, back up the description before re-raising.
if change_desc:
SaveDescriptionBackup(change_desc)
raise
if not self.GetIssue():
self.SetIssue(issue)
self.SetPatchset(patchset)
return 0
class _GerritChangelistImpl(_ChangelistCodereviewBase):
def __init__(self, changelist, auth_config=None, codereview_host=None):
# auth_config is Rietveld thing, kept here to preserve interface only.
super(_GerritChangelistImpl, self).__init__(changelist)
self._change_id = None
# Lazily cached values.
self._gerrit_host = None # e.g. chromium-review.googlesource.com
self._gerrit_server = None # e.g. https://chromium-review.googlesource.com
# Map from change number (issue) to its detail cache.
self._detail_cache = {}
if codereview_host is not None:
assert not codereview_host.startswith('https://'), codereview_host
self._gerrit_host = codereview_host
self._gerrit_server = 'https://%s' % codereview_host
def _GetGerritHost(self):
# Lazy load of configs.
self.GetCodereviewServer()
if self._gerrit_host and '.' not in self._gerrit_host:
# Abbreviated domain like "chromium" instead of chromium.googlesource.com.
# This happens for internal stuff http://crbug.com/614312.
parsed = urlparse.urlparse(self.GetRemoteUrl())
if parsed.scheme == 'sso':
print('WARNING: using non-https URLs for remote is likely broken\n'
' Your current remote is: %s' % self.GetRemoteUrl())
self._gerrit_host = '%s.googlesource.com' % self._gerrit_host
self._gerrit_server = 'https://%s' % self._gerrit_host
return self._gerrit_host
def _GetGitHost(self):
"""Returns git host to be used when uploading change to Gerrit."""
return urlparse.urlparse(self.GetRemoteUrl()).netloc
def GetCodereviewServer(self):
if not self._gerrit_server:
# If we're on a branch then get the server potentially associated
# with that branch.
if self.GetIssue():
self._gerrit_server = self._GitGetBranchConfigValue(
self.CodereviewServerConfigKey())
if self._gerrit_server:
self._gerrit_host = urlparse.urlparse(self._gerrit_server).netloc
if not self._gerrit_server:
# We assume repo to be hosted on Gerrit, and hence Gerrit server
# has "-review" suffix for lowest level subdomain.
parts = self._GetGitHost().split('.')
parts[0] = parts[0] + '-review'
self._gerrit_host = '.'.join(parts)
self._gerrit_server = 'https://%s' % self._gerrit_host
return self._gerrit_server
@classmethod
def IssueConfigKey(cls):
return 'gerritissue'
@classmethod
def PatchsetConfigKey(cls):
return 'gerritpatchset'
@classmethod
def CodereviewServerConfigKey(cls):
return 'gerritserver'
def EnsureAuthenticated(self, force, refresh=None):
"""Best effort check that user is authenticated with Gerrit server."""
if settings.GetGerritSkipEnsureAuthenticated():
# For projects with unusual authentication schemes.
# See http://crbug.com/603378.
return
# Lazy-loader to identify Gerrit and Git hosts.
if gerrit_util.GceAuthenticator.is_gce():
return
self.GetCodereviewServer()
git_host = self._GetGitHost()
assert self._gerrit_server and self._gerrit_host
cookie_auth = gerrit_util.CookiesAuthenticator()
gerrit_auth = cookie_auth.get_auth_header(self._gerrit_host)
git_auth = cookie_auth.get_auth_header(git_host)
if gerrit_auth and git_auth:
if gerrit_auth == git_auth:
return
all_gsrc = cookie_auth.get_auth_header('d0esN0tEx1st.googlesource.com')
print((
'WARNING: You have different credentials for Gerrit and git hosts:\n'
' %s\n'
' %s\n'
' Consider running the following command:\n'
' git cl creds-check\n'
' %s\n'
' %s') %
(git_host, self._gerrit_host,
('Hint: delete creds for .googlesource.com' if all_gsrc else ''),
cookie_auth.get_new_password_message(git_host)))
if not force:
confirm_or_exit('If you know what you are doing', action='continue')
return
else:
missing = (
([] if gerrit_auth else [self._gerrit_host]) +
([] if git_auth else [git_host]))
DieWithError('Credentials for the following hosts are required:\n'
' %s\n'
'These are read from %s (or legacy %s)\n'
'%s' % (
'\n '.join(missing),
cookie_auth.get_gitcookies_path(),
cookie_auth.get_netrc_path(),
cookie_auth.get_new_password_message(git_host)))
def EnsureCanUploadPatchset(self, force):
if not self.GetIssue():
return
# Warm change details cache now to avoid RPCs later, reducing latency for
# developers.
self._GetChangeDetail(
['DETAILED_ACCOUNTS', 'CURRENT_REVISION', 'CURRENT_COMMIT'])
status = self._GetChangeDetail()['status']
if status in ('MERGED', 'ABANDONED'):
DieWithError('Change %s has been %s, new uploads are not allowed' %
(self.GetIssueURL(),
'submitted' if status == 'MERGED' else 'abandoned'))
if gerrit_util.GceAuthenticator.is_gce():
return
cookies_user = gerrit_util.CookiesAuthenticator().get_auth_email(
self._GetGerritHost())
if self.GetIssueOwner() == cookies_user:
return
logging.debug('change %s owner is %s, cookies user is %s',
self.GetIssue(), self.GetIssueOwner(), cookies_user)
# Maybe user has linked accounts or something like that,
# so ask what Gerrit thinks of this user.
details = gerrit_util.GetAccountDetails(self._GetGerritHost(), 'self')
if details['email'] == self.GetIssueOwner():
return
if not force:
print('WARNING: Change %s is owned by %s, but you authenticate to Gerrit '
'as %s.\n'
'Uploading may fail due to lack of permissions.' %
(self.GetIssue(), self.GetIssueOwner(), details['email']))
confirm_or_exit(action='upload')
def _PostUnsetIssueProperties(self):
"""Which branch-specific properties to erase when unsetting issue."""
return ['gerritsquashhash']
def GetGerritObjForPresubmit(self):
return presubmit_support.GerritAccessor(self._GetGerritHost())
def GetStatus(self):
"""Apply a rough heuristic to give a simple summary of an issue's review
or CQ status, assuming adherence to a common workflow.
Returns None if no issue for this branch, or one of the following keywords:
* 'error' - error from review tool (including deleted issues)
* 'unsent' - no reviewers added
* 'waiting' - waiting for review
* 'reply' - waiting for uploader to reply to review
* 'lgtm' - Code-Review label has been set
* 'commit' - in the commit queue
* 'closed' - successfully submitted or abandoned
"""
if not self.GetIssue():
return None
try:
data = self._GetChangeDetail([
'DETAILED_LABELS', 'CURRENT_REVISION', 'SUBMITTABLE'])
except (httplib.HTTPException, GerritChangeNotExists):
return 'error'
if data['status'] in ('ABANDONED', 'MERGED'):
return 'closed'
if data['labels'].get('Commit-Queue', {}).get('approved'):
# The section will have an "approved" subsection if anyone has voted
# the maximum value on the label.
return 'commit'
if data['labels'].get('Code-Review', {}).get('approved'):
return 'lgtm'
if not data.get('reviewers', {}).get('REVIEWER', []):
return 'unsent'
owner = data['owner'].get('_account_id')
messages = sorted(data.get('messages', []), key=lambda m: m.get('updated'))
last_message_author = messages.pop().get('author', {})
while last_message_author:
if last_message_author.get('email') == COMMIT_BOT_EMAIL:
# Ignore replies from CQ.
last_message_author = messages.pop().get('author', {})
continue
if last_message_author.get('_account_id') == owner:
# Most recent message was by owner.
return 'waiting'
else:
# Some reply from non-owner.
return 'reply'
# Somehow there are no messages even though there are reviewers.
return 'unsent'
def GetMostRecentPatchset(self):
data = self._GetChangeDetail(['CURRENT_REVISION'])
patchset = data['revisions'][data['current_revision']]['_number']
self.SetPatchset(patchset)
return patchset
def FetchDescription(self, force=False):
data = self._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT'],
no_cache=force)
current_rev = data['current_revision']
return data['revisions'][current_rev]['commit']['message']
def UpdateDescriptionRemote(self, description, force=False):
if gerrit_util.HasPendingChangeEdit(self._GetGerritHost(), self.GetIssue()):
if not force:
confirm_or_exit(
'The description cannot be modified while the issue has a pending '
'unpublished edit. Either publish the edit in the Gerrit web UI '
'or delete it.\n\n', action='delete the unpublished edit')
gerrit_util.DeletePendingChangeEdit(self._GetGerritHost(),
self.GetIssue())
gerrit_util.SetCommitMessage(self._GetGerritHost(), self.GetIssue(),
description, notify='NONE')
def AddComment(self, message, publish=None):
gerrit_util.SetReview(self._GetGerritHost(), self.GetIssue(),
msg=message, ready=publish)
def GetCommentsSummary(self, readable=True):
# DETAILED_ACCOUNTS is to get emails in accounts.
messages = self._GetChangeDetail(
options=['MESSAGES', 'DETAILED_ACCOUNTS']).get('messages', [])
file_comments = gerrit_util.GetChangeComments(
self._GetGerritHost(), self.GetIssue())
# Build dictionary of file comments for easy access and sorting later.
# {author+date: {path: {patchset: {line: url+message}}}}
comments = collections.defaultdict(
lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))
for path, line_comments in file_comments.iteritems():
for comment in line_comments:
if comment.get('tag', '').startswith('autogenerated'):
continue
key = (comment['author']['email'], comment['updated'])
if comment.get('side', 'REVISION') == 'PARENT':
patchset = 'Base'
else:
patchset = 'PS%d' % comment['patch_set']
line = comment.get('line', 0)
url = ('https://%s/c/%s/%s/%s#%s%s' %
(self._GetGerritHost(), self.GetIssue(), comment['patch_set'], path,
'b' if comment.get('side') == 'PARENT' else '',
str(line) if line else ''))
comments[key][path][patchset][line] = (url, comment['message'])
summary = []
for msg in messages:
# Don't bother showing autogenerated messages.
if msg.get('tag') and msg.get('tag').startswith('autogenerated'):
continue
# Gerrit spits out nanoseconds.
assert len(msg['date'].split('.')[-1]) == 9
date = datetime.datetime.strptime(msg['date'][:-3],
'%Y-%m-%d %H:%M:%S.%f')
message = msg['message']
key = (msg['author']['email'], msg['date'])
if key in comments:
message += '\n'
for path, patchsets in sorted(comments.get(key, {}).items()):
if readable: