Merge commits up to v1.13.7
Had to resolve a trivial merge conflict in subcmds/sync.py related
to the upstream refactoring and our support for cache dir.
e778e57f11f2 2019-10-04 14:21:41 -0400 command: filter projects by active manifest groups
f1c5dd8a0fdf 2019-10-01 01:17:55 -0400 info: fix "current" output
2058c6364180 2019-10-05 00:18:41 -0400 Only import imp on py2
c8290ad49e44 2019-10-01 01:07:11 -0400 project: allow CurrentBranch to return None on errors
9775a3d5d2dc 2019-10-01 01:01:33 -0400 info: allow NoSuchProjectError to bubble up
9bfdfbe117d1 2019-09-30 22:46:45 -0400 version: add source versions & User-Agents to the output
2f0951b21648 2019-07-10 17:13:46 -0400 git_command: set GIT_HTTP_USER_AGENT on all requests
72ab852ca503 2019-10-01 00:18:46 -0400 grep: handle errors gracefully
0a9265e2d633 2019-09-30 23:59:27 -0400 diff: handle errors gracefully
dc1b59d2c0a7 2019-09-30 23:47:03 -0400 forall: exit 1 if we skip any repos
71b0f312b15b 2019-09-30 22:39:49 -0400 git_command: refactor User-Agent settings
369814b4a77a 2019-07-10 17:10:07 -0400 move UserAgent to git_command for wider user
e37aa5f331aa 2019-09-23 19:14:13 -0400 rebase: add basic coloring output
4a07798c826e 2019-09-23 18:54:30 -0400 rebase: add --fail-fast support
fb527e3f522a 2019-08-27 02:34:32 -0400 sync: create dedicated manifest project update func
6be76337a0b6 2019-09-17 16:27:18 -0400 repo: bump wrapper version
Change-Id: I5693cef680bd4b08368636b9408641b5c963c415
diff --git a/command.py b/command.py
index f7d20a2..9e113f1 100644
--- a/command.py
+++ b/command.py
@@ -175,7 +175,10 @@
self._ResetPathToProjectMap(all_projects_list)
for arg in args:
- projects = manifest.GetProjectsWithName(arg)
+ # We have to filter by manifest groups in case the requested project is
+ # checked out multiple times or differently based on them.
+ projects = [project for project in manifest.GetProjectsWithName(arg)
+ if project.MatchesGroups(groups)]
if not projects:
path = os.path.abspath(arg).replace('\\', '/')
@@ -200,7 +203,7 @@
for project in projects:
if not missing_ok and not project.Exists:
- raise NoSuchProjectError(arg)
+ raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
if not project.MatchesGroups(groups):
raise InvalidProjectGroupsError(arg)
diff --git a/git_command.py b/git_command.py
index 6742303..dc542c3 100644
--- a/git_command.py
+++ b/git_command.py
@@ -22,6 +22,7 @@
from signal import SIGTERM
from error import GitError
+from git_refs import HEAD
import platform_utils
from repo_trace import REPO_TRACE, IsTrace, Trace
from wrapper import Wrapper
@@ -98,6 +99,86 @@
return fun
git = _GitCall()
+
+def RepoSourceVersion():
+ """Return the version of the repo.git tree."""
+ ver = getattr(RepoSourceVersion, 'version', None)
+
+ # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+ # to initialize version info we provide.
+ if ver is None:
+ env = GitCommand._GetBasicEnv()
+
+ proj = os.path.dirname(os.path.abspath(__file__))
+ env[GIT_DIR] = os.path.join(proj, '.git')
+
+ p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
+ env=env)
+ if p.wait() == 0:
+ ver = p.stdout.read().strip().decode('utf-8')
+ if ver.startswith('v'):
+ ver = ver[1:]
+ else:
+ ver = 'unknown'
+ setattr(RepoSourceVersion, 'version', ver)
+
+ return ver
+
+
+class UserAgent(object):
+ """Mange User-Agent settings when talking to external services
+
+ We follow the style as documented here:
+ https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+ """
+
+ _os = None
+ _repo_ua = None
+ _git_ua = None
+
+ @property
+ def os(self):
+ """The operating system name."""
+ if self._os is None:
+ os_name = sys.platform
+ if os_name.lower().startswith('linux'):
+ os_name = 'Linux'
+ elif os_name == 'win32':
+ os_name = 'Win32'
+ elif os_name == 'cygwin':
+ os_name = 'Cygwin'
+ elif os_name == 'darwin':
+ os_name = 'Darwin'
+ self._os = os_name
+
+ return self._os
+
+ @property
+ def repo(self):
+ """The UA when connecting directly from repo."""
+ if self._repo_ua is None:
+ py_version = sys.version_info
+ self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+ RepoSourceVersion(),
+ self.os,
+ git.version_tuple().full,
+ py_version.major, py_version.minor, py_version.micro)
+
+ return self._repo_ua
+
+ @property
+ def git(self):
+ """The UA when running git."""
+ if self._git_ua is None:
+ self._git_ua = 'git/%s (%s) git-repo/%s' % (
+ git.version_tuple().full,
+ self.os,
+ RepoSourceVersion())
+
+ return self._git_ua
+
+user_agent = UserAgent()
+
def git_require(min_version, fail=False, msg=''):
git_version = git.version_tuple()
if min_version <= git_version:
@@ -125,17 +206,7 @@
ssh_proxy = False,
cwd = None,
gitdir = None):
- env = os.environ.copy()
-
- for key in [REPO_TRACE,
- GIT_DIR,
- 'GIT_ALTERNATE_OBJECT_DIRECTORIES',
- 'GIT_OBJECT_DIRECTORY',
- 'GIT_WORK_TREE',
- 'GIT_GRAFT_FILE',
- 'GIT_INDEX_FILE']:
- if key in env:
- del env[key]
+ env = self._GetBasicEnv()
# If we are not capturing std* then need to print it.
self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
@@ -155,6 +226,7 @@
if 'GIT_ALLOW_PROTOCOL' not in env:
_setenv(env, 'GIT_ALLOW_PROTOCOL',
'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+ _setenv(env, 'GIT_HTTP_USER_AGENT', user_agent.git)
if project:
if not cwd:
@@ -227,6 +299,23 @@
self.process = p
self.stdin = p.stdin
+ @staticmethod
+ def _GetBasicEnv():
+ """Return a basic env for running git under.
+
+ This is guaranteed to be side-effect free.
+ """
+ env = os.environ.copy()
+ for key in (REPO_TRACE,
+ GIT_DIR,
+ 'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+ 'GIT_OBJECT_DIRECTORY',
+ 'GIT_WORK_TREE',
+ 'GIT_GRAFT_FILE',
+ 'GIT_INDEX_FILE'):
+ env.pop(key, None)
+ return env
+
def Wait(self):
try:
p = self.process
diff --git a/main.py b/main.py
index 2ab79b5..6e74d5a 100755
--- a/main.py
+++ b/main.py
@@ -23,7 +23,6 @@
from __future__ import print_function
import getpass
-import imp
import netrc
import optparse
import os
@@ -34,6 +33,7 @@
if is_python3():
import urllib.request
else:
+ import imp
import urllib2
urllib = imp.new_module('urllib')
urllib.request = urllib2
@@ -46,7 +46,7 @@
from color import SetDefaultColoring
import event_log
from repo_trace import SetTrace
-from git_command import git, GitCommand
+from git_command import git, GitCommand, user_agent
from git_config import init_ssh, close_ssh
from command import InteractiveCommand
from command import MirrorSafeCommand
@@ -244,10 +244,6 @@
return result
-def _MyRepoPath():
- return os.path.dirname(__file__)
-
-
def _CheckWrapperVersion(ver, repo_path):
if not repo_path:
repo_path = '~/bin/repo'
@@ -299,51 +295,13 @@
continue
i += 1
-_user_agent = None
-
-def _UserAgent():
- global _user_agent
-
- if _user_agent is None:
- py_version = sys.version_info
-
- os_name = sys.platform
- if os_name == 'linux2':
- os_name = 'Linux'
- elif os_name == 'win32':
- os_name = 'Win32'
- elif os_name == 'cygwin':
- os_name = 'Cygwin'
- elif os_name == 'darwin':
- os_name = 'Darwin'
-
- p = GitCommand(
- None, ['describe', 'HEAD'],
- cwd = _MyRepoPath(),
- capture_stdout = True)
- if p.Wait() == 0:
- repo_version = p.stdout
- if len(repo_version) > 0 and repo_version[-1] == '\n':
- repo_version = repo_version[0:-1]
- if len(repo_version) > 0 and repo_version[0] == 'v':
- repo_version = repo_version[1:]
- else:
- repo_version = 'unknown'
-
- _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
- repo_version,
- os_name,
- git.version_tuple().full,
- py_version[0], py_version[1], py_version[2])
- return _user_agent
-
class _UserAgentHandler(urllib.request.BaseHandler):
def http_request(self, req):
- req.add_header('User-Agent', _UserAgent())
+ req.add_header('User-Agent', user_agent.repo)
return req
def https_request(self, req):
- req.add_header('User-Agent', _UserAgent())
+ req.add_header('User-Agent', user_agent.repo)
return req
def _AddPasswordFromUserInput(handler, msg, req):
diff --git a/project.py b/project.py
index 47f19dc..0e82216 100755
--- a/project.py
+++ b/project.py
@@ -231,6 +231,7 @@
def __init__(self, config):
Coloring.__init__(self, config, 'diff')
self.project = self.printer('header', attr='bold')
+ self.fail = self.printer('fail', fg='red')
class _Annotation(object):
@@ -866,10 +867,17 @@
@property
def CurrentBranch(self):
"""Obtain the name of the currently checked out branch.
- The branch name omits the 'refs/heads/' prefix.
- None is returned if the project is on a detached HEAD.
+
+ The branch name omits the 'refs/heads/' prefix.
+ None is returned if the project is on a detached HEAD, or if the work_git is
+ otheriwse inaccessible (e.g. an incomplete sync).
"""
- b = self.work_git.GetHead()
+ try:
+ b = self.work_git.GetHead()
+ except NoManifestException:
+ # If the local checkout is in a bad state, don't barf. Let the callers
+ # process this like the head is unreadable.
+ return None
if b.startswith(R_HEADS):
return b[len(R_HEADS):]
return None
@@ -1137,10 +1145,18 @@
cmd.append('--src-prefix=a/%s/' % self.relpath)
cmd.append('--dst-prefix=b/%s/' % self.relpath)
cmd.append('--')
- p = GitCommand(self,
- cmd,
- capture_stdout=True,
- capture_stderr=True)
+ try:
+ p = GitCommand(self,
+ cmd,
+ capture_stdout=True,
+ capture_stderr=True)
+ except GitError as e:
+ out.nl()
+ out.project('project %s/' % self.relpath)
+ out.nl()
+ out.fail('%s', str(e))
+ out.nl()
+ return False
has_diff = False
for line in p.process.stdout:
if not hasattr(line, 'encode'):
@@ -1151,7 +1167,7 @@
out.nl()
has_diff = True
print(line[:-1])
- p.Wait()
+ return p.Wait() == 0
# Publish / Upload ##
diff --git a/repo b/repo
index 76d7bdd..6762c75 100755
--- a/repo
+++ b/repo
@@ -33,7 +33,7 @@
# limitations under the License.
# increment this whenever we make important changes to this script
-VERSION = (1, 25)
+VERSION = (1, 26)
# increment this if the MAINTAINER_KEYS block is modified
KEYRING_VERSION = (1, 5)
diff --git a/subcmds/diff.py b/subcmds/diff.py
index 1f3abd8..fa41e70 100644
--- a/subcmds/diff.py
+++ b/subcmds/diff.py
@@ -37,5 +37,8 @@
help='Paths are relative to the repository root')
def Execute(self, opt, args):
+ ret = 0
for project in self.GetProjects(args):
- project.PrintWorkTreeDiff(opt.absolute)
+ if not project.PrintWorkTreeDiff(opt.absolute):
+ ret = 1
+ return ret
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 0be8d3b..c9de26b 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -323,10 +323,10 @@
cwd = project['worktree']
if not os.path.exists(cwd):
- if (opt.project_header and opt.verbose) \
- or not opt.project_header:
+ if ((opt.project_header and opt.verbose)
+ or not opt.project_header):
print('skipping %s/' % project['relpath'], file=sys.stderr)
- return
+ return 1
if opt.project_header:
stdin = subprocess.PIPE
diff --git a/subcmds/grep.py b/subcmds/grep.py
index a588a78..4dd85d5 100644
--- a/subcmds/grep.py
+++ b/subcmds/grep.py
@@ -15,15 +15,19 @@
# limitations under the License.
from __future__ import print_function
+
import sys
+
from color import Coloring
from command import PagedCommand
+from error import GitError
from git_command import git_require, GitCommand
class GrepColoring(Coloring):
def __init__(self, config):
Coloring.__init__(self, config, 'grep')
self.project = self.printer('project', attr='bold')
+ self.fail = self.printer('fail', fg='red')
class Grep(PagedCommand):
common = True
@@ -184,15 +188,25 @@
cmd_argv.extend(opt.revision)
cmd_argv.append('--')
+ git_failed = False
bad_rev = False
have_match = False
for project in projects:
- p = GitCommand(project,
- cmd_argv,
- bare = False,
- capture_stdout = True,
- capture_stderr = True)
+ try:
+ p = GitCommand(project,
+ cmd_argv,
+ bare=False,
+ capture_stdout=True,
+ capture_stderr=True)
+ except GitError as e:
+ git_failed = True
+ out.project('--- project %s ---' % project.relpath)
+ out.nl()
+ out.fail('%s', str(e))
+ out.nl()
+ continue
+
if p.Wait() != 0:
# no results
#
@@ -202,7 +216,7 @@
else:
out.project('--- project %s ---' % project.relpath)
out.nl()
- out.write("%s", p.stderr)
+ out.fail('%s', p.stderr.strip())
out.nl()
continue
have_match = True
@@ -231,7 +245,9 @@
for line in r:
print(line)
- if have_match:
+ if git_failed:
+ sys.exit(1)
+ elif have_match:
sys.exit(0)
elif have_rev and bad_rev:
for r in opt.revision:
diff --git a/subcmds/info.py b/subcmds/info.py
index be5a8f2..b1e92e3 100644
--- a/subcmds/info.py
+++ b/subcmds/info.py
@@ -16,7 +16,6 @@
from command import PagedCommand
from color import Coloring
-from error import NoSuchProjectError
from git_refs import R_M
class _Coloring(Coloring):
@@ -82,10 +81,8 @@
self.out.nl()
def printDiffInfo(self, args):
- try:
- projs = self.GetProjects(args)
- except NoSuchProjectError:
- return
+ # We let exceptions bubble up to main as they'll be well structured.
+ projs = self.GetProjects(args)
for p in projs:
self.heading("Project: ")
@@ -97,13 +94,19 @@
self.out.nl()
self.heading("Current revision: ")
- self.headtext(p.revisionExpr)
+ self.headtext(p.GetRevisionId())
self.out.nl()
+ currentBranch = p.CurrentBranch
+ if currentBranch:
+ self.heading('Current branch: ')
+ self.headtext(currentBranch)
+ self.out.nl()
+
localBranches = list(p.GetBranches().keys())
self.heading("Local Branches: ")
self.redtext(str(len(localBranches)))
- if len(localBranches) > 0:
+ if localBranches:
self.text(" [")
self.text(", ".join(localBranches))
self.text("]")
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index 9bc4460..dcb8b2a 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -17,9 +17,18 @@
from __future__ import print_function
import sys
+from color import Coloring
from command import Command
from git_command import GitCommand
+
+class RebaseColoring(Coloring):
+ def __init__(self, config):
+ Coloring.__init__(self, config, 'rebase')
+ self.project = self.printer('project', attr='bold')
+ self.fail = self.printer('fail', fg='red')
+
+
class Rebase(Command):
common = True
helpSummary = "Rebase local branches on upstream branch"
@@ -37,6 +46,9 @@
dest="interactive", action="store_true",
help="interactive rebase (single project only)")
+ p.add_option('--fail-fast',
+ dest='fail_fast', action='store_true',
+ help='Stop rebasing after first error is hit')
p.add_option('-f', '--force-rebase',
dest='force_rebase', action='store_true',
help='Pass --force-rebase to git rebase')
@@ -88,7 +100,15 @@
if opt.interactive:
common_args.append('-i')
+ config = self.manifest.manifestProject.config
+ out = RebaseColoring(config)
+ out.redirect(sys.stdout)
+
+ ret = 0
for project in all_projects:
+ if ret and opt.fail_fast:
+ break
+
cb = project.CurrentBranch
if not cb:
if one_project:
@@ -114,8 +134,10 @@
args.append(upbranch.LocalMerge)
- print('# %s: rebasing %s -> %s'
- % (project.relpath, cb, upbranch.LocalMerge), file=sys.stderr)
+ out.project('project %s: rebasing %s -> %s',
+ project.relpath, cb, upbranch.LocalMerge)
+ out.nl()
+ out.flush()
needs_stash = False
if opt.auto_stash:
@@ -127,13 +149,21 @@
stash_args = ["stash"]
if GitCommand(project, stash_args).Wait() != 0:
- return 1
+ ret += 1
+ continue
if GitCommand(project, args).Wait() != 0:
- return 1
+ ret += 1
+ continue
if needs_stash:
stash_args.append('pop')
stash_args.append('--quiet')
if GitCommand(project, stash_args).Wait() != 0:
- return 1
+ ret += 1
+
+ if ret:
+ out.fail('%i projects had errors', ret)
+ out.nl()
+
+ return ret
diff --git a/subcmds/sync.py b/subcmds/sync.py
index d390ed8..bca0375 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -843,6 +843,34 @@
return manifest_name
+ def _UpdateManifestProject(self, opt, mp, manifest_name):
+ """Fetch & update the local manifest project."""
+ if not opt.local_only:
+ start = time.time()
+ success = mp.Sync_NetworkHalf(quiet=opt.quiet,
+ current_branch_only=opt.current_branch_only,
+ no_tags=opt.no_tags,
+ optimized_fetch=opt.optimized_fetch,
+ submodules=self.manifest.HasSubmodules,
+ clone_filter=self.manifest.CloneFilter,
+ cache_dir=opt.cache_dir)
+ finish = time.time()
+ self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
+ start, finish, success)
+
+ if mp.HasChanges:
+ syncbuf = SyncBuffer(mp.config)
+ start = time.time()
+ mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+ clean = syncbuf.Finish()
+ self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
+ start, time.time(), clean)
+ if not clean:
+ sys.exit(1)
+ self._ReloadManifest(opt.manifest_name)
+ if opt.jobs is None:
+ self.jobs = self.manifest.default.sync_j
+
def ValidateOptions(self, opt, args):
if opt.force_broken:
print('warning: -f/--force-broken is now the default behavior, and the '
@@ -918,31 +946,7 @@
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
- if not opt.local_only:
- start = time.time()
- success = mp.Sync_NetworkHalf(quiet=opt.quiet,
- current_branch_only=opt.current_branch_only,
- no_tags=opt.no_tags,
- optimized_fetch=opt.optimized_fetch,
- submodules=self.manifest.HasSubmodules,
- clone_filter=self.manifest.CloneFilter,
- cache_dir=cache_dir)
- finish = time.time()
- self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
- start, finish, success)
-
- if mp.HasChanges:
- syncbuf = SyncBuffer(mp.config)
- start = time.time()
- mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
- clean = syncbuf.Finish()
- self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
- start, time.time(), clean)
- if not clean:
- sys.exit(1)
- self._ReloadManifest(manifest_name)
- if opt.jobs is None:
- self.jobs = self.manifest.default.sync_j
+ self._UpdateManifestProject(opt, mp, manifest_name)
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
diff --git a/subcmds/version.py b/subcmds/version.py
index 9fb694d..761172b 100644
--- a/subcmds/version.py
+++ b/subcmds/version.py
@@ -17,7 +17,7 @@
from __future__ import print_function
import sys
from command import Command, MirrorSafeCommand
-from git_command import git
+from git_command import git, RepoSourceVersion, user_agent
from git_refs import HEAD
class Version(Command, MirrorSafeCommand):
@@ -34,12 +34,20 @@
rp = self.manifest.repoProject
rem = rp.GetRemote(rp.remote.name)
- print('repo version %s' % rp.work_git.describe(HEAD))
+ # These might not be the same. Report them both.
+ src_ver = RepoSourceVersion()
+ rp_ver = rp.bare_git.describe(HEAD)
+ print('repo version %s' % rp_ver)
print(' (from %s)' % rem.url)
if Version.wrapper_path is not None:
print('repo launcher version %s' % Version.wrapper_version)
print(' (from %s)' % Version.wrapper_path)
+ if src_ver != rp_ver:
+ print(' (currently at %s)' % src_ver)
+
+ print('repo User-Agent %s' % user_agent.repo)
print('git %s' % git.version_tuple().full)
+ print('git User-Agent %s' % user_agent.git)
print('Python %s' % sys.version)
diff --git a/tests/test_git_command.py b/tests/test_git_command.py
index 928eb40..51171a3 100644
--- a/tests/test_git_command.py
+++ b/tests/test_git_command.py
@@ -18,6 +18,7 @@
from __future__ import print_function
+import re
import unittest
import git_command
@@ -47,3 +48,31 @@
self.assertLess(ver, (9999, 9999, 9999))
self.assertNotEqual('', ver.full)
+
+
+class UserAgentUnitTest(unittest.TestCase):
+ """Tests the UserAgent function."""
+
+ def test_smoke_os(self):
+ """Make sure UA OS setting returns something useful."""
+ os_name = git_command.user_agent.os
+ # We can't dive too deep because of OS/tool differences, but we can check
+ # the general form.
+ m = re.match(r'^[^ ]+$', os_name)
+ self.assertIsNotNone(m)
+
+ def test_smoke_repo(self):
+ """Make sure repo UA returns something useful."""
+ ua = git_command.user_agent.repo
+ # We can't dive too deep because of OS/tool differences, but we can check
+ # the general form.
+ m = re.match(r'^git-repo/[^ ]+ ([^ ]+) git/[^ ]+ Python/[0-9.]+', ua)
+ self.assertIsNotNone(m)
+
+ def test_smoke_git(self):
+ """Make sure git UA returns something useful."""
+ ua = git_command.user_agent.git
+ # We can't dive too deep because of OS/tool differences, but we can check
+ # the general form.
+ m = re.match(r'^git/[^ ]+ ([^ ]+) git-repo/[^ ]+', ua)
+ self.assertIsNotNone(m)
