Merge commits up to v2.20
* tag 'v2.20':
project: prune sample hooks
project: drop support for symlinking internal .git files
project: abort a bit earlier before migrating .git/
Revert "sync: dropped "NOTICE: --use-superproject is in beta ..." message."
project: stop symlinking info dir under .git/
project: stop symlinking description file under .git/
project: stop symlinking svn under .git/
project: clean up now unused code
sync: dropped "NOTICE: --use-superproject is in beta ..." message.
sync: With --mirror option, don't display no-use-superproject... message.
Change-Id: Id3d05eac7291d0adad9f13409966b584a3c27798
diff --git a/error.py b/error.py
index cbefcb7..80e6004 100644
--- a/error.py
+++ b/error.py
@@ -133,3 +133,7 @@
The common case is that the file wasn't present when we tried to run it.
"""
+
+class CacheApplyError(Exception):
+ """Thrown when errors happen in 'repo sync' with '--cache-dir' option.
+ """
diff --git a/project.py b/project.py
index 338feff..f1d6c9f 100644
--- a/project.py
+++ b/project.py
@@ -30,8 +30,9 @@
from color import Coloring
from git_command import GitCommand, git_require
from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
- ID_RE
+ ID_RE, RefSpec
from error import GitError, UploadError, DownloadError
+from error import CacheApplyError
from error import ManifestInvalidRevisionError, ManifestInvalidPathError
from error import NoManifestException
import platform_utils
@@ -1054,6 +1055,68 @@
_error("Cannot extract archive %s: %s", tarpath, str(e))
return False
+ def CachePopulate(self, cache_dir, url):
+ """Populate cache in the cache_dir.
+
+ Args:
+ cache_dir: Directory to cache git files from Google Storage.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if it fails to populate the git cache.
+ """
+ cmd = ['cache', 'populate', '--ignore_locks', '-v',
+ '--cache-dir', cache_dir, url]
+
+ if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0:
+ raise CacheApplyError('Failed to populate cache. cache_dir: %s '
+ 'url: %s' % (cache_dir, url))
+
+ def CacheExists(self, cache_dir, url):
+ """Check the existence of the cache files.
+
+ Args:
+ cache_dir: Directory to cache git files.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if the cache files do not exist.
+ """
+ cmd = ['cache', 'exists', '--quiet', '--cache-dir', cache_dir, url]
+
+ exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True)
+ if exist.Wait() != 0:
+ raise CacheApplyError('Failed to execute git cache exists cmd. '
+ 'cache_dir: %s url: %s' % (cache_dir, url))
+
+ if not exist.stdout or not exist.stdout.strip():
+ raise CacheApplyError('Failed to find cache. cache_dir: %s '
+ 'url: %s' % (cache_dir, url))
+ return exist.stdout.strip()
+
+ def CacheApply(self, cache_dir):
+ """Apply git cache files populated from Google Storage buckets.
+
+ Args:
+ cache_dir: Directory to cache git files.
+
+ Raises:
+ CacheApplyError if it fails to apply git caches.
+ """
+ remote = self.GetRemote(self.remote.name)
+
+ self.CachePopulate(cache_dir, remote.url)
+
+ mirror_dir = self.CacheExists(cache_dir, remote.url)
+
+ refspec = RefSpec(True, 'refs/heads/*',
+ 'refs/remotes/%s/*' % remote.name)
+
+ fetch_cache_cmd = ['fetch', mirror_dir, str(refspec)]
+ if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0:
+ raise CacheApplyError('Failed to fetch refs %s from %s' %
+ (mirror_dir, str(refspec)))
+
def Sync_NetworkHalf(self,
quiet=False,
verbose=False,
@@ -1068,6 +1131,7 @@
retry_fetches=0,
prune=False,
submodules=False,
+ cache_dir=None,
ssh_proxy=None,
clone_filter=None,
partial_clone_exclude=set()):
@@ -1130,7 +1194,22 @@
else:
alt_dir = None
+ applied_cache = False
+ # If cache_dir is provided, and it's a new repository without
+ # alternative_dir, bootstrap this project repo with the git
+ # cache files.
+ if cache_dir is not None and is_new and alt_dir is None:
+ try:
+ self.CacheApply(cache_dir)
+ applied_cache = True
+ is_new = False
+ except CacheApplyError as e:
+ _error('Could not apply git cache: %s', e)
+ _error('Please check if you have the right GS credentials.')
+ _error('Please check if the cache files exist in GS.')
+
if (clone_bundle
+ and not applied_cache
and alt_dir is None
and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)):
is_new = False
diff --git a/repo b/repo
index 2af77ac..a1cce4f 100755
--- a/repo
+++ b/repo
@@ -141,7 +141,7 @@
#
REPO_URL = os.environ.get('REPO_URL', None)
if not REPO_URL:
- REPO_URL = 'https://gerrit.googlesource.com/git-repo'
+ REPO_URL = 'https://chromium.googlesource.com/external/repo'
REPO_REV = os.environ.get('REPO_REV')
if not REPO_REV:
REPO_REV = 'stable'
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 707c5bb..80060b4 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -253,6 +253,11 @@
help='delete refs that no longer exist on the remote (default)')
p.add_option('--no-prune', dest='prune', action='store_false',
help='do not delete refs that no longer exist on the remote')
+ p.add_option('--cache-dir', dest='cache_dir', action='store',
+ help='Use git-cache to populate project cache into this '
+ 'directory. Bootstrap the local repository from this '
+ 'directory if the project cache exists. This applies '
+ 'to the projects on chromium and chrome-internal.')
if show_smart:
p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true',
@@ -367,6 +372,7 @@
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
prune=opt.prune,
+ cache_dir=opt.cache_dir,
ssh_proxy=self.ssh_proxy,
clone_filter=self.manifest.CloneFilter,
partial_clone_exclude=self.manifest.PartialCloneExclude)
@@ -892,6 +898,7 @@
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
submodules=self.manifest.HasSubmodules,
+ cache_dir=opt.cache_dir,
clone_filter=self.manifest.CloneFilter,
partial_clone_exclude=self.manifest.PartialCloneExclude)
finish = time.time()
@@ -939,6 +946,18 @@
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
+ cache_dir = opt.cache_dir
+ if cache_dir:
+ if self.manifest.IsMirror or self.manifest.IsArchive:
+ print('fatal: --cache-dir is not supported with mirror or archive '
+ 'repository.')
+ sys.exit(1)
+
+ if os.path.isfile(cache_dir):
+ print('fatal: %s: cache_dir must be a directory', cache_dir, file=sys.stderr)
+ sys.exit(1)
+ os.makedirs(opt.cache_dir, exist_ok=True)
+
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
