Merge tag 'v2.45'
repo v2.45
* tag 'v2.45':
project: Check if dotgit exists w/out symlink check
git: raise soft version to 2.7.4
git: raise hard version to 1.7.9
docs: release: add recent git/python/ssh/debian info
main: Stringify project name in error_info
diff --git a/error.py b/error.py
index 7958a0a..22153ff 100644
--- a/error.py
+++ b/error.py
@@ -187,3 +187,7 @@
The common case is that the file wasn't present when we tried to run it.
"""
+
+
+class CacheApplyError(Exception):
+ """Thrown when errors happen in 'repo sync' with '--cache-dir' option."""
diff --git a/project.py b/project.py
index 1f5e4c3..9563e7d 100644
--- a/project.py
+++ b/project.py
@@ -31,6 +31,7 @@
import urllib.parse
from color import Coloring
+from error import CacheApplyError
from error import DownloadError
from error import GitError
from error import ManifestInvalidPathError
@@ -45,7 +46,9 @@
from git_config import GetSchemeFromUrl
from git_config import GetUrlCookieFile
from git_config import GitConfig
+from git_config import ID_RE
from git_config import IsId
+from git_config import RefSpec
from git_refs import GitRefs
from git_refs import HEAD
from git_refs import R_HEADS
@@ -1230,6 +1233,83 @@
logger.error("error: Cannot extract archive %s: %s", tarpath, e)
return False
+ def CachePopulate(self, cache_dir, url):
+ """Populate cache in the cache_dir.
+
+ Args:
+ cache_dir: Directory to cache git files from Google Storage.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if it fails to populate the git cache.
+ """
+ cmd = [
+ "cache",
+ "populate",
+ "--ignore_locks",
+ "-v",
+ "--cache-dir",
+ cache_dir,
+ url,
+ ]
+
+ if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0:
+ raise CacheApplyError(
+ "Failed to populate cache. cache_dir: %s "
+ "url: %s" % (cache_dir, url)
+ )
+
+ def CacheExists(self, cache_dir, url):
+ """Check the existence of the cache files.
+
+ Args:
+ cache_dir: Directory to cache git files.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if the cache files do not exist.
+ """
+ cmd = ["cache", "exists", "--quiet", "--cache-dir", cache_dir, url]
+
+ exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True)
+ if exist.Wait() != 0:
+ raise CacheApplyError(
+ "Failed to execute git cache exists cmd. "
+ "cache_dir: %s url: %s" % (cache_dir, url)
+ )
+
+ if not exist.stdout or not exist.stdout.strip():
+ raise CacheApplyError(
+ "Failed to find cache. cache_dir: %s "
+ "url: %s" % (cache_dir, url)
+ )
+ return exist.stdout.strip()
+
+ def CacheApply(self, cache_dir):
+ """Apply git cache files populated from Google Storage buckets.
+
+ Args:
+ cache_dir: Directory to cache git files.
+
+ Raises:
+ CacheApplyError if it fails to apply git caches.
+ """
+ remote = self.GetRemote(self.remote.name)
+
+ self.CachePopulate(cache_dir, remote.url)
+
+ mirror_dir = self.CacheExists(cache_dir, remote.url)
+
+ refspec = RefSpec(
+ True, "refs/heads/*", "refs/remotes/%s/*" % remote.name
+ )
+
+ fetch_cache_cmd = ["fetch", mirror_dir, str(refspec)]
+ if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0:
+ raise CacheApplyError(
+ "Failed to fetch refs %s from %s" % (mirror_dir, str(refspec))
+ )
+
def Sync_NetworkHalf(
self,
quiet=False,
@@ -1245,6 +1325,7 @@
retry_fetches=0,
prune=False,
submodules=False,
+ cache_dir=None,
ssh_proxy=None,
clone_filter=None,
partial_clone_exclude=set(),
@@ -1355,8 +1436,23 @@
else:
alt_dir = None
+ applied_cache = False
+ # If cache_dir is provided, and it's a new repository without
+ # alternative_dir, bootstrap this project repo with the git
+ # cache files.
+ if cache_dir is not None and is_new and alt_dir is None:
+ try:
+ self.CacheApply(cache_dir)
+ applied_cache = True
+ is_new = False
+ except CacheApplyError as e:
+ _error("Could not apply git cache: %s", e)
+ _error("Please check if you have the right GS credentials.")
+ _error("Please check if the cache files exist in GS.")
+
if (
clone_bundle
+ and not applied_cache
and alt_dir is None
and self._ApplyCloneBundle(
initial=is_new, quiet=quiet, verbose=verbose
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 113e7a6..83e98ed 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -525,6 +525,15 @@
help="do not delete refs that no longer exist on the remote",
)
p.add_option(
+ "--cache-dir",
+ dest="cache_dir",
+ action="store",
+ help="Use git-cache to populate project cache into this "
+ "directory. Bootstrap the local repository from this "
+ "directory if the project cache exists. This applies "
+ "to the projects on chromium and chrome-internal.",
+ )
+ p.add_option(
"--auto-gc",
action="store_true",
default=None,
@@ -744,6 +753,7 @@
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
prune=opt.prune,
+ cache_dir=opt.cache_dir,
ssh_proxy=self.ssh_proxy,
clone_filter=project.manifest.CloneFilter,
partial_clone_exclude=project.manifest.PartialCloneExclude,
@@ -1583,6 +1593,7 @@
clone_filter=mp.manifest.CloneFilter,
partial_clone_exclude=mp.manifest.PartialCloneExclude,
clone_filter_for_depth=mp.manifest.CloneFilterForDepth,
+ cache_dir=opt.cache_dir,
)
if result.error:
errors.append(result.error)
@@ -1696,6 +1707,24 @@
if not opt.outer_manifest:
manifest = self.manifest
+ cache_dir = opt.cache_dir
+ if cache_dir:
+ if manifest.IsMirror or manifest.IsArchive:
+ print(
+ "fatal: --cache-dir is not supported with mirror or "
+ "archive repository."
+ )
+ sys.exit(1)
+
+ if os.path.isfile(cache_dir):
+ print(
+ "fatal: %s: cache_dir must be a directory",
+ cache_dir,
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ os.makedirs(opt.cache_dir, exist_ok=True)
+
if opt.manifest_name:
manifest.Override(opt.manifest_name)