Provide --cache-dir option in repo sync.
With the --cache-dir option, sync opertaion will first try to use git-cache
to populate the project cache in the cache_dir, then bootstrap the local
repository based on the project cache. This option only applies to projects
on Chromium and Chrome-internal.
BUG=chromium:632203
TEST=repo sync; repo sync project-name;
Change-Id: Ibf15c552108fc3a66545ff876f6b7548d7b84ac5
diff --git a/error.py b/error.py
index f2a7c4e..1050b8d 100644
--- a/error.py
+++ b/error.py
@@ -111,3 +111,7 @@
The common case is that the file wasn't present when we tried to run it.
"""
+
+class CacheApplyError(Exception):
+ """Thrown when errors happen in 'repo sync' with '--cache-dir' option.
+ """
diff --git a/project.py b/project.py
index 46e06bf..0eed02a 100644
--- a/project.py
+++ b/project.py
@@ -31,8 +31,9 @@
from color import Coloring
from git_command import GitCommand, git_require
from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
- ID_RE
+ ID_RE, RefSpec
from error import GitError, HookError, UploadError, DownloadError
+from error import CacheApplyError
from error import ManifestInvalidRevisionError
from error import NoManifestException
from trace import IsTrace, Trace
@@ -1181,6 +1182,68 @@
_error("Cannot extract archive %s: %s", tarpath, str(e))
return False
+ def CachePopulate(self, cache_dir, url):
+ """Populate cache in the cache_dir.
+
+ Args:
+ cache_dir: Directory to cache git files from Google Storage.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if it fails to populate the git cache.
+ """
+ cmd = ['cache', 'populate', '--ignore_locks', '-v',
+ '--cache-dir', cache_dir, url]
+
+ if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0:
+ raise CacheApplyError('Failed to populate cache. cache_dir: %s '
+ 'url: %s' % (cache_dir, url))
+
+ def CacheExists(self, cache_dir, url):
+ """Check the existence of the cache files.
+
+ Args:
+ cache_dir: Directory to cache git files.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if the cache files do not exist.
+ """
+ cmd = ['cache', 'exists', '--quiet', '--cache-dir', cache_dir, url]
+
+ exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True)
+ if exist.Wait() != 0:
+ raise CacheApplyError('Failed to execute git cache exists cmd. '
+ 'cache_dir: %s url: %s' % (cache_dir, url))
+
+ if not exist.stdout or not exist.stdout.strip():
+ raise CacheApplyError('Failed to find cache. cache_dir: %s '
+ 'url: %s' % (cache_dir, url))
+ return exist.stdout.strip()
+
+ def CacheApply(self, cache_dir):
+ """Apply git cache files populated from Google Storage buckets.
+
+ Args:
+ cache_dir: Directory to cache git files.
+
+ Raises:
+ CacheApplyError if it fails to apply git caches.
+ """
+ remote = self.GetRemote(self.remote.name)
+
+ self.CachePopulate(cache_dir, remote.url)
+
+ mirror_dir = self.CacheExists(cache_dir, remote.url)
+
+ refspec = RefSpec(True, 'refs/heads/*',
+ 'refs/remotes/%s/*' % remote.name)
+
+ fetch_cache_cmd = ['fetch', mirror_dir, str(refspec)]
+ if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0:
+ raise CacheApplyError('Failed to fetch refs %s from %s' %
+ (mirror_dir, str(refspec)))
+
def Sync_NetworkHalf(self,
quiet=False,
is_new=None,
@@ -1190,7 +1253,8 @@
no_tags=False,
archive=False,
optimized_fetch=False,
- prune=False):
+ prune=False,
+ cache_dir=None):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
"""
@@ -1242,7 +1306,22 @@
else:
alt_dir = None
+ applied_cache = False
+ # If cache_dir is provided, and it's a new repository without
+ # alternative_dir, bootstrap this project repo with the git
+ # cache files.
+ if cache_dir is not None and is_new and alt_dir is None:
+ try:
+ self.CacheApply(cache_dir)
+ applied_cache = True
+ is_new = False
+ except CacheApplyError as e:
+ _error('Could not apply git cache: %s', e)
+ _error('Please check if you have the right GS credentials.')
+ _error('Please check if the cache files exist in GS.')
+
if clone_bundle \
+ and not applied_cache \
and alt_dir is None \
and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
is_new = False
diff --git a/subcmds/sync.py b/subcmds/sync.py
index ecf2ffc..bbcf6d7 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -14,6 +14,7 @@
# limitations under the License.
from __future__ import print_function
+import glob
import json
import netrc
from optparse import SUPPRESS_HELP
@@ -64,7 +65,7 @@
except ImportError:
multiprocessing = None
-from git_command import GIT, git_require
+from git_command import GIT, git_require, GitCommand
from git_config import GetUrlCookieFile
from git_refs import R_HEADS, HEAD
import gitc_utils
@@ -239,6 +240,11 @@
help='only fetch projects fixed to sha1 if revision does not exist locally')
p.add_option('--prune', dest='prune', action='store_true',
help='delete refs that no longer exist on the remote')
+ p.add_option('--cache-dir', dest='cache_dir', action='store',
+ help='Use git-cache to populate project cache into this '
+ 'directory. Bootstrap the local repository from this '
+ 'directory if the project cache exists. This applies '
+ 'to the projects on chromium and chrome-internal.')
if show_smart:
p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true',
@@ -311,7 +317,8 @@
clone_bundle=not opt.no_clone_bundle,
no_tags=opt.no_tags, archive=self.manifest.IsArchive,
optimized_fetch=opt.optimized_fetch,
- prune=opt.prune)
+ prune=opt.prune,
+ cache_dir=opt.cache_dir)
self._fetch_times.Set(project, time.time() - start)
# Lock around all the rest of the code, since printing, updating a set
@@ -547,6 +554,30 @@
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
+ cache_dir = opt.cache_dir
+ if cache_dir:
+ if self.manifest.IsMirror or self.manifest.IsArchive:
+ print('fatal: --cache-dir is not supported with mirror or archive '
+ 'repository.')
+ sys.exit(1)
+
+ if os.path.exists(cache_dir):
+ if not os.path.isdir(cache_dir):
+ print('fatal: cache_dir must be a directory', file=sys.stderr)
+ sys.exit(1)
+ else:
+ # Unlock the locks in the cache_dir.
+ unlock_cmd = ['cache', 'unlock', '-vv', '--force', '--all',
+ '--cache-dir', cache_dir]
+ if GitCommand(None, unlock_cmd).Wait() != 0:
+ raise Exception('Failed to unlock cache_dir %s' % cache_dir)
+
+ locks = glob.glob(os.path.join(cache_dir, '*.lock'))
+ if locks:
+ raise Exception('Found %s after cache unlock.' % locks)
+ else:
+ os.makedirs(opt.cache_dir)
+
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
@@ -670,7 +701,8 @@
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
no_tags=opt.no_tags,
- optimized_fetch=opt.optimized_fetch)
+ optimized_fetch=opt.optimized_fetch,
+ cache_dir=cache_dir)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
