Merge commits up to v2.13.8 * tag 'v2.13.8': forall: handle missing project refs better sync: imply -c if --use-superproject option is used. start: add a --HEAD alias forall/list: delete spurious " superproject: Added --depth=1 argument to git fetch command. Change-Id: Ic2e4e637907d78914206bd4818446b65be6034da
diff --git a/error.py b/error.py index 25ff80d..524d521 100644 --- a/error.py +++ b/error.py
@@ -137,3 +137,7 @@ The common case is that the file wasn't present when we tried to run it. """ + +class CacheApplyError(Exception): + """Thrown when errors happen in 'repo sync' with '--cache-dir' option. + """
diff --git a/project.py b/project.py index 2567c57..7f6e236 100644 --- a/project.py +++ b/project.py
@@ -30,8 +30,9 @@ from color import Coloring from git_command import GitCommand, git_require from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ - ID_RE + ID_RE, RefSpec from error import GitError, UploadError, DownloadError +from error import CacheApplyError from error import ManifestInvalidRevisionError, ManifestInvalidPathError from error import NoManifestException import platform_utils @@ -1036,6 +1037,68 @@ _error("Cannot extract archive %s: %s", tarpath, str(e)) return False + def CachePopulate(self, cache_dir, url): + """Populate cache in the cache_dir. + + Args: + cache_dir: Directory to cache git files from Google Storage. + url: Git url of current repository. + + Raises: + CacheApplyError if it fails to populate the git cache. + """ + cmd = ['cache', 'populate', '--ignore_locks', '-v', + '--cache-dir', cache_dir, url] + + if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0: + raise CacheApplyError('Failed to populate cache. cache_dir: %s ' + 'url: %s' % (cache_dir, url)) + + def CacheExists(self, cache_dir, url): + """Check the existence of the cache files. + + Args: + cache_dir: Directory to cache git files. + url: Git url of current repository. + + Raises: + CacheApplyError if the cache files do not exist. + """ + cmd = ['cache', 'exists', '--quiet', '--cache-dir', cache_dir, url] + + exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True) + if exist.Wait() != 0: + raise CacheApplyError('Failed to execute git cache exists cmd. ' + 'cache_dir: %s url: %s' % (cache_dir, url)) + + if not exist.stdout or not exist.stdout.strip(): + raise CacheApplyError('Failed to find cache. cache_dir: %s ' + 'url: %s' % (cache_dir, url)) + return exist.stdout.strip() + + def CacheApply(self, cache_dir): + """Apply git cache files populated from Google Storage buckets. + + Args: + cache_dir: Directory to cache git files. + + Raises: + CacheApplyError if it fails to apply git caches. + """ + remote = self.GetRemote(self.remote.name) + + self.CachePopulate(cache_dir, remote.url) + + mirror_dir = self.CacheExists(cache_dir, remote.url) + + refspec = RefSpec(True, 'refs/heads/*', + 'refs/remotes/%s/*' % remote.name) + + fetch_cache_cmd = ['fetch', mirror_dir, str(refspec)] + if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0: + raise CacheApplyError('Failed to fetch refs %s from %s' % + (mirror_dir, str(refspec))) + def Sync_NetworkHalf(self, quiet=False, verbose=False, @@ -1050,7 +1113,8 @@ retry_fetches=0, prune=False, submodules=False, - clone_filter=None): + clone_filter=None, + cache_dir=None): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ @@ -1106,7 +1170,22 @@ else: alt_dir = None + applied_cache = False + # If cache_dir is provided, and it's a new repository without + # alternative_dir, bootstrap this project repo with the git + # cache files. + if cache_dir is not None and is_new and alt_dir is None: + try: + self.CacheApply(cache_dir) + applied_cache = True + is_new = False + except CacheApplyError as e: + _error('Could not apply git cache: %s', e) + _error('Please check if you have the right GS credentials.') + _error('Please check if the cache files exist in GS.') + if (clone_bundle + and not applied_cache and alt_dir is None and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)): is_new = False
diff --git a/repo b/repo index 768050e..a7c8dc6 100755 --- a/repo +++ b/repo
@@ -141,7 +141,7 @@ # REPO_URL = os.environ.get('REPO_URL', None) if not REPO_URL: - REPO_URL = 'https://gerrit.googlesource.com/git-repo' + REPO_URL = 'https://chromium.googlesource.com/external/repo' REPO_REV = os.environ.get('REPO_REV') if not REPO_REV: REPO_REV = 'stable'
diff --git a/subcmds/sync.py b/subcmds/sync.py index b14ad24..ba016e8 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py
@@ -252,6 +252,11 @@ help='number of times to retry fetches on transient errors') p.add_option('--prune', dest='prune', action='store_true', help='delete refs that no longer exist on the remote') + p.add_option('--cache-dir', dest='cache_dir', action='store', + help='Use git-cache to populate project cache into this ' + 'directory. Bootstrap the local repository from this ' + 'directory if the project cache exists. This applies ' + 'to the projects on chromium and chrome-internal.') if show_smart: p.add_option('-s', '--smart-sync', dest='smart_sync', action='store_true', @@ -380,6 +385,7 @@ optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, prune=opt.prune, + cache_dir=opt.cache_dir, clone_filter=clone_filter) self._fetch_times.Set(project, time.time() - start) @@ -751,7 +757,8 @@ optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, submodules=self.manifest.HasSubmodules, - clone_filter=self.manifest.CloneFilter) + clone_filter=self.manifest.CloneFilter, + cache_dir=opt.cache_dir) finish = time.time() self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, start, finish, success) @@ -797,6 +804,18 @@ opt.quiet = opt.output_mode is False opt.verbose = opt.output_mode is True + cache_dir = opt.cache_dir + if cache_dir: + if self.manifest.IsMirror or self.manifest.IsArchive: + print('fatal: --cache-dir is not supported with mirror or archive ' + 'repository.') + sys.exit(1) + + if os.path.isfile(cache_dir): + print('fatal: %s: cache_dir must be a directory', cache_dir, file=sys.stderr) + sys.exit(1) + os.makedirs(opt.cache_dir, exist_ok=True) + if opt.manifest_name: self.manifest.Override(opt.manifest_name)