Reland "Stop using compression for git cache update bootstrap."
This is a reland of 749139969f34788a65dc1e5e27484325035129ce
Original change's description:
> Stop using compression for git cache update bootstrap.
>
> Change git cache to stop zipping the git directory and instead upload directly to GS.
>
> Bug: 943696
> Change-Id: I310fb48ff3d7fd6781914e14e0e17530d5d9b328
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585029
> Commit-Queue: Karen Qian <karenqian@google.com>
> Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Bug: 943696
Change-Id: Ifd3a3a07445345ab647572682ebc6423fb7135b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585272
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Karen Qian <karenqian@google.com>
diff --git a/git_cache.py b/git_cache.py
index 4a2a237..6b20a44 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -258,6 +258,10 @@
# Not recognized.
return None
+ @property
+ def _gs_path(self):
+ return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
+
@classmethod
def FromPath(cls, path):
return cls(cls.CacheDirToUrl(path))
@@ -600,28 +604,43 @@
lockfile.unlock()
def update_bootstrap(self, prune=False):
- # The files are named <git number>.zip
+ # The folder is <git number>
gen_number = subprocess.check_output(
[self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
# Run Garbage Collect to compress packfile.
self.RunGit(['gc', '--prune=all'])
- # Creating a temp file and then deleting it ensures we can use this name.
- _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
- os.remove(tmp_zipfile)
- subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
- gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
- gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
- dest_name = '%s/%s.zip' % (gs_folder, gen_number)
- gsutil.call('cp', tmp_zipfile, dest_name)
- os.remove(tmp_zipfile)
- # Remove all other files in the same directory.
- if prune:
- _, ls_out, _ = gsutil.check_call('ls', gs_folder)
- for filename in ls_out.splitlines():
- if filename == dest_name:
- continue
- gsutil.call('rm', filename)
+ gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
+
+ src_name = self.mirror_path
+ dest_name = '%s/%s' % (self._gs_path, gen_number)
+
+ # check to see if folder already exists in gs
+ _, ls_out, ls_err = gsutil.check_call('ls', dest_name)
+ _, ls_out_ready, ls_err_ready = (
+ gsutil.check_call('ls', dest_name + '.ready'))
+
+ # only printing out errors because the folder/ready file
+ # might not exist yet, so it will error no matter what
+ if ls_err:
+ print('Failed to check GS:\n%s' % (ls_err))
+
+ if ls_err_ready:
+ print('Failed to check GS:\n%s' % (ls_err_ready))
+
+ if not (ls_out == '' and ls_out_ready == ''):
+ return
+
+ gsutil.call('-m', 'cp', '-r', src_name, dest_name)
+
+ #TODO(karenqian): prune old caches
+
+ # create .ready file and upload
+ _, ready_file_name = tempfile.mkstemp(suffix='.ready')
+ try:
+ gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name))
+ finally:
+ os.remove(ready_file_name)
@staticmethod
def DeleteTmpPackFiles(path):
