Add a uuid to the manifest and use it to invalidate the mtime cache The mtime cache is valid as long as the manifest we're updating is the same one that was computed when the mtime cache was written. Instead of associtaing these by path and mtime, just put a uuid in the manifest itself, updated each time it's written, and write that to the mtime cache.
diff --git a/tools/manifest/manifest.py b/tools/manifest/manifest.py index f27203d..daab1a1 100644 --- a/tools/manifest/manifest.py +++ b/tools/manifest/manifest.py
@@ -2,6 +2,7 @@ import itertools import json import os +import uuid from copy import deepcopy from multiprocessing import Pool, cpu_count from six import ( @@ -42,7 +43,7 @@ except ImportError: fast_json = json # type: ignore -CURRENT_VERSION = 8 # type: int +CURRENT_VERSION = 9 # type: int class ManifestError(Exception): @@ -117,14 +118,14 @@ return rv - class Manifest(object): - def __init__(self, tests_root=None, url_base="/"): + def __init__(self, tests_root=None, url_base="/", manifest_uuid=None): # type: (Optional[str], Text) -> None assert url_base is not None self._data = ManifestData(self) # type: ManifestData self.tests_root = tests_root # type: Optional[str] self.url_base = url_base # type: Text + self.uuid = manifest_uuid # type: Text def __iter__(self): # type: () -> Iterator[Tuple[str, Text, Set[ManifestItem]]] @@ -273,6 +274,7 @@ rv = {"url_base": self.url_base, "items": out_items, + "uuid": ensure_text(uuid.uuid4().hex), "version": CURRENT_VERSION} # type: Dict[Text, Any] return rv @@ -295,7 +297,9 @@ if version != CURRENT_VERSION: raise ManifestVersionMismatch - self = cls(tests_root, url_base=obj.get("url_base", "/")) + self = cls(tests_root, + url_base=obj.get("url_base", "/"), + manifest_uuid=obj.get("uuid")) if not hasattr(obj, "items"): raise ManifestError @@ -413,9 +417,12 @@ else: # If we didn't break there was an error raise - if write_manifest and changed: - write(manifest, manifest_path) - tree.dump_caches() + if write_manifest: + if changed: + uuid = write(manifest, manifest_path) + else: + uuid = manifest.uuid + tree.dump_caches(uuid) return manifest @@ -428,6 +435,8 @@ with open(manifest_path, "w") as f: # Use ',' instead of the default ', ' separator to prevent trailing # spaces: https://docs.python.org/2/library/json.html#json.dump - json.dump(manifest.to_json(caller_owns_obj=True), f, + manifest_json = manifest.to_json(caller_owns_obj=True) + json.dump(manifest_json, f, sort_keys=True, indent=1, separators=(',', ': ')) f.write("\n") + return manifest_json["uuid"]
diff --git a/tools/manifest/vcs.py b/tools/manifest/vcs.py index dcd4207..393d9b4 100644 --- a/tools/manifest/vcs.py +++ b/tools/manifest/vcs.py
@@ -48,7 +48,8 @@ manifest.url_base, manifest_path=manifest_path, cache_path=cache_root, - rebuild=rebuild) + rebuild=rebuild, + manifest_uuid=manifest.uuid) return tree @@ -90,7 +91,8 @@ class FileSystem(object): - def __init__(self, root, url_base, cache_path, manifest_path=None, rebuild=False): + def __init__(self, root, url_base, cache_path, manifest_path=None, rebuild=False, + manifest_uuid=None): # type: (bytes, Text, Optional[bytes], Optional[bytes], bool) -> None self.root = os.path.abspath(root) self.url_base = url_base @@ -98,7 +100,8 @@ self.mtime_cache = None if cache_path is not None: if manifest_path is not None: - self.mtime_cache = MtimeCache(cache_path, root, manifest_path, rebuild) + self.mtime_cache = MtimeCache(cache_path, root, manifest_path, manifest_uuid, + rebuild) if gitignore.has_ignore(root): self.ignore_cache = GitIgnoreCache(cache_path, root, rebuild) self.path_filter = gitignore.PathFilter(self.root, @@ -122,11 +125,11 @@ else: yield path, None, False - def dump_caches(self): + def dump_caches(self, new_manifest_uuid): # type: () -> None for cache in [self.mtime_cache, self.ignore_cache]: if cache is not None: - cache.dump() + cache.dump(new_manifest_uuid) class CacheFile(with_metaclass(abc.ABCMeta)): @@ -144,7 +147,7 @@ # type: () -> bytes pass - def dump(self): + def dump(self, new_manifest_uuid): # type: () -> None if not self.modified: return @@ -176,9 +179,11 @@ class MtimeCache(CacheFile): file_name = "mtime.json" - def __init__(self, cache_root, tests_root, manifest_path, rebuild=False): - # type: (bytes, bytes, bytes, bool) -> None + def __init__(self, cache_root, tests_root, manifest_path, manifest_uuid, + rebuild=False): + # type: (bytes, bytes, bytes, text, bool) -> None self.manifest_path = manifest_path + self.manifest_uuid = manifest_uuid super(MtimeCache, self).__init__(cache_root, tests_root, rebuild) def updated(self, rel_path, stat): @@ -195,30 +200,20 @@ def check_valid(self, data): # type: (Dict[Any, Any]) -> Dict[Any, Any] - if data.get("/tests_root") != self.tests_root: - self.modified = True - else: - if self.manifest_path is not None and os.path.exists(self.manifest_path): - mtime = os.path.getmtime(self.manifest_path) - if data.get("/manifest_path") != [self.manifest_path, mtime]: - self.modified = True - else: - self.modified = True + self.modified = not self.manifest_uuid or data.get("/manifest_uuid") != self.manifest_uuid if self.modified: data = {} - data["/tests_root"] = self.tests_root return data - def dump(self): + def dump(self, new_manifest_uuid): # type: () -> None - if self.manifest_path is None: - raise ValueError if not os.path.exists(self.manifest_path): return - mtime = os.path.getmtime(self.manifest_path) - self.data["/manifest_path"] = [self.manifest_path, mtime] - self.data["/tests_root"] = self.tests_root - super(MtimeCache, self).dump() + if not self.modified and self.manifest_uuid == new_manifest_uuid: + return + self.data["/manifest_uuid"] = new_manifest_uuid + self.manifest_uuid = new_manifest_uuid + super(MtimeCache, self).dump(new_manifest_uuid) class GitIgnoreCache(CacheFile, MutableMapping): # type: ignore