| #!/usr/bin/env python3 |
| # |
| # Updates upstream ANGLE into WebKit's Source/ThirdParty/ANGLE. |
| # |
| # Example use: |
| # cd ~ |
| # git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git |
| # export PATH=$PATH:$HOME/depot_tools |
| # |
| # cd ~/WebKit/OpenSource |
| # Tools/Scripts/update-angle --work-dir /tmp/angle-update |
| # When it stops for conflict resolution: |
| # a) open a tool to resolve it: |
| # code /tmp/angle-update/merge |
| # b) .. or resolve it from command line: |
| # cd /tmp/angle-update/merge && git status |
| # ... |
| # git add ... |
| # Continue the update: |
| # cd ~/WebKit/OpenSource |
| # Tools/Scripts/update-angle --continue |
| # |
| # Phases (see PHASES). The run is resumable: each phase records progress in a |
| # state file at the WebKit root, so a pause (merge conflicts) or failure can be |
| # resumed with --continue, or discarded with --abort. |
| # |
| # prepare Create a temp upstream checkout, fetch upstream, and resolve the |
| # previous commit (from ANGLE.plist) and the target commit. |
| # sync Write a .gclient and run `gclient sync` to populate third_party. |
| # Syncs at the previous revision (to capture the vendored merge |
| # base) and then at the target revision. |
| # merge 3-way merge upstream into WebKit's tree (may pause on conflicts). |
| # See the "Merge" section below for how/why this works. |
| # thirdparty Bring the vendored third_party subset (VENDORED) from the sync |
| # into the merged tree, 3-way-merging WebKit's local edits to |
| # vendored text files (binary/data files are overwritten). |
| # codegen_a Run upstream scripts/run_code_generation.py on the merged tree. |
| # codegen_b Regenerate WebKit-specific files: angle_commit.h, the *.cmake |
| # build files, and ANGLEShaderProgramVersion.h. |
| # bookkeep Update ANGLE.plist and changes.diff, and write |
| # the commit-message draft. |
| # finalize Export the merged tree into Source/ThirdParty/ANGLE, stage it, |
| # and print the remaining manual steps. |
| # |
| # python3 -m autopep8 --in-place --max-line-length 200 Tools/Scripts/update-angle |
| # |
| |
| import argparse |
| import glob |
| import json |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| |
| SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| WEBKIT_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, "..", "..")) |
| ANGLE_REL = os.path.join("Source", "ThirdParty", "ANGLE") |
| ANGLE_DIR = os.path.join(WEBKIT_ROOT, ANGLE_REL) |
| |
| COMMIT_MSG_FILE = os.path.join(WEBKIT_ROOT, "angle-update.COMMIT_EDITMSG") |
| STATE_FILE = os.path.join(WEBKIT_ROOT, "update-angle-state.txt") |
| |
| UPSTREAM_URL = "https://chromium.googlesource.com/angle/angle" |
| |
| PHASES = [ |
| "prepare", |
| "sync", |
| "merge", |
| "thirdparty", |
| "codegen_a", |
| "codegen_b", |
| "bookkeep", |
| "finalize", |
| ] |
| |
| # Explicitly vendored third_party source: dropped from the main merge, then |
| # brought in from the synced checkout by the `thirdparty` phase, which 3-way |
| # merges WebKit's local edits to vendored text files (binary/data overwritten). |
| # Extend as the build needs more upstream third_party sources. |
| _MUSTPASS = ("third_party/VK-GL-CTS/src/external/openglcts/data/gl_cts/data/" |
| "mustpass/gles/aosp_mustpass/main") |
| VENDORED = [ |
| "src/third_party/khronos/**", |
| "src/third_party/volk/**", |
| "src/third_party/ceval/**", |
| "src/third_party/libXNVCtrl/**", |
| "third_party/VK-GL-CTS/src/framework/**", |
| "third_party/VK-GL-CTS/src/execserver/**", |
| "third_party/VK-GL-CTS/src/executor/**", |
| "third_party/VK-GL-CTS/src/modules/gles2/**", |
| "third_party/VK-GL-CTS/src/modules/gles3/**", |
| "third_party/VK-GL-CTS/src/modules/glshared/**", |
| "third_party/VK-GL-CTS/src/data/gles2/**", |
| "third_party/VK-GL-CTS/src/data/gles3/**", |
| _MUSTPASS + "/gles2-main.txt", |
| _MUSTPASS + "/gles3-main.txt", |
| ] |
| |
| # Everything else dropped from the merge: WebKit-owned files (must never be |
| # touched by the merge), generated files (regenerated by codegen_b), and |
| # upstream paths WebKit never builds. |
| DROP = [ |
| # WebKit-owned. |
| "ANGLE.plist", |
| "ANGLE.xcodeproj/**", |
| "Configurations/**", |
| "**/CMakeLists.txt", |
| "*.cmake", |
| "linux.cmake", |
| "Makefile", |
| "Android.mk", |
| "adjust-angle-include-paths*", |
| "gni-to-cmake.py", |
| "ANGLEPrefix.h", |
| "changes.diff", |
| "WebKit/**", |
| # Upstream paths WebKit never builds. |
| "third_party/*/src/**", |
| "src/third_party/**", |
| "infra/**", |
| "parsetab.py", |
| "**/__pycache__/**", |
| "**/.DS_Store", |
| "third_party/googletest/**", |
| "third_party/kotlin_stdlib*", |
| "third_party/r8/custom_d8.jar", |
| "third_party/re2/**", |
| ] |
| |
| |
| _DROP_PATHSPECS = [":(glob)" + p for p in VENDORED + DROP] |
| |
| # Pathspec excludes for changes.diff: the :(exclude,glob) form of the same set. |
| _CHANGES_DIFF_EXCLUDES = [":(exclude,glob)" + p for p in VENDORED + DROP] |
| |
| |
| VERBOSE = False |
| QUIET = False |
| |
| |
| def log(msg): |
| """Progress output. Suppressed in --quiet mode.""" |
| if QUIET: |
| return |
| print(("[update-angle] " + msg) if VERBOSE else msg, flush=True) |
| |
| |
| def warn(msg): |
| """Warning. Always printed, even in --quiet mode.""" |
| print("[update-angle] WARNING: " + msg, flush=True) |
| |
| |
| def vlog(msg): |
| if VERBOSE: |
| log(msg) |
| |
| |
| def run(cmd, cwd=None, env=None, check=True, capture=False): |
| vlog("$ " + " ".join(cmd) + (" (cwd=%s)" % cwd if cwd else "")) |
| if capture: |
| return subprocess.run(cmd, cwd=cwd, env=env, check=check, |
| stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| text=True) |
| return subprocess.run(cmd, cwd=cwd, env=env, check=check) |
| |
| |
| def git(args, cwd, check=True, capture=True, index_file=None, strip=True): |
| env = None |
| if index_file is not None: |
| env = dict(os.environ) |
| env["GIT_INDEX_FILE"] = index_file |
| # Stable identity (env beats git config) for every git invocation, so any |
| # commit -- commit-tree, the `git merge` auto-commit (via gitcode), and the |
| # post-resolution commit -- never prompts or fails for lack of one. |
| ident = { |
| "GIT_AUTHOR_NAME": "ANGLE Update", |
| "GIT_AUTHOR_EMAIL": "update-angle@webkit.org", |
| "GIT_COMMITTER_NAME": "ANGLE Update", |
| "GIT_COMMITTER_EMAIL": "update-angle@webkit.org", |
| "GIT_AUTHOR_DATE": "2000-01-01T00:00:00Z", |
| "GIT_COMMITTER_DATE": "2000-01-01T00:00:00Z", |
| } |
| if env is None: |
| env = dict(os.environ) |
| env.update(ident) |
| try: |
| res = run(["git"] + args, cwd=cwd, env=env, check=check, capture=True) |
| except subprocess.CalledProcessError as e: |
| fail("git %s failed (exit %d):\n%s" |
| % (" ".join(args), e.returncode, (e.stderr or "").strip())) |
| if capture: |
| out = res.stdout if res.stdout is not None else "" |
| # strip=False preserves exact bytes, e.g. the leading space of a |
| # worktree-only `git status --porcelain` line. |
| return out.strip() if strip else out |
| return res |
| |
| |
| def gitcode(args, cwd): |
| """Run git via git() and return its exit code (never raises).""" |
| return git(args, cwd, check=False, capture=False).returncode |
| |
| |
| class State: |
| def __init__(self): |
| self.path = STATE_FILE |
| self.data = {} |
| |
| def load(self): |
| if os.path.exists(self.path): |
| with open(self.path) as f: |
| self.data = json.load(f) |
| return self.data |
| |
| def save(self): |
| with open(self.path, "w") as f: |
| json.dump(self.data, f, indent=2) |
| |
| def remove(self): |
| if os.path.exists(self.path): |
| os.remove(self.path) |
| |
| def set_phase(self, phase): |
| self.data["phase"] = phase |
| self.save() |
| |
| |
| def fail(msg): |
| print("[update-angle] ERROR: " + msg, file=sys.stderr, flush=True) |
| sys.exit(1) |
| |
| |
| class Paused(Exception): |
| def __init__(self, phase, instructions): |
| self.phase = phase |
| self.instructions = instructions |
| |
| |
| def materialize(work, commit): |
| """Build a tree object from `commit`, dropping the VENDORED + DROP paths |
| and submodule gitlinks (keeping what gets merged). Returns the tree SHA. |
| Uses a scratch index, no working tree.""" |
| scratch = os.path.join(work, ".materialize.index") |
| if os.path.exists(scratch): |
| os.remove(scratch) |
| git(["read-tree", commit], cwd=work, index_file=scratch) |
| remove = set() |
| # Dropped paths: matched by git's own glob engine via _DROP_PATHSPECS. |
| matched = git(["ls-files", "-z", "--"] + _DROP_PATHSPECS, |
| cwd=work, index_file=scratch, strip=False) |
| remove.update(p for p in matched.split("\0") if p) |
| # Submodule gitlinks (mode 160000) are always dropped -- WebKit never uses |
| # git submodules in its ANGLE copy, and a gitlink has no children, so the |
| # path globs above can't catch it. ls-files -s exposes the mode. |
| for line in git(["ls-files", "-s"], cwd=work, index_file=scratch).split("\n"): |
| if not line: |
| continue |
| meta, path = line.split("\t", 1) |
| if meta.split(" ", 1)[0] == "160000": |
| remove.add(path) |
| vlog("materialize(%s): dropping %d entries" % (commit[:10], len(remove))) |
| if remove: |
| # Feed paths NUL-separated to avoid arg-length / quoting issues. |
| env = dict(os.environ) |
| env["GIT_INDEX_FILE"] = scratch |
| subprocess.run( |
| ["git", "update-index", "-z", "--force-remove", "--stdin"], |
| cwd=work, env=env, input="\0".join(remove) + "\0", text=True, |
| check=True) |
| tree = git(["write-tree"], cwd=work, index_file=scratch) |
| os.remove(scratch) |
| return tree |
| |
| |
| def phase_prepare(ctx): |
| """Create the temp upstream checkout, fetch upstream, resolve commits, |
| wire WebKit's object store as an alternate so we can read WebKit's ANGLE |
| tree without fetching all of WebKit's history.""" |
| work = ctx["work"] |
| os.makedirs(work, exist_ok=True) |
| |
| if not os.path.exists(os.path.join(work, ".git")): |
| git(["init", "-q", work], cwd=WEBKIT_ROOT) |
| git(["remote", "add", "upstream", ctx["upstream_url"]], cwd=work) |
| # Diff drivers used by ANGLE's .gitattributes (*.mm -> objcpp, |
| # *.h -> objcppheader) so merge/diff hunk headers (xfuncname) are |
| # formatted correctly. Shared by the merge worktree via the common |
| # config. |
| git(["config", "diff.objcpp.xfuncname", r"^[-+@a-zA-Z_].*$"], cwd=work) |
| git(["config", "diff.objcppheader.xfuncname", r"^[@a-zA-Z_].*$"], cwd=work) |
| |
| # Read the previously-imported upstream commit from ANGLE.plist. |
| plist = os.path.join(ANGLE_DIR, "ANGLE.plist") |
| with open(plist) as f: |
| m = re.search(r"[a-f0-9]{40}", f.read()) |
| if not m: |
| fail("Could not find previous upstream hash in ANGLE.plist") |
| prev_hash = m.group(0) |
| |
| log("Fetching upstream ANGLE (target=%s) ..." % ctx["target"]) |
| git(["fetch", "-q", "upstream", "main"], cwd=work) |
| # The previous commit is normally an ancestor of main and thus already |
| # present. Only fetch it if missing, and WITHOUT --depth: a shallow repo |
| # would truncate `git rev-list HEAD --count`, giving a wrong |
| # ANGLE_COMMIT_POSITION in angle_commit.h. |
| if gitcode(["cat-file", "-e", prev_hash + "^{commit}"], work) != 0: |
| gitcode(["fetch", "-q", "upstream", prev_hash], work) |
| # The target is usually upstream/main (just fetched). If a specific commit |
| # not reachable from main was requested, fetch it explicitly. |
| target = ctx["target"] |
| if gitcode(["rev-parse", "-q", "--verify", target + "^{commit}"], work) != 0: |
| gitcode(["fetch", "-q", "upstream", target], work) |
| |
| target_hash = git(["rev-parse", target], cwd=work) |
| prev_hash = git(["rev-parse", prev_hash], cwd=work) |
| |
| # Make WebKit's objects readable from the work repo (for the `ours` tree). |
| alt = os.path.join(work, ".git", "objects", "info", "alternates") |
| webkit_objects = os.path.join(WEBKIT_ROOT, ".git", "objects") |
| with open(alt, "w") as f: |
| f.write(webkit_objects + "\n") |
| |
| ctx["state"].data["previous_hash"] = prev_hash |
| ctx["state"].data["target_hash"] = target_hash |
| log("previous=%s target=%s" % (prev_hash[:12], target_hash[:12])) |
| |
| # Check out the target so we can sync + run upstream codegen against it. |
| git(["checkout", "-q", "-f", target_hash], cwd=work) |
| |
| |
| def _is_text_file(path): |
| """Heuristic: a file is text (3-way mergeable) unless it has a NUL byte.""" |
| try: |
| with open(path, "rb") as f: |
| return b"\0" not in f.read(8192) |
| except OSError: |
| return False |
| |
| |
| def _iter_vendored(root): |
| """Yield (relpath, abspath) for every VENDORED-matched file under `root`, |
| skipping directories and anything inside a .git.""" |
| for pat in VENDORED: |
| for m in sorted(glob.glob(os.path.join(root, pat), recursive=True)): |
| rel = os.path.relpath(m, root) |
| if os.path.isfile(m) and ".git" not in rel.split(os.sep): |
| yield rel, m |
| |
| |
| def _gclient_sync(work): |
| log("Running gclient sync (this can take a while) ...") |
| # Hooks are kept on so clang-format is fetched for the codegen_a phase. |
| res = run(["gclient", "sync", "--no-history"], cwd=work, check=False) |
| if res.returncode != 0: |
| fail("gclient sync failed. Re-run with --skip-sync to vendor " |
| "third_party manually.") |
| |
| |
| def phase_sync(ctx): |
| """Write a minimal .gclient and run gclient sync in the work checkout. |
| |
| Syncs twice: first at the previous upstream revision to capture the vendored |
| files as they were then (the 3-way merge base used by `thirdparty`), then at |
| the target revision (the working state for codegen + the `theirs` vendored).""" |
| work = ctx["work"] |
| st = ctx["state"].data |
| if ctx["args"].skip_sync: |
| log("Skipping gclient sync (--skip-sync); vendored 3-way merge disabled.") |
| st["base_vendored_dir"] = None |
| return |
| # Disable every optional checkout so sync stays as small as possible. |
| custom_vars = { |
| "checkout_angle_mesa": False, |
| "checkout_angle_internal": False, |
| "checkout_angle_restricted_traces": False, |
| "checkout_angle_dawn_deps": False, |
| "checkout_angle_partition_alloc": False, |
| "checkout_angle_cl_deps": False, |
| } |
| gclient = ( |
| "solutions = [\n" |
| " {\n" |
| " \"name\": \".\",\n" |
| " \"url\": \"%s.git\",\n" |
| " \"deps_file\": \"DEPS\",\n" |
| " \"managed\": False,\n" |
| " \"custom_vars\": %s,\n" |
| " },\n" |
| "]\n" % (ctx["upstream_url"], repr(custom_vars)) # .gclient is Python: need True/False, not JSON |
| ) |
| with open(os.path.join(work, ".gclient"), "w") as f: |
| f.write(gclient) |
| |
| # Capture the previous revision's vendored TEXT files as the 3-way merge |
| # base, so WebKit's local edits to vendored files survive the roll instead |
| # of being overwritten. Binary/data files are never base-captured (they are |
| # overwritten wholesale by `thirdparty`). Sync at prev, copy out, then sync |
| # back to the target so the rest of the run sees the target state. |
| base_dir = os.path.join(ctx["work_root"], "base_vendored") |
| if os.path.exists(base_dir): |
| shutil.rmtree(base_dir) |
| log("Syncing previous revision (%s) to capture the vendored merge base ..." |
| % st["previous_hash"][:12]) |
| git(["checkout", "-q", "-f", st["previous_hash"]], cwd=work) |
| _gclient_sync(work) |
| nbase = 0 |
| for rel, abspath in _iter_vendored(work): |
| if not _is_text_file(abspath): |
| continue |
| dest = os.path.join(base_dir, rel) |
| os.makedirs(os.path.dirname(dest), exist_ok=True) |
| shutil.copy2(abspath, dest) |
| nbase += 1 |
| st["base_vendored_dir"] = base_dir |
| log("Captured %d base vendored text file(s)." % nbase) |
| |
| log("Syncing target revision (%s) ..." % st["target_hash"][:12]) |
| git(["checkout", "-q", "-f", st["target_hash"]], cwd=work) |
| _gclient_sync(work) |
| |
| |
| def _overlay_synced_deps(work, merge): |
| """Symlink gclient-synced content that exists in WORK but not in the MERGE |
| worktree, so codegen generators and clang-format can find their inputs |
| without copying gigabytes. Returns the list of created symlink paths (merge- |
| relative) for cleanup. Never shadows files that already exist in MERGE.""" |
| created = [] |
| for dirpath, dirnames, filenames in os.walk(work): |
| rel = os.path.relpath(dirpath, work) |
| rel = "" if rel == "." else rel |
| if ".git" in dirnames: |
| dirnames.remove(".git") |
| for d in list(dirnames): |
| rsub = os.path.join(rel, d) if rel else d |
| if not os.path.exists(os.path.join(merge, rsub)): |
| os.symlink(os.path.join(dirpath, d), os.path.join(merge, rsub)) |
| created.append(rsub) |
| dirnames.remove(d) # symlink covers the whole subtree |
| for fn in filenames: |
| rf = os.path.join(rel, fn) if rel else fn |
| mf = os.path.join(merge, rf) |
| if not os.path.lexists(mf): |
| os.symlink(os.path.join(dirpath, fn), mf) |
| created.append(rf) |
| return created |
| |
| |
| def phase_codegen_a(ctx): |
| """Run upstream run_code_generation.py for real against the MERGED tree, so |
| generated files reflect WebKit+upstream merged sources.""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| if ctx["args"].skip_codegen: |
| log("Skipping upstream codegen (--skip-codegen).") |
| return |
| |
| script = os.path.join(merge, "scripts", "run_code_generation.py") |
| if not os.path.exists(script): |
| warn("scripts/run_code_generation.py not found; skipping.") |
| return |
| |
| # Make the synced third_party / buildtools (and clang-format) visible. |
| links = _overlay_synced_deps(work, merge) |
| try: |
| log("Running upstream code generation on the merged tree ...") |
| res = run([sys.executable, "scripts/run_code_generation.py"], |
| cwd=merge, check=False) |
| if res.returncode != 0: |
| warn("run_code_generation.py returned %d; continuing with " |
| "merged outputs." % res.returncode) |
| finally: |
| # Remove the overlay symlinks so they never reach bookkeep/export. |
| for rel in links: |
| p = os.path.join(merge, rel) |
| if os.path.islink(p): |
| os.unlink(p) |
| |
| |
| # Merge: how upstream and WebKit content are combined, and why it works |
| # |
| # The merge operates on git *tree objects*, not on files in a checkout, so the |
| # fact that WebKit keeps ANGLE in a subdirectory never matters. Three trees: |
| # |
| # base = previous upstream commit, run through the path filter (materialize) |
| # theirs = target upstream commit, run through the SAME filter |
| # ours = WebKit's current ANGLE tree, read as `HEAD:Source/ThirdParty/ANGLE` |
| # |
| # `HEAD:Source/ThirdParty/ANGLE` resolves to the subtree object, whose entries |
| # are already ANGLE-root-relative -- byte-for-byte the same layout as upstream's |
| # root tree. So the two line up with no path rewriting (no git-filter-repo of |
| # WebKit history). `ours` is read from the WebKit repo via an alternates link |
| # wired up in `prepare`, so the temp work repo can see it without fetching any |
| # WebKit history. |
| # |
| # base/ours/theirs are wrapped into three commits that share `base` as their |
| # parent, then `git merge theirs` runs an ordinary 3-way merge with `base` as |
| # the merge base. Why it produces only real conflicts: |
| # |
| # * base and theirs go through the IDENTICAL filter (materialize drops the |
| # VENDORED + DROP paths and all submodule gitlinks). So anything WebKit |
| # excludes is absent from BOTH upstream sides -- WebKit's deletions have |
| # nothing to collide with, eliminating delete/modify ("deleted by them") |
| # conflicts. Only files WebKit actually modified can conflict. |
| # * theirs is the pure target tree (committed upstream generated files come |
| # through unchanged); WebKit-owned and generated files are dropped from the |
| # merge and regenerated afterwards (codegen_a/codegen_b), so they never |
| # conflict. |
| # |
| # `finalize` copies the merged tree back into the subdirectory -- the |
| # root->subdir direction is a plain directory copy, the inverse of reading the |
| # subtree above. |
| |
| |
| def phase_merge(ctx): |
| """Stage the 3-way snapshot merge in a dedicated worktree. |
| |
| See the "Merge" section comment above for how the three trees are formed |
| and why the merge yields only genuine (WebKit-modified) conflicts.""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| st = ctx["state"].data |
| |
| if os.path.exists(merge) and os.path.exists(os.path.join(merge, ".git")): |
| # Resuming: the worktree already exists. |
| _finish_merge(ctx) |
| return |
| |
| prev_hash = st["previous_hash"] |
| |
| log("Building filtered base / theirs trees ...") |
| base_tree = materialize(work, prev_hash) |
| # theirs is the pure target tree (committed upstream codegen outputs come |
| # through as-is); upstream codegen is re-run on the MERGED tree afterwards. |
| theirs_tree = materialize(work, st["target_hash"]) |
| ours_tree = git(["rev-parse", "HEAD:%s" % ANGLE_REL], cwd=WEBKIT_ROOT) |
| |
| base_commit = git(["commit-tree", base_tree, "-m", "base"], cwd=work) |
| ours_commit = git(["commit-tree", ours_tree, "-p", base_commit, |
| "-m", "ours"], cwd=work) |
| theirs_for_merge = git(["commit-tree", theirs_tree, "-p", base_commit, |
| "-m", "theirs"], cwd=work) |
| |
| log("Creating merge worktree ...") |
| if os.path.exists(merge): |
| shutil.rmtree(merge) |
| git(["worktree", "add", "--no-checkout", "-f", merge, ours_commit], cwd=work) |
| git(["checkout", "-q", "-f", ours_commit], cwd=merge) |
| |
| log("Merging upstream changes into WebKit's tree ...") |
| rc = gitcode(["merge", "--no-edit", theirs_for_merge], merge) |
| if rc != 0: |
| _pause_for_conflicts(ctx) |
| else: |
| log("Merge applied cleanly.") |
| |
| |
| def _pause_for_conflicts(ctx): |
| merge = ctx["merge"] |
| work = ctx["work"] |
| st = ctx["state"].data |
| conflicts = git(["diff", "--name-only", "--diff-filter=U"], cwd=merge) |
| lines = ["", "Merge conflicts must be resolved manually.", ""] |
| lines.append("Conflicted files (resolve in the merge worktree):") |
| lines.append(" %s" % merge) |
| lines.append("") |
| for f in conflicts.split("\n"): |
| if not f: |
| continue |
| lines.append(" CONFLICT: %s" % f) |
| # Attribution hint: which upstream commits touched this file. |
| upstream_log = git( |
| ["log", "--oneline", "%s..%s" % (st["previous_hash"], st["target_hash"]), |
| "--", f], cwd=work, check=False) |
| for ll in upstream_log.split("\n")[:8]: |
| if ll: |
| lines.append(" upstream: %s" % ll) |
| # Attribution hint: the last couple of WebKit commits that changed this |
| # file, skipping the "Update ANGLE" roll commits so the actual local |
| # patch authors are shown. |
| webkit_log = git( |
| ["log", "--oneline", "--invert-grep", "--grep=Update ANGLE", "-n", "2", |
| "--", os.path.join(ANGLE_REL, f)], cwd=WEBKIT_ROOT, check=False) |
| for ll in webkit_log.split("\n"): |
| if ll: |
| lines.append(" WebKit maybe: %s" % ll) |
| lines += [ |
| "", |
| "In this merge, 'current' (a.k.a. 'ours', the worktree's HEAD) is", |
| "WebKit's tree, and 'incoming' (a.k.a. 'theirs') is upstream ANGLE.", |
| "To take one whole side of a conflicted file:", |
| " git checkout --theirs <file> # incoming (upstream) version", |
| " git checkout --ours <file> # current (WebKit) version", |
| " git add <file>", |
| "", |
| "Guidance: for patches already upstreamed, take the incoming/theirs", |
| "(upstream) version; for WebKit-only patches, keep the current/ours", |
| "(WebKit) change.", |
| "", |
| "After resolving and 'git add'-ing each file in the worktree above,", |
| "re-run this script with --continue.", |
| ] |
| raise Paused("merge", "\n".join(lines)) |
| |
| |
| def _finish_merge(ctx): |
| merge = ctx["merge"] |
| # Verify the user finished the merge. |
| unmerged = git(["diff", "--name-only", "--diff-filter=U"], cwd=merge) |
| if unmerged.strip(): |
| raise Paused("merge", |
| "Still unresolved files in %s:\n %s\nResolve, 'git add', " |
| "then --continue." % (merge, unmerged.replace("\n", "\n "))) |
| if os.path.exists(os.path.join(merge, ".git")) and \ |
| gitcode(["rev-parse", "-q", "--verify", "MERGE_HEAD"], merge) == 0: |
| git(["commit", "--no-edit", "--no-verify"], cwd=merge, capture=False) |
| log("Merge resolved.") |
| |
| |
| def phase_thirdparty(ctx): |
| """Bring the vendored third_party files (VENDORED) from the target sync into |
| the merged tree, 3-way-merging WebKit's local edits. |
| |
| After the main merge, the merge worktree holds WebKit's vendored files |
| (`ours`). For each vendored file in the target sync (`theirs`): |
| * binary/data files are overwritten wholesale (WebKit never edits them); |
| * text files are 3-way merged with `git merge-file ours base theirs`, |
| where `base` is the previous revision captured in phase_sync. This |
| applies upstream's changes onto WebKit's edited file, preserving the |
| local edits and producing conflict markers only where both sides |
| touched the same lines (the run then pauses for manual resolution); |
| * files absent from `ours` (new upstream files) are copied in. |
| Each entry is a glob (a 'dir/**' subtree, a wildcard, or a single file).""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| if ctx["args"].skip_sync: |
| log("Skipping third_party update (--skip-sync).") |
| return |
| |
| base_dir = ctx["state"].data.get("base_vendored_dir") |
| empty = None # lazily-created empty file, used as base for files new in ours |
| overwritten = merged = added = 0 |
| conflicts = [] |
| matched_any = False |
| for rel, theirs in _iter_vendored(work): |
| matched_any = True |
| ours = os.path.join(merge, rel) |
| os.makedirs(os.path.dirname(ours), exist_ok=True) |
| if not os.path.exists(ours): |
| shutil.copy2(theirs, ours) # new upstream file |
| added += 1 |
| continue |
| if not _is_text_file(theirs): |
| shutil.copy2(theirs, ours) # binary/data: take upstream |
| overwritten += 1 |
| continue |
| base = os.path.join(base_dir, rel) if base_dir else None |
| if not base or not os.path.exists(base): |
| if empty is None: |
| fd, empty = tempfile.mkstemp(prefix="angle_empty_base_") |
| os.close(fd) |
| base = empty |
| # git merge-file applies (theirs - base) onto ours, in place. |
| rc = gitcode(["merge-file", "-L", "WebKit", "-L", "base", "-L", "upstream", |
| ours, base, theirs], merge) |
| if rc != 0: |
| conflicts.append(rel) |
| else: |
| merged += 1 |
| if empty: |
| os.remove(empty) |
| if not matched_any: |
| warn("No VENDORED files matched in the sync (manifest stale?).") |
| log("Vendored third_party: %d merged, %d new, %d binary overwritten, " |
| "%d conflict(s)." % (merged, added, overwritten, len(conflicts))) |
| if conflicts: |
| lines = ["", "Vendored third_party files conflicted during the 3-way merge.", |
| "Resolve the conflict markers in the merge worktree, then resume:", |
| " %s" % merge, ""] |
| for c in conflicts: |
| lines.append(" CONFLICT: %s" % c) |
| fail("\n".join(lines)) |
| |
| |
| def _run_webkit_source_generators(angle_dir): |
| """Run the WebKit-specific generators that only need the ANGLE tree: |
| gni-to-cmake (*.cmake) and program_serialize_data_version |
| (WebKit/ANGLEShaderProgramVersion.h). Operates in `angle_dir` in place.""" |
| # gni -> cmake build files. |
| gni = os.path.join(angle_dir, "gni-to-cmake.py") |
| if os.path.exists(gni): |
| conversions = [ |
| ("src/compiler.gni", "Compiler.cmake", None), |
| ("src/libGLESv2.gni", "GLESv2.cmake", None), |
| ("src/libANGLE/renderer/d3d/BUILD.gn", "D3D.cmake", |
| "src/libANGLE/renderer/d3d/"), |
| ("src/libANGLE/renderer/gl/BUILD.gn", "GL.cmake", |
| "src/libANGLE/renderer/gl/"), |
| ("src/libANGLE/renderer/metal/BUILD.gn", "Metal.cmake", |
| "src/libANGLE/renderer/metal/"), |
| ] |
| for src, out, prepend in conversions: |
| if not os.path.exists(os.path.join(angle_dir, src)): |
| continue |
| # Invoke as ./gni-to-cmake.py so the path embedded in the generated |
| # file header matches WebKit's convention (relative, not absolute). |
| cmd = ["./gni-to-cmake.py", src, out] |
| if prepend: |
| cmd += ["--prepend", prepend] |
| run(cmd, cwd=angle_dir, check=False) |
| log("Regenerated *.cmake build files") |
| |
| # ANGLEShaderProgramVersion.h -- hash over all ANGLE sources. |
| psdv = os.path.join(angle_dir, "src", "program_serialize_data_version.py") |
| if os.path.exists(psdv): |
| fd, listing = tempfile.mkstemp(prefix="angle_sources_", suffix=".txt") |
| os.close(fd) |
| with open(listing, "w") as f: |
| for dirpath, _dirs, files in os.walk(os.path.join(angle_dir, "src")): |
| for fn in files: |
| if fn.lower().endswith((".h", ".cpp", ".c", ".cc", |
| ".mm", ".inc")): |
| f.write(os.path.relpath(os.path.join(dirpath, fn), angle_dir) + "\n") |
| run([sys.executable, psdv, "WebKit/ANGLEShaderProgramVersion.h", listing], |
| cwd=angle_dir, check=False) |
| os.remove(listing) |
| log("Regenerated WebKit/ANGLEShaderProgramVersion.h") |
| |
| |
| def phase_codegen_b(ctx): |
| """Regenerate WebKit-specific generated/owned files on the merged tree.""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| |
| # angle_commit.h -- generated against the target commit (work HEAD). |
| commit_id = os.path.join(work, "src", "commit_id.py") |
| if os.path.exists(commit_id): |
| fd, tmp = tempfile.mkstemp(prefix="angle_commit_", suffix=".h") |
| os.close(fd) |
| run([sys.executable, commit_id, "gen", tmp], cwd=work, check=False) |
| if os.path.exists(tmp): |
| dest = os.path.join(merge, "WebKit", "angle_commit.h") |
| os.makedirs(os.path.dirname(dest), exist_ok=True) |
| shutil.move(tmp, dest) |
| log("Regenerated WebKit/angle_commit.h") |
| |
| _run_webkit_source_generators(merge) |
| |
| |
| def phase_bookkeep(ctx): |
| """Update ANGLE.plist and changes.diff, and write the commit message.""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| target_hash = ctx["state"].data["target_hash"] |
| prev_hash = ctx["state"].data["previous_hash"] |
| target_date = git(["show", "-s", "--format=%cs", target_hash], cwd=work) |
| |
| # ANGLE.plist: 40-char hash (appears in OpenSourceVersion and the SCM |
| # checkout command) + Update date (the target commit's date). |
| plist = os.path.join(merge, "ANGLE.plist") |
| if os.path.exists(plist): |
| with open(plist) as f: |
| text = f.read() |
| text = re.sub(r"(?<![a-f0-9])[a-f0-9]{40}(?![a-f0-9])", target_hash, text) |
| text = re.sub(r"<string>\d{4}-\d{2}-\d{2}</string>", |
| "<string>%s</string>" % target_date, text) |
| with open(plist, "w") as f: |
| f.write(text) |
| log("Updated ANGLE.plist") |
| |
| # changes.diff: merged tree vs upstream target, restricted to the merged, |
| # non-excluded paths (the VENDORED + DROP set is excluded). |
| # --ignore-submodules=all suppresses the dropped-submodule gitlinks (the |
| # filter removes all submodules, but the raw target tree still has them). |
| git(["add", "-A"], cwd=merge) |
| merged_tree = git(["write-tree"], cwd=merge) |
| diff = git(["diff", "--ignore-submodules=all", "--full-index", "-b", |
| "--ignore-cr-at-eol", |
| target_hash, merged_tree, "--"] + _CHANGES_DIFF_EXCLUDES, |
| cwd=work, check=False) |
| with open(os.path.join(merge, "changes.diff"), "w") as f: |
| f.write(diff + ("\n" if diff and not diff.endswith("\n") else "")) |
| log("Regenerated changes.diff") |
| |
| # Commit message draft at the WebKit root (copy it out and delete it). |
| upstream_log = git(["log", "--oneline", "%s..%s" % (prev_hash, target_hash), |
| "--pretty=%h %s"], cwd=work, check=False) |
| msg = [ |
| "Update ANGLE to %s (%s)" % (target_date, target_hash), |
| "Need the bug URL (OOPS!).", |
| "Include a Radar link (OOPS!).", |
| "", |
| "Reviewed by NOBODY (OOPS!)", |
| "", |
| "Contains upstream commits:", |
| upstream_log, |
| "", |
| ] |
| with open(COMMIT_MSG_FILE, "w") as f: |
| f.write("\n".join(msg)) |
| log("Wrote %s" % os.path.relpath(COMMIT_MSG_FILE, WEBKIT_ROOT)) |
| |
| |
| def phase_finalize(ctx): |
| """Export the merged tree into WebKit and stage it.""" |
| work = ctx["work"] |
| merge = ctx["merge"] |
| |
| log("Exporting merged tree into %s ..." % ANGLE_REL) |
| |
| # The set of files the merged tree should contain (worktree was `git add |
| # -A`-ed during bookkeep). These are repo-relative to the merge worktree. |
| merged_files = set( |
| f for f in git(["ls-files"], cwd=merge).split("\n") if f) |
| |
| # Copy/update merged files into the real ANGLE dir. Crucially NO --delete: |
| # the real ANGLE dir may contain untracked, gitignored gclient artifacts |
| # (third_party submodule checkouts) that we must never destroy. |
| rsync = shutil.which("rsync") |
| if rsync: |
| res = run([rsync, "-a", "--exclude=.git", "--exclude=.gclient", |
| merge.rstrip("/") + "/", ANGLE_DIR.rstrip("/") + "/"], |
| check=False, capture=True) |
| if res.returncode != 0: |
| fail("rsync failed (exit %d):\n%s" |
| % (res.returncode, (res.stderr or "").strip())) |
| else: |
| for rel in merged_files: |
| src = os.path.join(merge, rel) |
| dst = os.path.join(ANGLE_DIR, rel) |
| os.makedirs(os.path.dirname(dst), exist_ok=True) |
| shutil.copy2(src, dst) |
| |
| # Handle true deletions explicitly: files tracked under ANGLE in WebKit |
| # that are absent from the merged tree. This removes only tracked files, |
| # never untracked gclient content. |
| prefix = ANGLE_REL.replace(os.sep, "/") + "/" |
| tracked = git(["ls-files", ANGLE_REL], cwd=WEBKIT_ROOT).split("\n") |
| deletions = [] |
| for path in tracked: |
| if not path: |
| continue |
| rel = path[len(prefix):] if path.startswith(prefix) else path |
| if rel not in merged_files: |
| deletions.append(path) |
| if deletions: |
| log("Removing %d file(s) deleted upstream." % len(deletions)) |
| git(["rm", "-q", "--ignore-unmatch", "--"] + deletions, |
| cwd=WEBKIT_ROOT, check=False, capture=False) |
| |
| # A vendored third_party subtree may sit inside a gclient-cloned git repo |
| # (e.g. third_party/VK-GL-CTS/src/.git from a developer's local sync). An |
| # embedded .git makes `git add` record a gitlink instead of the vendored |
| # files, so strip any .git at or above each vendored path (within ANGLE_DIR) |
| # before staging. The non-vendored remainder of the synced tree stays on |
| # disk as plain (gitignored) files; a later `gclient sync` re-clones it. |
| angle_root = os.path.normpath(ANGLE_DIR) |
| stripped = set() |
| for pat in VENDORED: |
| subdir = pat[:-len("/**")] if pat.endswith("/**") else pat |
| cur = os.path.normpath(os.path.join(ANGLE_DIR, subdir)) |
| while cur != angle_root and cur.startswith(angle_root + os.sep): |
| dotgit = os.path.join(cur, ".git") |
| if cur not in stripped and os.path.lexists(dotgit): |
| if os.path.isdir(dotgit) and not os.path.islink(dotgit): |
| shutil.rmtree(dotgit, ignore_errors=True) |
| else: |
| os.remove(dotgit) |
| stripped.add(cur) |
| cur = os.path.dirname(cur) |
| if stripped: |
| log("Stripped embedded .git from %d vendored subtree root(s)." |
| % len(stripped)) |
| |
| # Stage everything (gitignored gclient artifacts are skipped by git). |
| # The commit message draft lives at the WebKit root, outside ANGLE_REL, so |
| # it is never staged. |
| git(["add", "-A", ANGLE_REL], cwd=WEBKIT_ROOT) |
| |
| # Tear down the temp work area, unless the user gave an explicit --work-dir |
| # (then leave it in place for inspection / reuse). |
| if ctx["state"].data.get("keep_work_dir"): |
| log("Work dir kept at %s (user-specified --work-dir)." % ctx["work_root"]) |
| else: |
| gitcode(["worktree", "remove", "--force", merge], work) |
| shutil.rmtree(work, ignore_errors=True) |
| |
| log("") |
| log("ANGLE update is staged in %s." % ANGLE_REL) |
| log("Remaining MANUAL steps:") |
| log(" 1. Mirror the *.cmake changes into ANGLE.xcodeproj and make it build.") |
| log(" Review: git -C %s diff --staged -- %s/Compiler.cmake %s/GLESv2.cmake" |
| % (WEBKIT_ROOT, ANGLE_REL, ANGLE_REL)) |
| log(" 2. make debug SCHEME='Tools (ANGLE)' && Source/ThirdParty/ANGLE/WebKit/run-angle-tests") |
| log(" 3. make debug && run-webkit-tests fast/canvas webgl") |
| log(" 4. git commit -F %s, fill in the" |
| % os.path.relpath(COMMIT_MSG_FILE, WEBKIT_ROOT)) |
| log(" bug/Radar/reviewer, then delete the file before committing.") |
| log(" 5. update-angle --skip-sync --skip-codegen --generate-only") |
| |
| |
| PHASE_FUNCS = { |
| "prepare": phase_prepare, |
| "sync": phase_sync, |
| "merge": phase_merge, |
| "thirdparty": phase_thirdparty, |
| "codegen_a": phase_codegen_a, |
| "codegen_b": phase_codegen_b, |
| "bookkeep": phase_bookkeep, |
| "finalize": phase_finalize, |
| } |
| |
| |
| def require_gclient(): |
| """Ensure gclient (depot_tools) is on PATH; the sync/codegen phases need it. |
| Skipped by callers only when both --skip-sync and --skip-codegen are set.""" |
| if shutil.which("gclient") is None: |
| fail("gclient not found on PATH. Add depot_tools to your PATH " |
| "(e.g. export PATH=\"$PATH:$HOME/depot_tools\"), or re-run with " |
| "--skip-sync.") |
| |
| |
| def check_clean_worktree(): |
| # Allow uncommitted edits to this script itself, so it can be developed and |
| # tested without committing. |
| script_rel = os.path.relpath(os.path.realpath(__file__), WEBKIT_ROOT) |
| status = git(["status", "--porcelain"], cwd=WEBKIT_ROOT, strip=False) |
| dirty = [] |
| for l in status.split("\n"): |
| if not l: |
| continue |
| path = l[3:] |
| if path == script_rel: |
| continue |
| # Dirty = tracked changes anywhere, or untracked files inside ANGLE. |
| if not l.startswith("??") or path.startswith(ANGLE_REL): |
| dirty.append(l) |
| if dirty: |
| fail("WebKit has uncommitted/untracked changes. Commit, stash, or " |
| "remove them first:\n" + "\n".join(dirty)) |
| |
| |
| def run_from(ctx, start_phase): |
| start = PHASES.index(start_phase) |
| for phase in PHASES[start:]: |
| ctx["state"].set_phase(phase) |
| log("Phase: %s" % phase) |
| try: |
| PHASE_FUNCS[phase](ctx) |
| except Paused as p: |
| ctx["state"].set_phase(p.phase) |
| print(p.instructions) |
| sys.exit(0) |
| # Completed: remove state file. |
| ctx["state"].remove() |
| |
| |
| def _make_ctx(args, state, work_root): |
| return { |
| "args": args, |
| "state": state, |
| "work": os.path.join(work_root, "upstream"), |
| "merge": os.path.join(work_root, "merge"), |
| "work_root": work_root, |
| "target": args.commit, |
| "upstream_url": args.upstream_url, |
| } |
| |
| |
| DEFAULT_WORK_ROOT = os.path.join(tempfile.gettempdir(), "webkit-update-angle") |
| |
| |
| def main(): |
| global VERBOSE, QUIET |
| parser = argparse.ArgumentParser( |
| description="Update upstream ANGLE into WebKit's Source/ThirdParty/ANGLE.") |
| parser.add_argument("commit", nargs="?", default="upstream/main", |
| help="Target upstream commit (default: upstream/main).") |
| parser.add_argument("-v", "--verbose", action="store_true") |
| parser.add_argument("-q", "--quiet", action="store_true", |
| help="Only print errors, warnings, and the " |
| "conflict-resolution note.") |
| parser.add_argument("--continue", dest="cont", action="store_true", |
| help="Resume an in-progress update (after a pause).") |
| parser.add_argument("--abort", action="store_true", |
| help="Discard the in-progress update and clean up.") |
| parser.add_argument("--generate-only", dest="generate_only", |
| action="store_true", |
| help="Re-run the update against the revision the tree " |
| "already has (from ANGLE.plist), to regenerate code " |
| "and changes.diff via the normal work-dir pipeline.") |
| parser.add_argument("--skip-sync", action="store_true", |
| help="Skip gclient sync and third_party update.") |
| parser.add_argument("--skip-codegen", action="store_true", |
| help="Skip upstream run_code_generation.py (Tier A).") |
| parser.add_argument("--work-dir", default=None, |
| help="Work directory (default: %s). Recorded in the " |
| "state file so --continue need not repeat it." |
| % DEFAULT_WORK_ROOT) |
| parser.add_argument("--upstream-url", default=UPSTREAM_URL, |
| help="Upstream ANGLE git URL (default: %s)." % UPSTREAM_URL) |
| args = parser.parse_args() |
| VERBOSE = args.verbose |
| QUIET = args.quiet |
| |
| state = State() |
| |
| if args.generate_only: |
| # Regenerate code + changes.diff by running the normal work-dir pipeline |
| # against the revision the tree already has (from ANGLE.plist) -- i.e. an |
| # update to the current revision -- rather than generating in place. |
| with open(os.path.join(ANGLE_DIR, "ANGLE.plist")) as f: |
| m = re.search(r"[a-f0-9]{40}", f.read()) |
| if not m: |
| fail("Could not find current ANGLE revision in ANGLE.plist") |
| args.commit = m.group(0) |
| # Falls through to the fresh-run pipeline below (target = current rev). |
| |
| if args.abort: |
| data = state.load() if os.path.exists(state.path) else {} |
| work_root = data.get("work_dir") or args.work_dir or DEFAULT_WORK_ROOT |
| ctx = _make_ctx(args, state, work_root) |
| keep = data.get("keep_work_dir") or (args.work_dir is not None) |
| if os.path.exists(ctx["work"]): |
| gitcode(["worktree", "remove", "--force", ctx["merge"]], ctx["work"]) |
| state.remove() |
| if keep: |
| log("Aborted; state file removed. Work dir kept at %s." % work_root) |
| else: |
| shutil.rmtree(work_root, ignore_errors=True) |
| log("Aborted; work area %s and state file removed." % work_root) |
| return |
| |
| if args.cont: |
| data = state.load() |
| if not data: |
| fail("No in-progress update found (no %s)." % state.path) |
| work_root = data.get("work_dir") or DEFAULT_WORK_ROOT |
| ctx = _make_ctx(args, state, work_root) |
| ctx["target"] = data.get("target_hash", ctx["target"]) |
| # Honor the flags / locations from the original invocation. |
| args.skip_sync = data.get("skip_sync", args.skip_sync) |
| args.skip_codegen = data.get("skip_codegen", args.skip_codegen) |
| ctx["upstream_url"] = data.get("upstream_url", ctx["upstream_url"]) |
| if not (args.skip_sync and args.skip_codegen): |
| require_gclient() |
| run_from(ctx, data.get("phase", "prepare")) |
| return |
| |
| # Fresh run. |
| if os.path.exists(state.path): |
| fail("An update is already in progress (%s). Use --continue or --abort." |
| % state.path) |
| if not (args.skip_sync and args.skip_codegen): |
| require_gclient() |
| check_clean_worktree() |
| work_root = args.work_dir or DEFAULT_WORK_ROOT |
| ctx = _make_ctx(args, state, work_root) |
| shutil.rmtree(work_root, ignore_errors=True) |
| os.makedirs(work_root, exist_ok=True) |
| # Persist work dir + flags so --continue behaves consistently. |
| state.data["work_dir"] = work_root |
| # A user-specified --work-dir is left in place after success/--abort; only |
| # the default temp work dir is auto-removed. |
| state.data["keep_work_dir"] = args.work_dir is not None |
| state.data["skip_sync"] = args.skip_sync |
| state.data["skip_codegen"] = args.skip_codegen |
| state.data["upstream_url"] = ctx["upstream_url"] |
| state.save() |
| run_from(ctx, "prepare") |
| |
| |
| if __name__ == "__main__": |
| main() |
