| #!/usr/bin/env python3 |
| |
| from __future__ import print_function |
| |
| import argparse |
| import generate_amalgamation |
| import hashlib |
| import os |
| import string |
| import subprocess |
| import sys |
| |
| |
| class UnstagedFiles(Exception): |
| pass |
| |
| |
| class UnknownHash(Exception): |
| pass |
| |
| |
| class IncorrectType(Exception): |
| pass |
| |
| |
| class bcolors: |
| HEADER = '\033[95m' |
| OKBLUE = '\033[94m' |
| OKGREEN = '\033[92m' |
| WARNING = '\033[93m' |
| FAIL = '\033[91m' |
| ENDC = '\033[0m' |
| |
| |
| def _print_command(cmd): |
| """Print the command to be executed to the console. |
| |
| Use a different color so that it can be easily seen amongst the output |
| commands. |
| """ |
| if (isinstance(cmd, list)): |
| cmd = ' '.join(cmd) |
| print('{}{}{}'.format(bcolors.OKBLUE, cmd, bcolors.ENDC)) |
| |
| |
| class ManifestEntry(object): |
| """Represents a single entry in a SQLite manifest.""" |
| |
| def __init__(self, entry_type, items): |
| if not len(entry_type) == 1: |
| raise IncorrectType(entry_type) |
| self.entry_type = entry_type |
| self.items = items |
| |
| def get_hash_type(self): |
| """Return the type of hash used for this entry.""" |
| last_item = self.items[-1] |
| if not all(c in string.hexdigits for c in last_item): |
| print( |
| '"{}" doesn\'t appear to be a hash.'.format(last_item), |
| file=sys.stderr) |
| raise UnknownHash() |
| elif len(last_item) == 40: |
| return 'sha1' |
| elif len(last_item) == 64: |
| return 'sha3' |
| else: |
| raise UnknownHash('Incorrect length {} for {}'.format( |
| len(last_item), last_item)) |
| |
| @staticmethod |
| def calc_hash(data, method): |
| """Return the string sha1 or sha3 hash digest for the given data.""" |
| if method == 'sha3': |
| h = hashlib.sha3_256() |
| elif method == 'sha1': |
| h = hashlib.sha1() |
| else: |
| assert False |
| h.update(data) |
| return h.hexdigest() |
| |
| @staticmethod |
| def calc_file_hash(fname, method): |
| """Return the string sha1 or sha3 hash digest for the given file.""" |
| with open(fname, 'rb') as input_file: |
| return ManifestEntry.calc_hash(input_file.read(), method) |
| |
| def update_file_hash(self): |
| """Calculates a new file hash for this entry.""" |
| self.items[1] = ManifestEntry.calc_file_hash(self.items[0], |
| self.get_hash_type()) |
| |
| def __str__(self): |
| return '{} {}'.format(self.entry_type, ' '.join(self.items)) |
| |
| |
| class Manifest(object): |
| """A deserialized SQLite manifest.""" |
| |
| def __init__(self): |
| self.entries = [] |
| |
| def find_file_entry(self, fname): |
| """Given a file path return the entry. Returns None if none found.""" |
| for entry in self.entries: |
| if entry.entry_type == 'F' and entry.items[0] == fname: |
| return entry |
| return None |
| |
| |
| class ManifestSerializer(object): |
| """De/serialize SQLite manifests.""" |
| |
| @staticmethod |
| def read_stream(input_stream): |
| """Deserialize a manifest from an input stream and return a Manifest |
| object.""" |
| _manifest = Manifest() |
| for line in input_stream.readlines(): |
| items = line.split() |
| if not items: |
| continue |
| _manifest.entries.append(ManifestEntry(items[0], items[1:])) |
| return _manifest |
| |
| @staticmethod |
| def read_file(fname): |
| """Deserialize a manifest file and return a Manifest object.""" |
| with open(fname) as input_stream: |
| return ManifestSerializer.read_stream(input_stream) |
| |
| @staticmethod |
| def write_stream(manifest, output_stream): |
| """Serialize the given manifest to the given stream.""" |
| for entry in manifest.entries: |
| print(str(entry), file=output_stream) |
| |
| @staticmethod |
| def write_file(manifest, fname): |
| """Serialize the given manifest to the specified file.""" |
| with open(fname, 'w') as output_stream: |
| ManifestSerializer.write_stream(manifest, output_stream) |
| |
| |
| class Git(object): |
| @staticmethod |
| def _get_status(): |
| changes = [] |
| for line in subprocess.check_output(['git', 'status', |
| '--porcelain']).splitlines(): |
| changes.append(line.decode('utf-8')) |
| return changes |
| |
| @staticmethod |
| def get_staged_changes(): |
| changes = [] |
| for line in Git._get_status(): |
| entry = line[0:2] |
| if entry == 'M ': |
| changes.append(line.split()[1]) |
| return changes |
| |
| @staticmethod |
| def get_unstaged_changes(): |
| changes = [] |
| for line in Git._get_status(): |
| entry = line[0:2] |
| if entry == ' M': |
| changes.append(line.split()[1]) |
| return changes |
| |
| @staticmethod |
| def get_unmerged_changes(): |
| changes = [] |
| for line in Git._get_status(): |
| entry = line[0:2] |
| if entry == 'UU': |
| changes.append(line.split()[1]) |
| return changes |
| |
| |
| class CherryPicker(object): |
| """Class to cherry pick commits in a SQLite Git repository.""" |
| |
| # The binary file extenions for files committed to the SQLite repository. |
| # This is used as a simple way of detecting files that cannot (simply) be |
| # resolved in a merge conflict. This script will automatically ignore |
| # all conflicted files with any of these extensions. If, in the future, new |
| # binary types are added then a conflict will arise during cherry-pick and |
| # the user will need to resolve it. |
| binary_extensions = ( |
| '.data', |
| '.db', |
| '.ico', |
| '.jpg', |
| '.png', |
| ) |
| |
| def __init__(self): |
| self._print_cmds = True |
| self._update_amangamation = True |
| |
| def _take_head_version(self, file_path): |
| subprocess.call( |
| 'git show HEAD:{} > {}'.format(file_path, file_path), shell=True) |
| subprocess.call('git add {}'.format(file_path), shell=True) |
| |
| @staticmethod |
| def _is_binary_file(file_path): |
| _, file_extension = os.path.splitext(file_path) |
| return file_extension in CherryPicker.binary_extensions |
| |
| @staticmethod |
| def _append_cherry_pick_comments(comments): |
| # TODO(cmumford): Figure out how to append comments on cherry picks |
| pass |
| |
| def _cherry_pick_git_commit(self, commit_id): |
| """Cherry-pick a given Git commit into the current branch.""" |
| cmd = ['git', 'cherry-pick', '-x', commit_id] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| returncode = subprocess.call(cmd) |
| # The manifest and manifest.uuid contain Fossil hashes. Restore to |
| # HEAD version and update only when all conflicts have been resolved. |
| comments = None |
| self._take_head_version('manifest') |
| self._take_head_version('manifest.uuid') |
| for unmerged_file in Git.get_unmerged_changes(): |
| if CherryPicker._is_binary_file(unmerged_file): |
| print('{} is a binary file, keeping branch version.'.format( |
| unmerged_file)) |
| self._take_head_version(unmerged_file) |
| if not comments: |
| comments = [ |
| 'Cherry-pick notes', '==============================' |
| ] |
| comments.append( |
| '{} is binary file (with conflict). Keeping branch version' |
| .format(unmerged_file)) |
| if comments: |
| CherryPicker._append_cherry_pick_comments(comments) |
| self.continue_cherry_pick() |
| |
| @staticmethod |
| def _is_git_commit_id(commit_id): |
| return len(commit_id) == 40 |
| |
| def _find_git_commit_id(self, fossil_commit_id): |
| cmd = [ |
| 'git', '--no-pager', 'log', '--color=never', '--all', |
| '--pretty=format:%H', '--grep={}'.format(fossil_commit_id), |
| 'origin/master' |
| ] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| for line in subprocess.check_output(cmd).splitlines(): |
| return line.decode('utf-8') |
| # Not found. |
| assert False |
| |
| def cherry_pick(self, commit_id): |
| """Cherry-pick a given commit into the current branch. |
| |
| Can cherry-pick a given Git or a Fossil commit. |
| """ |
| if not CherryPicker._is_git_commit_id(commit_id): |
| commit_id = self._find_git_commit_id(commit_id) |
| self._cherry_pick_git_commit(commit_id) |
| |
| def _generate_amalgamation(self): |
| for config_name in ['chromium', 'dev']: |
| generate_amalgamation.make_aggregate(config_name) |
| generate_amalgamation.extract_sqlite_api(config_name) |
| |
| def _add_amalgamation(self): |
| os.chdir(generate_amalgamation._SQLITE_SRC_DIR) |
| for config_name in ['chromium', 'dev']: |
| cmd = [ |
| 'git', 'add', |
| generate_amalgamation.get_amalgamation_dir(config_name) |
| ] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| subprocess.check_call(cmd) |
| |
| def _update_manifests(self): |
| """Update the SQLite's Fossil manifest files. |
| |
| This isn't strictly necessary as the manifest isn't used during |
| any build, and manifest.uuid is the Fossil commit ID (which |
| has no meaning in a Git repo). However, keeping these updated |
| helps make it more obvious that a commit originated in |
| Git and not Fossil. |
| """ |
| manifest = ManifestSerializer.read_file('manifest') |
| files_not_in_manifest = ('manifest', 'manifest.uuid') |
| for fname in Git.get_staged_changes(): |
| if fname in files_not_in_manifest: |
| continue |
| entry = manifest.find_file_entry(fname) |
| if not entry: |
| print( |
| 'Cannot find manifest entry for "{}"'.format(fname), |
| file=sys.stderr) |
| sys.exit(1) |
| manifest.find_file_entry(fname).update_file_hash() |
| ManifestSerializer.write_file(manifest, 'manifest') |
| cmd = ['git', 'add', 'manifest'] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| subprocess.check_call(cmd) |
| # manifest.uuid contains the hash from the Fossil repository which |
| # doesn't make sense in a Git branch. Just write all zeros. |
| with open('manifest.uuid', 'w') as output_file: |
| print('0' * 64, file=output_file) |
| cmd = ['git', 'add', 'manifest.uuid'] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| subprocess.check_call(cmd) |
| |
| def continue_cherry_pick(self): |
| if Git.get_unstaged_changes() or Git.get_unmerged_changes(): |
| raise UnstagedFiles() |
| self._update_manifests() |
| if self._update_amangamation: |
| self._generate_amalgamation() |
| self._add_amalgamation() |
| cmd = ['git', 'cherry-pick', '--continue'] |
| if self._print_cmds: |
| _print_command(' '.join(cmd)) |
| subprocess.check_call(cmd) |
| |
| |
| if __name__ == '__main__': |
| desc = 'A script for cherry-picking commits from the SQLite repo.' |
| parser = argparse.ArgumentParser(description=desc) |
| parser.add_argument( |
| 'commit', nargs='*', help='The commit ids to cherry pick (in order)') |
| parser.add_argument( |
| '--continue', |
| dest='cont', |
| action='store_true', |
| help='Continue the cherry-pick once conflicts have been resolved') |
| namespace = parser.parse_args() |
| cherry_picker = CherryPicker() |
| if namespace.cont: |
| try: |
| cherry_picker.continue_cherry_pick() |
| sys.exit(0) |
| except UnstagedFiles: |
| print( |
| 'There are still unstaged files to resolve before continuing.') |
| sys.exit(1) |
| num_picked = 0 |
| for commit_id in namespace.commit: |
| try: |
| cherry_picker.cherry_pick(commit_id) |
| num_picked += 1 |
| except UnstagedFiles: |
| print( |
| '\nThis cherry-pick contains conflicts. Please resolve them ') |
| print('(e.g git mergetool) and rerun this script ' |
| '`sqlite_cherry_picker.py --continue`') |
| print('or `git cherry-pick --abort`.') |
| if commit_id != namespace.commit[-1]: |
| msg = ( |
| 'NOTE: You have only successfully cherry-picked {} out of ' |
| '{} commits.') |
| print(msg.format(num_picked, len(namespace.commit))) |
| sys.exit(1) |
