crate_ebuild_help.py

#!/usr/bin/env python3
# Copyright 2022 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Help creating a Rust ebuild with CRATES.

This script is meant to help someone creating a Rust ebuild of the type
currently used by sys-apps/ripgrep and sys-apps/rust-analyzer.

In these ebuilds, the CRATES variable is used to list all dependencies, rather
than creating an ebuild for each dependency. This style of ebuild can be used
for a crate which is only intended for use in the chromiumos SDK, and which has
many dependencies which otherwise won't be used.

To create such an ebuild, there are essentially two tasks that must be done:

1. Determine all transitive dependent crates and version and list them in the
CRATES variable. Ignore crates that are already included in the main crate's
repository.

2. Find which dependent crates are not already on a chromeos mirror, retrieve
them from crates.io, and upload them to `gs://chromeos-localmirror/distfiles`.

This script parses the crate's lockfile to list transitive dependent crates,
and either lists crates to be uploaded or actually uploads them.

Of course these can be done manually instead. If you choose to do these steps
manually, I recommend *not* using the `cargo download` tool, and instead obtain
dependent crates at
`https://crates.io/api/v1/crates/{crate_name}/{crate_version}/download`.

Example usage:

    # Here we instruct the script to ignore crateA and crateB, presumably
    # because they are already included in the same repository as some-crate.
    # This will not actually upload any crates to `gs`.
    python3 crate_ebuild_help.py --lockfile some-crate/Cargo.lock \
            --ignore crateA --ignore crateB --dry-run

    # Similar to the above, but here we'll actually carry out the uploads.
    python3 crate_ebuild_help.py --lockfile some-crate/Cargo.lock \
            --ignore crateA --ignore crateB

See the ebuild files for ripgrep or rust-analyzer for other details.
"""

import argparse
import concurrent.futures
from pathlib import Path
import subprocess
import tempfile
from typing import List, Tuple
import urllib.request

# Python 3.11 has `tomllib`, so maybe eventually we can switch to that.
import toml


def run(args: List[str]) -> bool:
    result = subprocess.run(
        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False
    )
    return result.returncode == 0


def run_check(args: List[str]):
    subprocess.run(
        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
    )


def gs_address_exists(address: str) -> bool:
    # returns False if the file isn't there
    return run(["gsutil.py", "ls", address])


def crate_already_uploaded(crate_name: str, crate_version: str) -> bool:
    filename = f"{crate_name}-{crate_version}.crate"
    return gs_address_exists(
        f"gs://chromeos-localmirror/distfiles/{filename}"
    ) or gs_address_exists(f"gs://chromeos-mirror/gentoo/distfiles/{filename}")


def download_crate(crate_name: str, crate_version: str, localpath: Path):
    urllib.request.urlretrieve(
        f"https://crates.io/api/v1/crates/{crate_name}/{crate_version}/download",
        localpath,
    )


def upload_crate(crate_name: str, crate_version: str, localpath: Path):
    run_check(
        [
            "gsutil.py",
            "cp",
            "-n",
            "-a",
            "public-read",
            str(localpath),
            f"gs://chromeos-localmirror/distfiles/{crate_name}-{crate_version}.crate",
        ]
    )


def main():
    parser = argparse.ArgumentParser(
        description="Help prepare a Rust crate for an ebuild."
    )
    parser.add_argument(
        "--lockfile",
        type=str,
        required=True,
        help="Path to the lockfile of the crate in question.",
    )
    parser.add_argument(
        "--ignore",
        type=str,
        action="append",
        required=False,
        default=[],
        help="Ignore the crate by this name (may be used multiple times).",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Don't actually download/upload crates, just print their names.",
    )
    ns = parser.parse_args()

    to_ignore = set(ns.ignore)

    toml_contents = toml.load(ns.lockfile)
    packages = toml_contents["package"]

    crates = [
        (pkg["name"], pkg["version"])
        for pkg in packages
        if pkg["name"] not in to_ignore
    ]
    crates.sort()

    print("Dependent crates:")
    for name, version in crates:
        print(f"{name}-{version}")
    print()

    if ns.dry_run:
        print("Crates that would be uploaded (skipping ones already uploaded):")
    else:
        print("Uploading crates (skipping ones already uploaded):")

    def maybe_upload(crate: Tuple[str, str]) -> str:
        name, version = crate
        if crate_already_uploaded(name, version):
            return ""
        if not ns.dry_run:
            with tempfile.TemporaryDirectory() as temp_dir:
                path = Path(temp_dir.name, f"{name}-{version}.crate")
                download_crate(name, version, path)
                upload_crate(name, version, path)
        return f"{name}-{version}"

    # Simple benchmarking on my machine with rust-analyzer's Cargo.lock, using
    # the --dry-run option, gives a wall time of 277 seconds with max_workers=1
    # and 70 seconds with max_workers=4.
    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
        crates_len = len(crates)
        for i, s in enumerate(executor.map(maybe_upload, crates)):
            if s:
                j = i + 1
                print(f"[{j}/{crates_len}] {s}")
    print()


if __name__ == "__main__":
    main()