tools/crates/lib/cargo.py

# python3
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Provide abstractions and helpers for the Rust Cargo build tool.

In addition to data types representing Cargo concepts, this module has helpers
for parsing and generating Cargo.toml files, for running the Cargo tool, and for
parsing its output."""

from __future__ import annotations

from enum import Enum
import os
import subprocess
import sys
import toml
import typing
from typing import Any, Optional

from lib import common
from lib import consts


class CrateKey:
    """A unique identifier for any given third-party crate.

    This is a combination of the crate's name and its epoch, since we have at
    most one crate for a given epoch.

    The name and version/epoch are directly from the crate and are not
    normalized.
    """

    def __init__(self, name: str, version: str):
        self.name = name
        self.epoch = common.version_epoch_dots(version)

    def __repr__(self) -> str:
        return "CrateKey({} v{})".format(self.name, self.epoch)

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, CrateKey):
            return NotImplemented
        return self.name == other.name and self.epoch == other.epoch

    def __hash__(self) -> int:
        return hash("{}: {}".format(self.name, self.epoch))


class CrateUsage(Enum):
    """The ways that a crate's library can be used from other crates."""
    FOR_NORMAL = 1,  # Used from another crate's lib/binary outputs.
    FOR_BUILDRS = 2,  # Used from another crates's build.rs.
    FOR_TESTS = 3,  # Used from another crate's tests.

    def gn_target_name(self) -> str:
        """The name to use for a gn target.

        This is the name of the target used for generating the target in the
        BUILD.gn file. The name is based on how the target will be used, since
        crates have different features enabled when being built for use in
        tests, or for use from a build.rs build script."""
        if self == CrateUsage.FOR_NORMAL:
            return CrateBuildOutput.NORMAL.gn_target_name_for_dep()
        elif self == CrateUsage.FOR_BUILDRS:
            return CrateBuildOutput.BUILDRS.gn_target_name_for_dep()
        elif self == CrateUsage.FOR_TESTS:
            return CrateBuildOutput.TESTS.gn_target_name_for_dep()
        else:
            return NotImplemented


class CrateBuildOutput(Enum):
    """The various build outputs when building a crate."""
    NORMAL = 1  # Building the crate's normal output.
    BUILDRS = 2  # Building the crate's build.rs.
    TESTS = 3  # Building the crate's tests.

    def as_dep_usage(self) -> CrateUsage:
        if self == CrateBuildOutput.NORMAL:
            return CrateUsage.FOR_NORMAL
        elif self == CrateBuildOutput.BUILDRS:
            return CrateUsage.FOR_BUILDRS
        elif self == CrateBuildOutput.TESTS:
            return CrateUsage.FOR_TESTS
        else:
            assert False  # Unhandled CrateBuildOutput?

    def gn_target_name_for_dep(self):
        """The name to use for gn dependency targets.

        This is the name of the target to use for a dependency in the `deps`,
        `build_deps`, or `dev_deps` section of a BUILD.gn target. The name
        depends on what kind of dependency it is, since crates have different
        features enabled when being built for use in tests, or for use from a
        build.rs build script."""
        if self == CrateBuildOutput.NORMAL:
            return "lib"
        if self == CrateBuildOutput.BUILDRS:
            return "buildrs_support"
        if self == CrateBuildOutput.TESTS:
            return "test_support"

    def _cargo_tree_edges(self) -> str:
        """Get the argument for `cargo tree --edges`

        Returns what to pass to the --edges argument when running `cargo tree`
        to see the dependencies of a given build output."""
        if self == CrateBuildOutput.NORMAL:
            return "normal"
        elif self == CrateBuildOutput.BUILDRS:
            return "build"
        elif self == CrateBuildOutput.TESTS:
            return "dev"
        else:
            return NotImplemented


def run_cargo_tree(path: str, build: CrateBuildOutput,
                   target_arch: Optional[str], depth: Optional[int],
                   features: list) -> list[str]:
    """Runs `cargo tree` on the Cargo.toml file at `path`.

    Note that `cargo tree` actually invokes `rustc` a bunch to collect its
    output, but it does not appear to actually compile anything. Additionally,
    we are running `cargo tree` in a temp directory with placeholder rust files
    present to satisfy `cargo tree`, so no source code from crates.io should
    be compiled, or run, by this tool.

    Args:
        target_arch: one of the ALL_RUSTC_ARCH which are targets understood by
          rustc, and shown by `rustc --print target-list`. Or none, in which
          case the current machine's architecture is used.

    Returns:
        The output of cargo tree, with split by lines into a list.
    """
    tree_cmd = [
        "cargo",
        "tree",
        "--manifest-path",
        path,
        "--edges",
        build._cargo_tree_edges(),
        "--format={p} {f}",
        "-v",
    ]
    if target_arch:
        tree_cmd += ["--target", target_arch]
    if depth is not None:
        tree_cmd += ["--depth", str(depth)]
    if "default" not in features:
        tree_cmd += ["--no-default-features"]
    features = [f for f in features if not f == "default"]
    if features:
        tree_cmd += ["--features", ",".join(features)]
    try:
        r = subprocess.check_output(tree_cmd, text=True, stderr=subprocess.PIPE)
    except subprocess.CalledProcessError as e:
        print()
        print(' '.join(tree_cmd))
        print(e.stderr)
        raise e
    return r.splitlines()


def add_required_cargo_fields(toml_3p):
    """Add required fields for a Cargo.toml to be parsed by `cargo tree`."""
    toml_3p["package"] = {
        "name": "chromium",
        "version": "1.0.0",
    }
    return toml_3p


class ListOf3pCargoToml:
    """A typesafe cache of info about local third-party Cargo.toml files."""

    class CargoToml:
        def __init__(self, name: str, epoch: str, path: str):
            self.name = name
            self.epoch = epoch
            self.path = path

    def __init__(self, list_of: list[CargoToml]):
        self._list_of = list_of


def write_cargo_toml_in_tempdir(
        dir: str,
        all_3p_tomls: ListOf3pCargoToml,
        orig_toml_parsed: Optional[dict[str, Any]] = None,
        orig_toml_path: Optional[str] = None,
        verbose: bool = False) -> str:
    """Write a temporary Cargo.toml file that will work with `cargo tree`.

    Creates a copy of a Cargo.toml, specified in `orig_toml_path`, in to the
    temp directory specified by `dir` and sets up the temp dir so that running
    `cargo` will succeed. Also points all crates named in `all_3p_tomls` to
    the downloaded versions.

    Exactly one of `orig_toml_parsed` or `orig_toml_path` must be specified.

    Args:
        dir: An OS path to a temp directory where the Cargo.toml file is to be
          written.
        all_3p_tomls: A cache of local third-party Cargo.toml files, crated by
          gen_list_of_3p_cargo_toml(). The generated Cargo.toml will be patched
          to point `cargo tree` to local Cargo.tomls for dependencies in order
          to see local changes.
        orig_toml_parsed: The Cargo.toml file contents to write, as a
          dictionary.
        orig_toml_path: An OS path to the Cargo.toml file which should be copied
          into the output Cargo.toml.
        verbose: Whether to print verbose output, including the full TOML
          content.

    Returns:
        The OS path to the output Cargo.toml file in `dir`, for convenience.
    """
    assert bool(orig_toml_parsed) ^ bool(orig_toml_path)
    orig_toml_text: Optional[str] = None
    if orig_toml_path:
        with open(orig_toml_path, "r") as f:
            orig_toml_text = f.read()
        orig_toml_parsed = dict(toml.loads(orig_toml_text))

    # This assertion is necessary for type checking. Now mypy deduces
    # orig_toml_parsed's type as dict[str, Any] instead of Optional[...]
    assert orig_toml_parsed is not None

    orig_name = orig_toml_parsed["package"]["name"]
    orig_epoch = common.version_epoch_dots(
        orig_toml_parsed["package"]["version"])

    if all_3p_tomls is None:
        all_3p_tomls = ListOf3pCargoToml([])

    # Since we're putting a Cargo.toml in a temp dir, cargo won't be
    # able to find the src/lib.rs and will bail out, so we make it.
    os.mkdir(os.path.join(dir, "src"))
    with open(os.path.join(dir, "src", "lib.rs"), mode="w") as f:
        f.write("lib.rs")
    # Same thing for build.rs, as some Cargo.toml flags make it go looking
    # for a build script to verify it exists.
    if not "build" in orig_toml_parsed["package"]:
        with open(os.path.join(dir, "build.rs"), mode="w") as f:
            f.write("build.rs")
    # And [[bin]] targets, if they have a name but no path, expect to
    # find a file at src/bin/%name%.rs or at src/main.rs, though when
    # one is preferred is unclear. It seems to always work with the
    # former one though, but not always with the latter.
    if "bin" in orig_toml_parsed:
        os.mkdir(os.path.join(dir, "src", "bin"))
        for bin in orig_toml_parsed["bin"]:
            if "path" not in bin and "name" in bin:
                with open(os.path.join(dir, "src", "bin",
                                       "{}.rs".format(bin["name"])),
                          mode="w") as f:
                    f.write("bin main.rs")
    # Workspaces in a crate's Cargo.toml need to point to other Cargo.toml files
    # on disk, and those Cargo.toml files require a lib or binary source as
    # well. We don't support building workspaces, but cargo will die if it can't
    # find them.
    if "workspace" in orig_toml_parsed:
        for m in orig_toml_parsed["workspace"].get("members", []):
            workspace_dir = os.path.join(dir, *(m.split("/")))
            os.makedirs(workspace_dir)
            with open(os.path.join(workspace_dir, "Cargo.toml"), mode="w") as f:
                f.write(consts.FAKE_EMPTY_CARGO_TOML)
            bin_dir = os.path.join(workspace_dir, "src", "bin")
            os.makedirs(bin_dir)
            with open(os.path.join(bin_dir, "main.rs"), mode="w") as f:
                f.write("workspace {} bin main.rs".format(m))

    # Generate a patch that points the current crate, to the temp dir, and all
    # others to `consts.THIRD_PARTY`. This is to deal with build/dev deps that
    # transitively depend back on the current crate. Otherwise it gets seen in
    # 2 paths.
    patch: dict[str, Any] = {"patch": {"crates-io": {}}}
    cwd = os.getcwd()
    for in_3p in all_3p_tomls._list_of:
        if in_3p.name == orig_name and in_3p.epoch == orig_epoch:
            # If this is the crate we're creating a temp Cargo.toml for, point
            # the patch to the temp dir.
            abspath = dir
        else:
            # Otherwise, point the patch to the downloaded third-party crate's
            # dir.
            abspath = os.path.join(cwd, in_3p.path)
        patch_name = ("{}_v{}".format(
            in_3p.name, common.version_epoch_normalized(in_3p.epoch)))
        patch["patch"]["crates-io"][patch_name] = {
            "version": in_3p.epoch,
            "path": abspath,
            "package": in_3p.name,
        }

    tmp_cargo_toml_path = os.path.join(dir, "Cargo.toml")
    # This is the third-party Cargo.toml file. Note that we do not write
    # the `orig_toml_parsed` as the python parser does not like the contents
    # of some Cargo.toml files that cargo is just fine with. So we write the
    # contents without a round trip through the parser.
    if orig_toml_text:
        cargo_toml_text = orig_toml_text
    else:
        cargo_toml_text = toml.dumps(orig_toml_parsed)
    # We attach our "patch" keys onto it to redirect all crates.io
    # dependencies into `consts.THIRD_PARTY`.
    cargo_toml_text = cargo_toml_text + toml.dumps(patch)
    # Generate our own (temp) copy of a Cargo.toml for the dependency
    # that we will run `cargo tree` against.
    with open(tmp_cargo_toml_path, mode="w") as tmp_cargo_toml:
        tmp_cargo_toml.write(cargo_toml_text)
    if verbose:
        print("Writing to %s:" % tmp_cargo_toml_path)
        print("=======")
        print(cargo_toml_text)
        print("=======")

    return tmp_cargo_toml_path


def gen_list_of_3p_cargo_toml() -> ListOf3pCargoToml:
    """Create a cached view of existing third-party crates.

    Find all the third-party crates present and cache them for generating
    Cargo.toml files in temp dirs that will point to them."""
    list_of: list[ListOf3pCargoToml.CargoToml] = []
    for normalized_crate_name in os.listdir(common.os_third_party_dir()):
        crate_dir = common.os_crate_name_dir(normalized_crate_name)
        if not os.path.isdir(crate_dir):
            continue
        for v_epoch in os.listdir(crate_dir):
            epoch = v_epoch.replace("v", "").replace("_", ".")
            filepath = common.os_crate_cargo_dir(normalized_crate_name,
                                                 epoch,
                                                 rel_path=["Cargo.toml"])
            if os.path.exists(filepath):
                cargo_toml = toml.load(filepath)
                # Note this can't use the directory name because it was
                # normalized, so we read the real name from the Cargo.toml.
                name = cargo_toml["package"]["name"]
                assert common.crate_name_normalized(
                    name) == normalized_crate_name
                # The version epoch comes from the directory name.
                list_of += [
                    ListOf3pCargoToml.CargoToml(
                        name, epoch,
                        common.os_crate_cargo_dir(normalized_crate_name, epoch))
                ]
    return ListOf3pCargoToml(list_of)