projects/populate-workspace.py

#!/usr/bin/env python3
# Copyright 2022 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Populates the top-level Cargo.toml with all necessary workspace entries.

Also updates cargo-vet's policy information with all of the crates discovered
in the tree.
"""

import argparse
import logging
import os
from pathlib import Path
import subprocess
import sys
import textwrap
from typing import List


sys.path.append(str(Path(__file__).resolve().parent.parent / "scripts"))
import rust_crates


rust_crates.die_if_running_as_root()
rust_crates.run_inside_chroot()
rust_crates.install_tomli_and_reexec_if_unavailable()

import tomli
import tomli_w


WORKSPACE_FILE_HEADER = """\
# Copyright 2022 The ChromiumOS Authors.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# !! Autogenerated by `populate-workspace.py`; please don't edit. !!

[workspace.metadata.vet]
store = { path = '../cargo-vet' }

"""

# Tools which only run on the host. These are subject to less stringent audit
# requirements.
HOST_TOOLS = frozenset(
    (
        "bindgen-deps",
        "chromeos-dbus-bindings-deps",
        "cxxbridge-cmd-deps",
        "dbus-codegen-deps",
        "factory_installer-deps",
        "flashrom_tester-deps",
        "prjoxide-deps",
        "pyprjoxide-deps",
    )
)

# Dependencies that should not be treated as crates.io dependencies.
NON_CRATES_IO_DEPS = (
    "deqp-runner",
    "dlib",
    "wayland-commons",
    "wayland-scanner",
    "wayland-server",
    "wayland-sys",
)


def update_cargo_vet_info(
    projects_dir: Path, projects: List[str], use_relaxed_target_criteria: bool
):
    """Updates cargo-vet's config.toml with requirements for all `projects`."""
    cargo_vet_config = projects_dir.parent / "cargo-vet" / "config.toml"
    config = cargo_vet_config.read_text(encoding="utf-8")

    cargo_vet_policy = tomli.loads(config).get("policy", ())
    project_names = []
    unseen_io_deps = set(
        x for x in NON_CRATES_IO_DEPS if x not in cargo_vet_policy
    )
    for project in projects:
        cargo_toml = projects_dir / project / "Cargo.toml"
        with cargo_toml.open("rb") as f:
            project_name = tomli.load(f)["package"]["name"]
        project_names.append((project, project_name))
        unseen_io_deps.discard(project_name)

    add_projects = [x for x in project_names if x[1] not in cargo_vet_policy]
    add_projects += ((None, x) for x in unseen_io_deps)
    if not add_projects:
        logging.info("No cargo-vet-related updates to make.")
        return

    host_safety_criteria = "safe-to-run"
    target_safety_criteria = "rule-of-two-safe-to-deploy"

    # Stick stuff at the end and have `cargo-vet` handle making it
    # pretty/sorting it during `vendor.py`.
    with cargo_vet_config.open("ab") as f:
        for project_path, project_name in add_projects:
            if project_path:
                logging.info(
                    "Adding entry %s from %s into cargo-vet",
                    project_path,
                    project_name,
                )
            else:
                logging.info(
                    "Synthesizing entry %s for cargo-vet",
                    project_name,
                )

            if project_name in HOST_TOOLS:
                safety_criteria = host_safety_criteria
                note = "Only safe-to-run is required, as this is a host tool."
            elif use_relaxed_target_criteria:
                safety_criteria = host_safety_criteria
                note = "Using relaxed target criteria to help with migration."
            else:
                safety_criteria = target_safety_criteria
                note = None

            toml_project_policy = {
                "criteria": [safety_criteria, "crypto-safe"],
                "dev-criteria": [host_safety_criteria, "crypto-safe"],
            }

            if note:
                toml_project_policy["notes"] = note

            if project_name in NON_CRATES_IO_DEPS:
                toml_project_policy["audit-as-crates-io"] = False

            toml_to_write = {
                "policy": {
                    project_name: toml_project_policy,
                }
            }

            f.write(b"\n")
            tomli_w.dump(toml_to_write, f)

    # NOTE(b/274643706): as referenced, we defer formatting to `vendor.py`. For
    # some reason, `cargo-vet` will run `cargo manifest` even if it's only
    # asked to format files. Cargo.lock may be out of date, and we don't want
    # to modify that here, so we cannot format here.

    logging.info("Cargo vet info updated.")


def find_projects(projects_dir: Path) -> List[str]:
    """Returns a list of projects under `projects_dir`.

    This creates src/ directories for any project that does not currently have
    one, for convenience.
    """
    projects = []
    for dir_path, subdirs, files in os.walk(projects_dir):
        if "Cargo.toml" not in files:
            continue

        dir_path = Path(dir_path)
        if dir_path == projects_dir:
            continue

        projects.append(dir_path.relative_to(projects_dir))

        # It's a waste to descend into src directories.
        if "src" in subdirs:
            del subdirs[subdirs.index("src")]
        else:
            # ...Though if src/ doesn't exist, Cargo will get confused.
            # Synthesize one with a nop lib.rs.
            src = dir_path / "src"
            src.mkdir()
            (src / "lib.rs").write_bytes(b"")
            logging.info("Synthesized src/lib.rs for %s", dir_path)

    projects.sort()
    return projects


def get_parser() -> argparse.ArgumentParser:
    """Gets the arg parser for this script."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--use-strict-target-criteria",
        action="store_true",
        help="Require rule-of-two-safe-to-deploy for target crates.",
    )
    parser.add_argument(
        "--skip-version-check",
        action="store_true",
        help="Don't exit if our repo isn't up-to-date.",
    )
    return parser


def main(argv: List[str]):
    """Main function."""
    opts = get_parser().parse_args(argv)

    logging.basicConfig(
        format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: "
        "%(message)s",
        level=logging.INFO,
    )

    projects_dir = Path(__file__).resolve().parent
    if not opts.skip_version_check:
        rust_crates.exit_if_head_is_not_up_to_date(
            projects_dir.parent, disable_check_flag="--skip-version-check"
        )
    projects = find_projects(projects_dir)
    assert projects, f"No projects found under {projects_dir}"
    logging.info("Identified %d projects", len(projects))

    workspace_toml_file = projects_dir / "Cargo.toml"
    with workspace_toml_file.open("wb") as f:
        f.write(WORKSPACE_FILE_HEADER.encode(encoding="utf-8"))
        toml_to_write = {
            "workspace": {
                "members": [str(x) for x in projects],
            },
        }
        tomli_w.dump(toml_to_write, f)

    logging.info("Workspace Cargo.toml successfully written.")
    update_cargo_vet_info(
        projects_dir,
        projects,
        use_relaxed_target_criteria=not opts.use_strict_target_criteria,
    )


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))