tools/crates/gnrt/gen.rs

// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use crate::config;
use crate::crates::{self, CrateFiles, Epoch, NormalizedName, VendoredCrate};
use crate::deps;
use crate::gn;
use crate::paths;
use crate::util::{
    check_exit_ok, check_spawn, check_wait_with_output, create_dirs_if_needed, init_handlebars,
    run_cargo_metadata,
};
use crate::GenCommandArgs;

use std::collections::{HashMap, HashSet};
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;

use anyhow::{ensure, format_err, Context, Result};

pub fn generate(
    args: GenCommandArgs,
    tools: &paths::ToolPaths,
    paths: &paths::ChromiumPaths,
) -> Result<()> {
    if args.for_std.is_some() {
        generate_for_std(args, tools, paths)
    } else {
        generate_for_third_party(args, tools, paths)
    }
}

fn generate_for_std(
    args: GenCommandArgs,
    tools: &paths::ToolPaths,
    paths: &paths::ChromiumPaths,
) -> Result<()> {
    // Load config file, which applies rustenv and cfg flags to some std crates.
    let config_file_contents = std::fs::read_to_string(paths.std_config_file).unwrap();
    let config: config::BuildConfig = toml::de::from_str(&config_file_contents).unwrap();

    let template_path =
        paths.std_config_file.parent().unwrap().join(&config.gn_config.build_file_template);

    let handlebars = init_handlebars(&template_path)?;

    // The Rust source tree, containing the standard library and vendored
    // dependencies.
    let rust_src_root = args.for_std.as_ref().unwrap();

    println!("Generating stdlib GN rules from {}", rust_src_root);

    let cargo_config = std::fs::read_to_string(paths.std_fake_root_config_template)
        .unwrap()
        .replace("RUST_SRC_ROOT", rust_src_root);
    std::fs::write(
        paths.strip_template(paths.std_fake_root_config_template).unwrap(),
        cargo_config,
    )
    .unwrap();

    let cargo_toml = std::fs::read_to_string(paths.std_fake_root_cargo_template)
        .unwrap()
        .replace("RUST_SRC_ROOT", rust_src_root);
    std::fs::write(paths.strip_template(paths.std_fake_root_cargo_template).unwrap(), cargo_toml)
        .unwrap();
    // Convert the `rust_src_root` to a Path hereafter.
    let rust_src_root = paths.root.join(Path::new(rust_src_root));

    // Delete the Cargo.lock if it exists.
    let mut std_fake_root_cargo_lock = paths.std_fake_root.to_path_buf();
    std_fake_root_cargo_lock.push("Cargo.lock");
    if let Err(e) = std::fs::remove_file(std_fake_root_cargo_lock) {
        match e.kind() {
            // Ignore if it already doesn't exist.
            std::io::ErrorKind::NotFound => (),
            _ => panic!("io error while deleting Cargo.lock: {e}"),
        }
    }

    // The Cargo.toml files in the Rust toolchain may use nightly Cargo
    // features, but the cargo binary is beta. This env var enables the
    // beta cargo binary to allow nightly features anyway.
    // https://github.com/rust-lang/rust/commit/2e52f4deb0544480b6aefe2c0cc1e6f3c893b081
    let cargo_extra_env: HashMap<std::ffi::OsString, std::ffi::OsString> =
        [("RUSTC_BOOTSTRAP".into(), "1".into())].into_iter().collect();

    // Use offline to constrain dependency resolution to those in the Rust src
    // tree and vendored crates. Ideally, we'd use "--locked" and use the
    // upstream Cargo.lock, but this is not straightforward since the rust-src
    // component is not a full Cargo workspace. Since the vendor dir we package
    // is generated with "--locked", the outcome should be the same.
    let cargo_extra_options = vec!["--offline".to_string()];

    // Compute the set of crates we need to build libstd. Note this
    // contains a few kinds of entries:
    // * Rust workspace packages (e.g. core, alloc, std, unwind, etc)
    // * Non-workspace packages supplied in Rust source tree (e.g. stdarch)
    // * Vendored third-party crates (e.g. compiler_builtins, libc, etc)
    // * rust-std-workspace-* shim packages which direct std crates.io
    //   dependencies to the correct lib{core,alloc,std} when depended on by the
    //   Rust codebase (see
    //   https://github.com/rust-lang/rust/tree/master/library/rustc-std-workspace-core)
    let mut dependencies = deps::collect_dependencies(
        &run_cargo_metadata(
            paths.std_fake_root.into(),
            tools,
            cargo_extra_options,
            cargo_extra_env,
        )?,
        Some(vec![config.resolve.root.clone()]),
        None,
        &config,
    )?;

    // Filter out any crates' dependencies removed by config file.
    for dep in dependencies.iter_mut() {
        let combined: HashSet<&str> =
            config.get_combined_set(&dep.package_name, |crate_cfg| &crate_cfg.remove_deps);
        if combined.is_empty() {
            continue;
        }

        for kind in [&mut dep.dependencies, &mut dep.build_dependencies] {
            kind.retain(|dep_of_dep| !combined.contains(dep_of_dep.package_name.as_str()));
        }
    }

    // Remove any excluded dep entries.
    dependencies
        .retain(|dep| !config.resolve.remove_crates.iter().any(|r| **r == dep.package_name));

    // Remove dev dependencies since tests aren't run. Also remove build deps
    // since we configure flags and env vars manually. Include the root
    // explicitly since it doesn't get a dependency_kinds entry.
    dependencies.retain(|dep| dep.dependency_kinds.contains_key(&deps::DependencyKind::Normal));

    dependencies.sort_unstable_by(|a, b| {
        a.package_name.cmp(&b.package_name).then(a.version.cmp(&b.version))
    });
    for dep in dependencies.iter_mut() {
        // Rehome stdlib deps from the `rust_src_root` to where they will be installed
        // in the Chromium checkout.
        let gn_prefix = paths.root.join(paths.rust_src_installed);
        if let Some(lib) = dep.lib_target.as_mut() {
            ensure!(
                lib.root.canonicalize().unwrap().starts_with(&rust_src_root),
                "Found dependency that was not locally available: {} {}\n{:?}",
                dep.package_name,
                dep.version,
                dep
            );

            if let Ok(remain) = lib.root.canonicalize().unwrap().strip_prefix(&rust_src_root) {
                lib.root = gn_prefix.join(remain);
            }
        }

        if let Some(path) = dep.build_script.as_mut() {
            if let Ok(remain) = path.canonicalize().unwrap().strip_prefix(&rust_src_root) {
                *path = gn_prefix.join(remain);
            }
        }
    }

    let third_party_deps = dependencies.iter().filter(|dep| !dep.is_local).collect::<Vec<_>>();

    // Check that all resolved third party deps are available. First, collect
    // the set of third-party dependencies vendored in the Rust source package.
    let vendored_crates: HashSet<VendoredCrate> =
        crates::collect_std_vendored_crates(&rust_src_root.join(paths.rust_src_vendor_subdir))
            .unwrap()
            .into_iter()
            .collect();

    // Collect vendored dependencies, and also check that all resolved
    // dependencies point to our Rust source package. Build rules will be
    // generated for these crates separately from std, alloc, and core which
    // need special treatment.
    for dep in third_party_deps.iter() {
        // Only process deps with a library target: we are producing build rules
        // for the standard library, so transitive binary dependencies don't
        // make sense.
        if dep.lib_target.is_none() {
            continue;
        }

        vendored_crates
            .get(&VendoredCrate { name: dep.package_name.clone(), version: dep.version.clone() })
            .ok_or_else(|| {
                format_err!(
                    "Resolved dependency does not match any vendored crate: {} {}",
                    dep.package_name,
                    dep.version
                )
            })?;
    }

    let crate_inputs: HashMap<VendoredCrate, CrateFiles> = dependencies
        .iter()
        .filter(|p| p.lib_target.is_some())
        .map(|p| {
            crates::collect_std_crate_files(p, &config, crates::IncludeCrateTargets::LibOnly)
                .expect("missing a stdlib input file, did you gclient sync?")
        })
        .collect();

    let build_file = gn::build_file_from_std_deps(
        dependencies.iter(),
        paths,
        &config,
        gn::NameLibStyle::PackageName,
        |crate_id| crate_inputs.get(crate_id).unwrap(),
    )?;

    if args.dump_template_input {
        return serde_json::to_writer_pretty(
            std::fs::File::create("gnrt-template-input.json").context("opening dump file")?,
            &build_file,
        )
        .context("dumping gn information");
    }

    let gn_str = handlebars.render("template", &build_file)?;
    write_build_file(&paths.std_build.join("BUILD.gn"), gn_str).unwrap();

    Ok(())
}

fn generate_for_third_party(
    args: GenCommandArgs,
    tools: &paths::ToolPaths,
    paths: &paths::ChromiumPaths,
) -> Result<()> {
    let config_file_contents = std::fs::read_to_string(paths.third_party_config_file).unwrap();
    let config: config::BuildConfig = toml::de::from_str(&config_file_contents).unwrap();

    let template_path =
        paths.third_party_config_file.parent().unwrap().join(&config.gn_config.build_file_template);
    let handlebars = init_handlebars(&template_path)?;

    println!("Generating third-party GN rules from {}", paths.third_party_cargo_root.display());

    let cargo_extra_options = vec![
        // Use offline to constrain dependency resolution to locally vendored crates.
        "--offline".to_string(),
        // Use locked to prevent updating dependencies at the same time as generating
        // metadata.
        "--locked".to_string(),
    ];

    // Compute the set of all third-party crates.
    let mut dependencies = deps::collect_dependencies(
        &run_cargo_metadata(
            paths.third_party_cargo_root.into(),
            tools,
            cargo_extra_options,
            HashMap::new(),
        )?,
        Some(vec![config.resolve.root.clone()]),
        None,
        &config,
    )?;

    // Filter out any crates' dependencies removed by config file.
    for dep in dependencies.iter_mut() {
        let all: Option<&Vec<String>> = Some(&config.all_config.remove_deps);
        let per: Option<&Vec<String>> =
            config.per_crate_config.get(&dep.package_name).map(|config| &config.remove_deps);

        let combined: Vec<&String> = all.into_iter().chain(per).flatten().collect();
        if combined.is_empty() {
            continue;
        }

        for kind in [&mut dep.dependencies, &mut dep.build_dependencies] {
            kind.retain(|dep_of_dep| !combined.iter().any(|r| **r == dep_of_dep.package_name));
        }
    }

    // Remove any excluded dep entries.
    dependencies
        .retain(|dep| !config.resolve.remove_crates.iter().any(|r| **r == dep.package_name));

    // Remove dev dependencies since tests aren't run.
    dependencies.retain(|dep| {
        dep.dependency_kinds.contains_key(&deps::DependencyKind::Normal)
        // TODO: Needed?
            || dep.dependency_kinds.contains_key(&deps::DependencyKind::Build)
    });

    dependencies.sort_unstable_by(|a, b| {
        a.package_name.cmp(&b.package_name).then(a.version.cmp(&b.version))
    });

    let crate_inputs: HashMap<VendoredCrate, CrateFiles> = dependencies
        .iter()
        .map(|p| {
            crates::collect_std_crate_files(p, &config, crates::IncludeCrateTargets::LibAndBin)
                .expect(&format!(
                    "missing a crate input file for '{}'. Dependencies are not vendored?",
                    p.package_name
                ))
        })
        .collect();

    // If there are multiple crates with the same epoch, this is unexpected.
    // Bail out.
    {
        let mut found = HashSet::new();
        for dep in &dependencies {
            let epoch = crates::Epoch::from_version(&dep.version);
            if found.insert((&dep.package_name, epoch)) == false {
                Err(format_err!(
                    "Two '{}' crates found with the same {} epoch",
                    dep.package_name,
                    epoch
                ))?
            }
        }
    }

    // Split up the dependencies by crate and epoch.
    let all_build_files: HashMap<PathBuf, gn::BuildFile> = {
        let mut map = HashMap::new();
        for dep in &dependencies {
            let build_file = gn::build_file_from_std_deps(
                std::iter::once(dep),
                paths,
                &config,
                // TODO(danakj): Change to PackageName for consistency?
                gn::NameLibStyle::LibLiteral,
                |crate_id| crate_inputs.get(crate_id).unwrap(),
            )?;
            let path = paths
                .third_party
                .join(NormalizedName::from_crate_name(&dep.package_name).as_str())
                .join(Epoch::from_version(&dep.version).to_string());
            let previous = map.insert(path, build_file);
            if previous.is_some() {
                Err(format_err!(
                    "multiple versions of crate {} with the same epoch",
                    dep.package_name
                ))?
            }
        }
        map
    };

    for (dir, _) in &all_build_files {
        create_dirs_if_needed(dir).context(format!("dir: {}", dir.display()))?;
    }

    if args.dump_template_input {
        for (dir, build_file) in &all_build_files {
            serde_json::to_writer_pretty(
                std::fs::File::create(dir.join("gnrt-template-input.json"))
                    .context("opening dump file")?,
                &build_file,
            )
            .context("dumping gn information")?;
        }
        return Ok(());
    }

    for (dir, build_file) in &all_build_files {
        let gn_str = handlebars.render("template", &build_file)?;
        write_build_file(&dir.join("BUILD.gn"), gn_str).unwrap();
    }
    Ok(())
}

fn write_build_file(path: &Path, content: String) -> Result<()> {
    let cmd_name = "gn format";
    let output_handle = fs::File::create(path)
        .with_context(|| format!("Could not create GN output file {}", path.to_string_lossy()))?;

    // Spawn a child process to format GN rules. The formatted GN is written to
    // the file `output_handle`.
    let mut child = check_spawn(
        process::Command::new(if cfg!(windows) { "gn.bat" } else { "gn" })
            .arg("format")
            .arg("--stdin")
            .stdin(process::Stdio::piped())
            .stdout(output_handle),
        cmd_name,
    )?;

    write!(io::BufWriter::new(child.stdin.take().unwrap()), "{}", content)
        .context("Failed to write to GN format process")?;
    check_exit_ok(&check_wait_with_output(child, cmd_name)?, cmd_name)
}