blob: 52b66eeace80c6f1c3109b6e731873bc1730d2eb [file] [log] [blame]
// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use gnrt_lib::*;
use crates::{ChromiumVendoredCrate, StdVendoredCrate};
use manifest::*;
use crate::util::{check_exit_ok, check_spawn, check_wait_with_output, create_dirs_if_needed};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;
use anyhow::{ensure, format_err, Context, Result};
pub fn generate(args: &clap::ArgMatches, paths: &paths::ChromiumPaths) -> Result<()> {
if args.get_flag("for-std") {
// This is not fully implemented. Currently, it will print data helpful
// for development then quit.
generate_for_std(&args, &paths)
} else {
generate_for_third_party(&args, &paths)
}
}
fn generate_for_third_party(args: &clap::ArgMatches, paths: &paths::ChromiumPaths) -> Result<()> {
let manifest_contents =
String::from_utf8(fs::read(paths.third_party.join("third_party.toml")).unwrap()).unwrap();
let mut third_party_manifest: ThirdPartyManifest =
toml::de::from_str(&manifest_contents).context("Could not parse third_party.toml")?;
// Collect special fields from third_party.toml.
//
// TODO(crbug.com/1291994): handle visibility separately for each kind.
let mut deps_visibility = HashMap::<ChromiumVendoredCrate, crates::Visibility>::new();
let mut build_script_outputs = HashMap::<ChromiumVendoredCrate, Vec<String>>::new();
let mut gn_variables_libs = HashMap::<ChromiumVendoredCrate, String>::new();
let mut walk_deps = |dep_name: &str, dep_spec: &Dependency, visibility: crates::Visibility| {
let (version_req, is_public, dep_outputs, gn_variables_lib): (
&_,
bool,
&[_],
Option<&String>,
) = match dep_spec {
Dependency::Short(version_req) => (version_req, true, &[], None),
Dependency::Full(dep) => (
dep.version.as_ref().unwrap(),
dep.allow_first_party_usage,
&dep.build_script_outputs,
dep.gn_variables_lib.as_ref(),
),
};
let epoch = crates::Epoch::from_version_req_str(&version_req.0);
let crate_id = ChromiumVendoredCrate { name: dep_name.to_string(), epoch };
deps_visibility.insert(
crate_id.clone(),
if is_public { visibility } else { crates::Visibility::ThirdParty },
);
if !dep_outputs.is_empty() {
build_script_outputs.insert(crate_id.clone(), dep_outputs.to_vec());
}
if gn_variables_lib.is_some() {
gn_variables_libs.insert(crate_id, gn_variables_lib.unwrap().clone());
}
};
for (dep_name, dep_spec) in &third_party_manifest.dependency_spec.dev_dependencies {
walk_deps(dep_name, dep_spec, crates::Visibility::TestOnlyAndThirdParty)
}
for (dep_name, dep_spec) in &third_party_manifest.dependency_spec.dependencies {
walk_deps(dep_name, dep_spec, crates::Visibility::Public)
}
// [build-dependencies] is not used in third_party.toml.
// For crates used in first-party tests, we do not build a separate library from
// production (unlike standard Rust tests, and those found in third-party
// crates.) So we merge the dev_dependencies from third_party.toml into the
// regular dependencies.
third_party_manifest
.dependency_spec
.dependencies
.extend(std::mem::take(&mut third_party_manifest.dependency_spec.dev_dependencies));
// Rebind as immutable.
let (third_party_manifest, deps_visibility, build_script_outputs, gn_variables_libs) =
(third_party_manifest, deps_visibility, build_script_outputs, gn_variables_libs);
// Traverse our third-party directory to collect the set of vendored crates.
// Used to generate Cargo.toml [patch] sections, and later to check against
// `cargo metadata`'s dependency resolution to ensure we have all the crates
// we need. We sort `crates` for a stable ordering of [patch] sections.
let mut crates = crates::collect_third_party_crates(paths.third_party.clone()).unwrap();
crates.sort_unstable_by(|a, b| a.0.cmp(&b.0));
// Generate a fake root Cargo.toml for dependency resolution.
let cargo_manifest = generate_fake_cargo_toml(
third_party_manifest,
crates.iter().map(|(c, _)| manifest::PatchSpecification {
package_name: c.name.clone(),
patch_name: c.patch_name(),
path: c.crate_path(),
}),
);
if args.get_flag("output-cargo-toml") {
println!("{}", toml::ser::to_string(&cargo_manifest).unwrap());
return Ok(());
}
// Create a fake package: Cargo.toml and an empty main.rs. This allows cargo
// to construct a full dependency graph as if Chrome were a cargo package.
write!(
io::BufWriter::new(fs::File::create(paths.third_party.join("Cargo.toml")).unwrap()),
"# {}\n\n{}",
AUTOGENERATED_FILE_HEADER,
toml::to_string(&cargo_manifest).unwrap()
)
.unwrap();
create_dirs_if_needed(&paths.third_party.join("src")).unwrap();
write!(
io::BufWriter::new(fs::File::create(paths.third_party.join("src/main.rs")).unwrap()),
"// {}",
AUTOGENERATED_FILE_HEADER
)
.unwrap();
// Run `cargo metadata` and process the output to get a list of crates we
// depend on.
let mut command = cargo_metadata::MetadataCommand::new();
command.current_dir(&paths.third_party);
let dependencies = deps::collect_dependencies(&command.exec().unwrap(), None, None);
// Compare cargo's dependency resolution with the crates we have on disk. We
// want to ensure:
// * Each resolved dependency matches with a crate we discovered (no missing
// deps).
// * Each discovered crate matches with a resolved dependency (no unused
// crates).
let mut has_error = false;
let present_crates: HashSet<&ChromiumVendoredCrate> = crates.iter().map(|(c, _)| c).collect();
// Construct the set of requested third-party crates, ensuring we don't have
// duplicate epochs. For example, if we resolved two versions of a
// dependency with the same major version, we cannot continue.
let mut req_crates = HashSet::<ChromiumVendoredCrate>::new();
for package in &dependencies {
if !req_crates.insert(package.third_party_crate_id()) {
panic!("found another requested package with the same name and epoch: {:?}", package);
}
}
let req_crates = req_crates;
for dep in dependencies.iter() {
if !present_crates.contains(&dep.third_party_crate_id()) {
has_error = true;
println!("Missing dependency: {} {}", dep.package_name, dep.version);
for edge in dep.dependency_path.iter() {
println!(" {edge}");
}
} else if !dep.is_local {
// Transitive deps may be requested with version requirements stricter than
// ours: e.g. 1.57 instead of just major version 1. If the version we have
// checked in, e.g. 1.56, has the same epoch but doesn't meet the version
// requirement, the symptom is Cargo will resolve the dependency to an
// upstream source instead of our local path. We must detect this case to
// ensure correctness.
has_error = true;
println!(
"Resolved {} {} to an upstream source. The requested version \
likely has the same epoch as the discovered crate but \
something has a more stringent version requirement.",
dep.package_name, dep.version
);
println!(" Resolved version: {}", dep.version);
}
}
for present_crate in present_crates.iter() {
if !req_crates.contains(present_crate) {
has_error = true;
println!("Unused crate: {present_crate}");
}
}
ensure!(!has_error, "Dependency resolution failed");
let build_files: HashMap<ChromiumVendoredCrate, gn::BuildFile> =
gn::build_files_from_chromium_deps(
&dependencies,
&paths,
&crates.iter().cloned().collect(),
&build_script_outputs,
&deps_visibility,
&gn_variables_libs,
);
// Before modifying anything make sure we have a one-to-one mapping of
// discovered crates and build file data.
for (crate_id, _) in build_files.iter() {
// This shouldn't happen, but check anyway in case we have a strange
// logic error above.
assert!(present_crates.contains(&crate_id));
}
for crate_id in present_crates.iter() {
if !build_files.contains_key(*crate_id) {
println!("Error: discovered crate {crate_id}, but no build file was generated.");
has_error = true;
}
}
ensure!(!has_error, "Generated build rules don't match input dependencies");
// Wipe all previous BUILD.gn files. If we fail, we don't want to leave a
// mix of old and new build files.
for build_file in crates.iter().map(|(crate_id, _)| build_file_path(crate_id, &paths)) {
if build_file.exists() {
fs::remove_file(&build_file).unwrap();
}
}
// Generate build files, wiping the previous ones so we don't have any stale
// build rules.
for (crate_id, _) in crates.iter() {
let build_file_path = build_file_path(crate_id, &paths);
let build_file_data = match build_files.get(&crate_id) {
Some(build_file) => build_file,
None => panic!("missing build data for {crate_id}"),
};
write_build_file(&build_file_path, build_file_data).unwrap();
}
Ok(())
}
fn generate_for_std(_args: &clap::ArgMatches, paths: &paths::ChromiumPaths) -> Result<()> {
// Load config file, which applies rustenv and cfg flags to some std crates.
let config_file_contents = std::fs::read_to_string(paths.std_config_file).unwrap();
let config: config::BuildConfig = toml::de::from_str(&config_file_contents).unwrap();
// Run `cargo metadata` from the std package in the Rust source tree (which
// is a workspace).
let mut command = cargo_metadata::MetadataCommand::new();
command.current_dir(paths.std_fake_root);
// Delete the Cargo.lock if it exists.
let mut std_fake_root_cargo_lock = paths.std_fake_root.to_path_buf();
std_fake_root_cargo_lock.push("Cargo.lock");
if let Err(e) = std::fs::remove_file(std_fake_root_cargo_lock) {
match e.kind() {
// Ignore if it already doesn't exist.
std::io::ErrorKind::NotFound => (),
_ => panic!("io error while deleting Cargo.lock: {e}"),
}
}
// Use offline to constrain dependency resolution to those in the Rust src
// tree and vendored crates. Ideally, we'd use "--locked" and use the
// upstream Cargo.lock, but this is not straightforward since the rust-src
// component is not a full Cargo workspace. Since the vendor dir we package
// is generated with "--locked", the outcome should be the same.
command.other_options(vec!["--offline".to_string()]);
// Compute the set of crates we need to build to build libstd. Note this
// contains a few kinds of entries:
// * Rust workspace packages (e.g. core, alloc, std, unwind, etc)
// * Non-workspace packages supplied in Rust source tree (e.g. stdarch)
// * Vendored third-party crates (e.g. compiler_builtins, libc, etc)
// * rust-std-workspace-* shim packages which direct std crates.io
// dependencies to the correct lib{core,alloc,std} when depended on by the
// Rust codebase (see
// https://github.com/rust-lang/rust/tree/master/library/rustc-std-workspace-core)
//
// libtest is the root of the std crate dependency tree, so start there.
let mut dependencies =
deps::collect_dependencies(&command.exec().unwrap(), Some(vec!["test".to_string()]), None);
// Remove dev dependencies since tests aren't run. Also remove build deps
// since we configure flags and env vars manually. Include libtest
// explicitly since, as the root of collect_dependencies(), it doesn't get a
// dependency_kinds entry.
dependencies.retain(|dep| {
dep.package_name == "test"
|| dep.dependency_kinds.contains_key(&deps::DependencyKind::Normal)
});
dependencies.sort_unstable_by(|a, b| {
a.package_name.cmp(&b.package_name).then(a.version.cmp(&b.version))
});
let third_party_deps = dependencies.iter().filter(|dep| !dep.is_local).collect::<Vec<_>>();
// Check that all resolved third party deps are available. First, collect
// the set of third-party dependencies vendored in the Rust source package.
let vendored_crates: HashMap<StdVendoredCrate, manifest::CargoPackage> =
crates::collect_std_vendored_crates(paths.rust_src_vendor).unwrap().into_iter().collect();
// Collect vendored dependencies, and also check that all resolved
// dependencies point to our Rust source package. Build rules will be
// generated for these crates separately from std, alloc, and core which
// need special treatment.
let src_prefix = paths.root.join(paths.rust_src);
for dep in third_party_deps.iter() {
// Only process deps with a library target: we are producing build rules
// for the standard library, so transitive binary dependencies don't
// make sense.
let lib = match &dep.lib_target {
Some(lib) => lib,
None => continue,
};
ensure!(
lib.root.canonicalize().unwrap().starts_with(&src_prefix),
"Found dependency that was not locally available: {} {}\n{:?}",
dep.package_name,
dep.version,
dep
);
vendored_crates
.get_key_value(&StdVendoredCrate {
name: dep.package_name.clone(),
version: dep.version.clone(),
// Placeholder value for lookup.
is_latest: false,
})
.ok_or_else(|| {
format_err!(
"Resolved dependency does not match any vendored crate: {} {}",
dep.package_name,
dep.version
)
})?;
}
let build_file = gn::build_file_from_std_deps(dependencies.iter(), paths, &config);
write_build_file(&paths.std_build.join("BUILD.gn"), &build_file).unwrap();
Ok(())
}
fn build_file_path(crate_id: &ChromiumVendoredCrate, paths: &paths::ChromiumPaths) -> PathBuf {
let mut path = paths.root.clone();
path.push(&paths.third_party);
path.push(crate_id.build_path());
path.push("BUILD.gn");
path
}
fn write_build_file(path: &Path, build_file: &gn::BuildFile) -> Result<()> {
let cmd_name = "gn format";
let output_handle = fs::File::create(path)
.with_context(|| format!("Could not create GN output file {}", path.to_string_lossy()))?;
// Spawn a child process to format GN rules. The formatted GN is written to
// the file `output_handle`.
let mut child = check_spawn(
&mut process::Command::new("gn")
.arg("format")
.arg("--stdin")
.stdin(process::Stdio::piped())
.stdout(output_handle),
cmd_name,
)?;
write!(io::BufWriter::new(child.stdin.take().unwrap()), "{}", build_file.display())
.context("Failed to write to GN format process")?;
check_exit_ok(&check_wait_with_output(child, cmd_name)?, cmd_name)
}
/// A message prepended to autogenerated files. Notes this tool generated it and
/// not to edit directly.
static AUTOGENERATED_FILE_HEADER: &'static str = "!!! DO NOT EDIT -- Autogenerated by gnrt from third_party.toml. Edit that file instead. See tools/crates/gnrt.";