| // Copyright 2023 The ChromiumOS Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| use std::io::Write; |
| use std::os::unix::fs; |
| use std::os::unix::fs::PermissionsExt; |
| use std::path::Path; |
| use std::sync::Arc; |
| |
| use std::{env::current_dir, path::PathBuf}; |
| |
| use crate::digest_repo::digest_repo_main; |
| use crate::dump_package::dump_package_main; |
| use crate::dump_profile::dump_profile_main; |
| use crate::generate_repo::generate_repo_main; |
| |
| use alchemist::data::Vars; |
| use alchemist::fakechroot; |
| use alchemist::toolchain::ToolchainConfig; |
| use alchemist::{ |
| config::{ |
| bundle::ConfigBundle, profile::Profile, site::SiteSettings, ConfigNode, ConfigNodeValue, |
| ConfigSource, PackageMaskKind, PackageMaskUpdate, SimpleConfigSource, UseUpdate, |
| UseUpdateFilter, UseUpdateKind, |
| }, |
| ebuild::{metadata::CachedEBuildEvaluator, CachedPackageLoader, PackageLoader}, |
| fakechroot::{enter_fake_chroot, PathTranslator}, |
| repository::RepositorySet, |
| resolver::PackageResolver, |
| toolchain::load_toolchains, |
| }; |
| use anyhow::{bail, Context, Result}; |
| use clap::{ArgAction, Parser, Subcommand}; |
| use tempfile::TempDir; |
| |
| /// Returns true when running inside ChromeOS SDK chroot. |
| /// Do not use this function to change the alchemist's behavior inside/outside |
| /// the chroot without allowing the user to make the decision. Instead, you |
| /// should look at command line flag values, such as |
| /// [`Args::use_portage_site_configs`], whose default values are computed with |
| /// this function. |
| fn is_inside_chroot() -> Result<bool> { |
| Path::new("/etc/cros_chroot_version") |
| .try_exists() |
| .context("Failed to stat /etc/cros_chroot_version") |
| } |
| |
| #[derive(Parser, Debug)] |
| #[command(name = "alchemist")] |
| #[command(author = "ChromiumOS Authors")] |
| #[command(about = "Analyzes Portage trees", long_about = None)] |
| pub struct Args { |
| /// Board name to build packages for. |
| #[arg(short = 'b', long, value_name = "NAME", global = true)] |
| board: Option<String>, |
| |
| /// Build packages for the host. |
| #[arg( |
| long, |
| default_value_t = false, |
| // Following settings are need to allow --flag[=(false|true)]. |
| default_missing_value = "true", |
| num_args = 0..=1, |
| require_equals = true, |
| action = ArgAction::Set, |
| global = true, |
| )] |
| host: bool, |
| |
| /// USE flags to override. |
| #[arg(long, global = true)] |
| use_flags: Option<String>, |
| |
| /// Profile of the board. |
| #[arg( |
| short = 'p', |
| long, |
| value_name = "PROFILE", |
| default_value = "base", |
| global = true |
| )] |
| profile: String, |
| |
| /// Name of the host repository. |
| #[arg(long, value_name = "NAME", default_value = "amd64-host", global = true)] |
| host_board: String, |
| |
| /// Profile name of the host target. |
| #[arg( |
| long, |
| value_name = "PROFILE", |
| default_value = "sdk/bootstrap", |
| global = true |
| )] |
| host_profile: String, |
| |
| /// Uses the Portage site configs found at `/etc` and `/build/$BOARD/etc`. |
| /// |
| /// If this flag is set to false, Portage site configs internally generated |
| /// by alchemist are used to load and evaluate Portage packages. This is |
| /// useful when running alchemist outside the CrOS chroot. |
| /// |
| /// The default value is true if alchemist is running inside the CrOS |
| /// chroot; otherwise false. |
| #[arg( |
| long, |
| default_value_t = is_inside_chroot().unwrap(), |
| // Following settings are need to allow --flag[=(false|true)]. |
| default_missing_value = "true", |
| num_args = 0..=1, |
| require_equals = true, |
| action = ArgAction::Set, |
| global = true, |
| )] |
| use_portage_site_configs: bool, |
| |
| /// Builds first-party 9999 ebuilds even if they are not cros-workon'ed. |
| /// |
| /// Precisely speaking, by setting this flag, alchemist accepts packages |
| /// that are cros-workon'able (inherits cros-workon.eclass), |
| /// non-manually-upreved (CROS_WORKON_MANUAL_UPREV is not set to 1), |
| /// ebuilds whose version is 9999, even if they're marked unstable in |
| /// KEYWORDS. |
| /// |
| /// By setting this flag to true, you can build most of first-party packages |
| /// from the checked out source code, which is much easier to work with. |
| /// |
| /// The default value is true if alchemist is running outside the CrOS |
| /// chroot; otherwise false. |
| /// |
| /// We may change the default value of this flag in the future. |
| #[arg( |
| long, |
| default_value_t = !is_inside_chroot().unwrap(), |
| // Following settings are need to allow --flag[=(false|true)]. |
| default_missing_value = "true", |
| num_args = 0..=1, |
| require_equals = true, |
| action = ArgAction::Set, |
| global = true, |
| )] |
| force_accept_9999_ebuilds: bool, |
| |
| /// Path to the ChromiumOS source directory root. |
| /// If unset, it is inferred from the current directory. |
| #[arg(short = 's', long, value_name = "DIR", global = true)] |
| source_dir: Option<String>, |
| |
| #[command(subcommand)] |
| command: Commands, |
| } |
| |
| #[derive(Subcommand, Debug)] |
| pub enum Commands { |
| /// Dumps information of packages. |
| DumpPackage { |
| #[command(flatten)] |
| args: crate::dump_package::Args, |
| }, |
| /// Dumps profile information. |
| DumpProfile { |
| #[command(flatten)] |
| args: crate::dump_profile::Args, |
| }, |
| /// Generates a Bazel repository containing overlays and packages. |
| GenerateRepo { |
| /// Output directory path. |
| #[arg(short = 'o', long, value_name = "PATH")] |
| output_dir: PathBuf, |
| |
| #[arg(long)] |
| /// An output path for a json-encoded Vec<deps::Repository>. |
| output_repos_json: PathBuf, |
| }, |
| /// Generates a digest of the repository that can be used to indicate if |
| /// any of the overlays, ebuilds, eclasses, etc have changed. |
| DigestRepo { |
| /// Directory used to store a (file_name, mtime) => digest cache. |
| #[command(flatten)] |
| args: crate::digest_repo::Args, |
| }, |
| } |
| |
| fn default_source_dir() -> Result<PathBuf> { |
| for dir in current_dir()?.ancestors() { |
| if dir.join(".repo").exists() { |
| return Ok(dir.to_owned()); |
| } |
| } |
| bail!( |
| "Cannot locate the CrOS source checkout directory from the current directory; \ |
| consider passing --source-dir option" |
| ); |
| } |
| |
| fn build_override_config_source( |
| sysroot: &Path, |
| use_portage_site_configs: bool, |
| ) -> Result<SimpleConfigSource> { |
| let mut masked = vec![ |
| // HACK: Mask chromeos-base/chromeos-lacros-9999 as it's not functional. |
| PackageMaskUpdate { |
| kind: PackageMaskKind::Mask, |
| atom: "=chromeos-base/chromeos-lacros-9999".parse().unwrap(), |
| }, |
| // We don't want to build 9999 llvm-project ebuilds as they currently require |
| // the whole .git directory. This causes problems because everything will |
| // cache bust between hosts and syncs. |
| PackageMaskUpdate { |
| kind: PackageMaskKind::Mask, |
| atom: "=sys-libs/scudo-9999".parse().unwrap(), |
| }, |
| PackageMaskUpdate { |
| kind: PackageMaskKind::Mask, |
| atom: "=sys-devel/llvm-9999".parse().unwrap(), |
| }, |
| ]; |
| let toolchain_categories = [ |
| "sys-libs", |
| "cross-aarch64-cros-linux-gnu", |
| "cross-x86_64-cros-linux-gnux32", |
| "cross-i686-cros-linux-gnu", |
| "cross-x86_64-cros-linux-gnu", |
| "cross-armv7m-cros-eabi", |
| "cross-armv7a-cros-linux-gnueabihf", |
| ]; |
| |
| let toolchain_packages = ["libcxx", "compiler-rt", "llvm-libunwind"]; |
| for category in toolchain_categories { |
| for package_name in toolchain_packages { |
| masked.push(PackageMaskUpdate { |
| kind: PackageMaskKind::Mask, |
| atom: format!("={category}/{package_name}-9999").parse().unwrap(), |
| }); |
| } |
| } |
| |
| let mut nodes = vec![ConfigNode { |
| sources: vec![], |
| value: ConfigNodeValue::PackageMasks(masked), |
| }]; |
| |
| // When using Portage site configs inside the chroot, we need to override |
| // the PKGDIR, PORTAGE_TMPDIR, and PORT_LOGDIR that are defined in |
| // make.conf.amd64-host because they are pointing to the BROOT. |
| // When using fakechroot, we set the values in the generated |
| // make.conf.host_setup file. |
| if use_portage_site_configs { |
| nodes.push(ConfigNode { |
| sources: vec![], |
| value: ConfigNodeValue::Vars(Vars::from_iter([ |
| ( |
| "PKGDIR".to_string(), |
| format!("{}/packages/", sysroot.display()), |
| ), |
| ( |
| "PORTAGE_TMPDIR".to_string(), |
| format!("{}/tmp/", sysroot.display()), |
| ), |
| ( |
| "PORT_LOGDIR".to_string(), |
| format!("{}/tmp/portage/logs/", sysroot.display()), |
| ), |
| ])), |
| }); |
| } |
| Ok(SimpleConfigSource::new(nodes)) |
| } |
| |
| fn setup_tools() -> Result<TempDir> { |
| let mut alchemist = std::env::current_exe()?; |
| |
| let hermetic_dir = alchemist.parent().unwrap(); |
| // Check if we're using the hermetic launcher. |
| // The hermetic launcher requires mktemp, tail, and tar, none of which |
| // exist in the environment which ver_rs and ver_test run. |
| if hermetic_dir.join("_real_binary").is_file() { |
| // Instead of running the self-extracting binary, run the already |
| // extracted binary. This means we don't need those tools to exist. |
| let content = format!( |
| r#" |
| #!/bin/bash |
| |
| exec "{:?}" --argv0 "$0" --library-path "{:?}" --inhibit-rpath "" "{:?}" "$@" |
| "#, |
| alchemist, |
| hermetic_dir.join("_hermetic_lib"), |
| hermetic_dir.join("_real_binary") |
| ); |
| |
| alchemist = hermetic_dir.join("launcher"); |
| let mut f = std::fs::File::create(&alchemist)?; |
| f.write(content.as_bytes())?; |
| let mut permissions = f.metadata()?.permissions(); |
| permissions.set_mode(0o755); |
| f.set_permissions(permissions)?; |
| } |
| |
| let tools_dir = tempfile::tempdir()?; |
| |
| fs::symlink(&alchemist, tools_dir.path().join("ver_test"))?; |
| fs::symlink(&alchemist, tools_dir.path().join("ver_rs"))?; |
| |
| Ok(tools_dir) |
| } |
| |
| /// Container that contains all the data structures for a specific board. |
| pub struct TargetData { |
| pub sysroot: PathBuf, |
| pub board: String, |
| pub profile: String, |
| pub repos: Arc<RepositorySet>, |
| pub config: Arc<ConfigBundle>, |
| pub loader: Arc<CachedPackageLoader>, |
| pub resolver: PackageResolver, |
| pub toolchains: ToolchainConfig, |
| pub profile_path: PathBuf, |
| } |
| |
| fn load_board( |
| repos: RepositorySet, |
| evaluator: &Arc<CachedEBuildEvaluator>, |
| board: &str, |
| profile_name: &str, |
| root_dir: &Path, |
| use_flags: Option<String>, |
| use_portage_site_configs: bool, |
| force_accept_9999_ebuilds: bool, |
| ) -> Result<TargetData> { |
| let repos = Arc::new(repos); |
| |
| // Load configurations. |
| let (config, profile_path) = { |
| let profile = Profile::load_default(root_dir, &repos)?; |
| let site_settings = SiteSettings::load(root_dir)?; |
| let override_source = build_override_config_source(root_dir, use_portage_site_configs)?; |
| |
| let profile_path = profile.profile_path().to_path_buf(); |
| |
| let mut config_sources = vec![ |
| // The order matters. |
| Box::new(profile) as Box<dyn ConfigSource>, |
| Box::new(site_settings) as Box<dyn ConfigSource>, |
| Box::new(override_source) as Box<dyn ConfigSource>, |
| ]; |
| if let Some(flags) = use_flags { |
| let use_override = SimpleConfigSource::new(vec![ConfigNode { |
| sources: vec![], |
| value: ConfigNodeValue::Uses(vec![UseUpdate { |
| kind: UseUpdateKind::Set, |
| filter: UseUpdateFilter { |
| atom: None, |
| stable_only: false, |
| }, |
| use_tokens: flags, |
| }]), |
| }]); |
| config_sources.push(Box::new(use_override) as Box<dyn ConfigSource>); |
| } |
| |
| (ConfigBundle::from_sources(config_sources), profile_path) |
| }; |
| |
| let config = Arc::new(config); |
| |
| let loader = Arc::new(CachedPackageLoader::new(PackageLoader::new( |
| Arc::clone(evaluator), |
| Arc::clone(&config), |
| force_accept_9999_ebuilds, |
| ))); |
| |
| let resolver = |
| PackageResolver::new(Arc::clone(&repos), Arc::clone(&config), Arc::clone(&loader)); |
| |
| let toolchains = load_toolchains(&repos.get_repos())?; |
| |
| Ok(TargetData { |
| sysroot: root_dir.into(), |
| board: board.to_string(), |
| profile: profile_name.to_string(), |
| repos, |
| config, |
| loader, |
| resolver, |
| toolchains, |
| profile_path, |
| }) |
| } |
| |
| pub fn alchemist_main(args: Args) -> Result<()> { |
| if args.board.is_none() && !args.host { |
| bail!("Either --board or --host should be specified.") |
| } |
| if args.board.is_some() && args.host { |
| bail!("--board and --host shouldn't be specified together."); |
| } |
| |
| let source_dir = match args.source_dir { |
| Some(s) => PathBuf::from(s), |
| None => default_source_dir()?, |
| }; |
| let src_dir = source_dir.join("src"); |
| |
| let host_target = fakechroot::BoardTarget { |
| board: &args.host_board, |
| profile: &args.host_profile, |
| }; |
| |
| let board_target = if let Some(board) = args.board.as_ref() { |
| let profile = &args.profile; |
| |
| // We don't support a board ROOT with two different profiles. |
| if board == host_target.board && profile != host_target.profile { |
| bail!( |
| "--profile ({}) must match --host-profile ({})", |
| profile, |
| host_target.profile |
| ); |
| } |
| |
| Some(fakechroot::BoardTarget { board, profile }) |
| } else { |
| None |
| }; |
| |
| // Enter a fake chroot when running outside a cros chroot. |
| let translator = if args.use_portage_site_configs { |
| // TODO: What do we do here? |
| PathTranslator::noop() |
| } else { |
| let targets = if let Some(board_target) = board_target.as_ref() { |
| if board_target.board == host_target.board { |
| vec![&host_target] |
| } else { |
| vec![board_target, &host_target] |
| } |
| } else { |
| vec![&host_target] |
| }; |
| enter_fake_chroot(&targets, &source_dir)? |
| }; |
| |
| let tools_dir = setup_tools()?; |
| |
| let target_data = if let Some(board_target) = board_target { |
| let root_dir = Path::new("/build").join(board_target.board); |
| if is_inside_chroot()? && !root_dir.try_exists()? { |
| bail!( |
| "\n\ |
| *****\n\ |
| \t\tYou are running inside the CrOS SDK and `{}` doesn't exist.\n\ |
| \n\ |
| \t\tPlease run the following command to create the board's sysroot and try again:\n\ |
| \t\t$ setup_board --board {} --profile {}\n\ |
| \t\tWhen building public artifacts from an internal manifest, add --public.\n\ |
| \n\ |
| *****", |
| root_dir.display(), |
| board_target.board, |
| board_target.profile, |
| ); |
| } |
| |
| let repos = RepositorySet::load("board", &root_dir)?; |
| |
| Some((root_dir, repos, board_target)) |
| } else { |
| None |
| }; |
| |
| let host_data = { |
| let root_dir = Path::new("/build").join(host_target.board); |
| if is_inside_chroot()? && !root_dir.try_exists()? { |
| bail!( |
| "\n\ |
| *****\n\ |
| \t\tYou are running inside the CrOS SDK and `{}` doesn't exist.\n\ |
| \n\ |
| \t\tPlease run the following command to create the host sysroot and try again:\n\ |
| \t\t$ ~/chromiumos/chromite/shell/create_sdk_board_root \ |
| --board {} --profile {}\n\ |
| \n\ |
| *****", |
| root_dir.display(), |
| host_target.board, |
| host_target.profile, |
| ); |
| } |
| let repos = RepositorySet::load("host", &root_dir)?; |
| (root_dir, repos, host_target) |
| }; |
| |
| // We share an evaluator between both config ROOTS so we only have to parse |
| // the ebuilds once. |
| let evaluator = Arc::new(CachedEBuildEvaluator::new( |
| [target_data.as_ref().map(|x| &x.1), Some(&host_data.1)] |
| .into_iter() |
| .flatten() |
| .flat_map(|x| x.get_repos()) |
| .cloned() |
| .collect(), |
| tools_dir.path(), |
| )); |
| |
| let target = if let Some((root_dir, repos, board_target)) = target_data { |
| Some(load_board( |
| repos, |
| &evaluator, |
| board_target.board, |
| board_target.profile, |
| &root_dir, |
| args.use_flags.clone(), |
| args.use_portage_site_configs, |
| args.force_accept_9999_ebuilds, |
| )?) |
| } else { |
| None |
| }; |
| |
| #[allow(clippy::match_single_binding)] |
| let host = match host_data { |
| (root_dir, repos, host_target) => load_board( |
| repos, |
| &evaluator, |
| host_target.board, |
| host_target.profile, |
| &root_dir, |
| args.use_flags, |
| args.use_portage_site_configs, |
| args.force_accept_9999_ebuilds, |
| )?, |
| }; |
| |
| match args.command { |
| Commands::DumpPackage { args: local_args } => { |
| dump_package_main(&host, target.as_ref(), local_args)?; |
| } |
| Commands::DumpProfile { args: local_args } => { |
| dump_profile_main(&target.unwrap_or(host), local_args)?; |
| } |
| Commands::GenerateRepo { |
| output_dir, |
| output_repos_json, |
| } => { |
| generate_repo_main( |
| &host, |
| target.as_ref(), |
| &translator, |
| &src_dir, |
| &output_dir, |
| &output_repos_json, |
| )?; |
| } |
| Commands::DigestRepo { args: local_args } => { |
| digest_repo_main(&host, target.as_ref(), local_args)?; |
| } |
| } |
| |
| Ok(()) |
| } |