blob: 77f562d82a4a7451a018a428cccdc66c03159f13 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2022 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''Assembles a Rust toolchain in-tree linked against the LLVM revision
specified in //tools/clang/scripts/update.py.
Builds a Rust toolchain bootstrapped from an untrusted rustc build.
Rust has an official boostrapping build. At a high level:
1. A "stage 0" Rust is downloaded from Rust's official server. This
is one major release before the version being built. E.g. if building trunk
the latest beta is downloaded. If building stable 1.57.0, stage0 is stable
1.56.1.
2. Stage 0 libstd is built. This is different than the libstd downloaded above.
3. Stage 1 rustc is built with rustc from (1) and libstd from (2)
2. Stage 1 libstd is built with stage 1 rustc. Later artifacts built with
stage 1 rustc are built with stage 1 libstd.
Further stages are possible and continue as expected. Additionally, the build
can be extensively customized. See for details:
https://rustc-dev-guide.rust-lang.org/building/bootstrapping.html
This script clones the Rust repository, checks it out to a defined revision,
builds LLVM and Clang via //tools/clang/scripts/build.py, then builds rustc and
libstd against it. Clang is included in the build for libclang, which is needed
for building and shipping bindgen.
Ideally our build would begin with our own trusted stage0 rustc. As it is
simpler, for now we use an official build.
TODO(https://crbug.com/1245714): Do a proper 3-stage build
'''
import argparse
import base64
import collections
import hashlib
import json
import platform
import os
import re
import shutil
import string
import subprocess
import sys
import urllib
from pathlib import Path
# Get variables and helpers from Clang update script.
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'clang',
'scripts'))
from build import (AddCMakeToPath, AddZlibToPath, CheckoutGitRepo,
DownloadDebianSysroot, GetLibXml2Dirs, LLVM_BUILD_TOOLS_DIR,
RunCommand)
from update import (CHROMIUM_DIR, DownloadAndUnpack, EnsureDirExists,
GetDefaultHostOs, RmTree, UpdatePackage)
from update_rust import (RUST_REVISION, RUST_TOOLCHAIN_OUT_DIR,
STAGE0_JSON_SHA256, THIRD_PARTY_DIR, VERSION_SRC_PATH,
GetRustClangRevision)
EXCLUDED_TESTS = [
# https://github.com/rust-lang/rust/issues/45222 which appears to have
# regressed as of a recent LLVM update. This test is purely performance
# related, not correctness.
os.path.join('tests', 'codegen', 'issue-45222.rs'),
# https://github.com/rust-lang/rust/issues/96497
os.path.join('tests', 'codegen', 'issue-96497-slice-size-nowrap.rs'),
]
EXCLUDED_TESTS_WINDOWS = [
# https://github.com/rust-lang/rust/issues/96464
os.path.join('tests', 'codegen', 'vec-shrink-panik.rs'),
]
EXCLUDED_TESTS_MAC = [
# https://crbug.com/1479875 This fails on Mac. It relates to the large code
# model which we don't use, so suppress it for now.
os.path.join('tests', 'ui', 'thread-local', 'thread-local-issue-37508.rs'),
]
CLANG_SCRIPTS_DIR = os.path.join(CHROMIUM_DIR, 'tools', 'clang', 'scripts')
RUST_GIT_URL = ('https://chromium.googlesource.com/external/' +
'github.com/rust-lang/rust')
RUST_SRC_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-src')
RUST_BOOTSTRAP_DIST_RS = os.path.join(RUST_SRC_DIR, 'src', 'bootstrap',
'dist.rs')
STAGE0_JSON_PATH = os.path.join(RUST_SRC_DIR, 'src', 'stage0.json')
# Download crates.io dependencies to rust-src subdir (rather than $HOME/.cargo)
CARGO_HOME_DIR = os.path.join(RUST_SRC_DIR, 'cargo-home')
RUST_SRC_VERSION_FILE_PATH = os.path.join(RUST_SRC_DIR, 'src', 'version')
RUST_SRC_GIT_COMMIT_INFO_FILE_PATH = os.path.join(RUST_SRC_DIR,
'git-commit-info')
RUST_TOOLCHAIN_LIB_DIR = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'lib')
RUST_TOOLCHAIN_SRC_DIST_DIR = os.path.join(RUST_TOOLCHAIN_LIB_DIR, 'rustlib',
'src', 'rust')
RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR = os.path.join(RUST_TOOLCHAIN_SRC_DIST_DIR,
'vendor')
RUST_CONFIG_TEMPLATE_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'config.toml.template')
RUST_CARGO_CONFIG_TEMPLATE_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'cargo-config.toml.template')
RUST_SRC_VENDOR_DIR = os.path.join(RUST_SRC_DIR, 'vendor')
RUST_HOST_LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party',
'rust-toolchain-intermediate',
'llvm-host-build')
RUST_HOST_LLVM_INSTALL_DIR = os.path.join(CHROMIUM_DIR, 'third_party',
'rust-toolchain-intermediate',
'llvm-host-install')
RUST_CROSS_TARGET_LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party',
'rust-toolchain-intermediate',
'llvm-target-build')
RUST_CROSS_TARGET_LLVM_INSTALL_DIR = os.path.join(
CHROMIUM_DIR, 'third_party', 'rust-toolchain-intermediate',
'llvm-target-install')
# CIPD Versions from:
# - List all platforms
# cipd ls infra/3pp/static_libs/openssl/
# - Find all versions for a platform
# cipd instances infra/3pp/static_libs/openssl/linux-amd64
# - Find the version tag for a version
# cipd desc infra/3pp/static_libs/openssl/linux-amd64 --version <instance id>
# - A version tag looks like: `version:2@1.1.1j.chromium.2` and we
# store the part after the `2@` here.
CIPD_DOWNLOAD_URL = f'https://chrome-infra-packages.appspot.com/dl'
OPENSSL_CIPD_LINUX_AMD_PATH = 'infra/3pp/static_libs/openssl/linux-amd64'
OPENSSL_CIPD_LINUX_AMD_VERSION = '1.1.1j.chromium.2'
OPENSSL_CIPD_MAC_AMD_PATH = 'infra/3pp/static_libs/openssl/mac-amd64'
OPENSSL_CIPD_MAC_AMD_VERSION = '1.1.1j.chromium.2'
OPENSSL_CIPD_MAC_ARM_PATH = 'infra/3pp/static_libs/openssl/mac-amd64'
OPENSSL_CIPD_MAC_ARM_VERSION = '1.1.1j.chromium.2'
# TODO(crbug.com/1271215): Pull Windows OpenSSL from 3pp when it exists.
if sys.platform == 'win32':
LD_PATH_FLAG = '/LIBPATH:'
else:
LD_PATH_FLAG = '-L'
BUILD_TARGETS = [
'library/proc_macro', 'library/std', 'src/tools/cargo', 'src/tools/clippy',
'src/tools/rustfmt'
]
# Desired tools and libraries in our Rust toolchain.
DISTRIBUTION_ARTIFACTS = [
'cargo', 'clippy', 'compiler/rustc', 'library/std', 'rust-analyzer',
'rustfmt', 'src'
]
DISTRIBUTION_ARTIFACTS_SKIPPED_CROSS_COMPILE = [
# Cargo depends on OpenSSL and we don't have a cross-compile of the
# library available. This means gnrt will not function on M1 macs until
# we fix this, or stop depending on cross-compile.
'cargo',
# When cross-compiling this fails to build terribly, as it can't even
# find the Rust stdlib.
'rust-analyzer',
]
# Which test suites to run. Any failure will fail the build.
TEST_SUITES = [
'library/std',
'tests/codegen',
'tests/ui',
]
def AddOpenSSLToEnv(build_mac_arm):
"""Download OpenSSL, and add to OPENSSL_DIR."""
ssl_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'openssl')
if sys.platform == 'darwin':
if platform.machine() == 'arm64' or build_mac_arm:
ssl_url = (f'{CIPD_DOWNLOAD_URL}/{OPENSSL_CIPD_MAC_ARM_PATH}'
f'/+/version:2@{OPENSSL_CIPD_MAC_ARM_VERSION}')
else:
ssl_url = (f'{CIPD_DOWNLOAD_URL}/{OPENSSL_CIPD_MAC_AMD_PATH}'
f'/+/version:2@{OPENSSL_CIPD_MAC_AMD_VERSION}')
elif sys.platform == 'win32':
ssl_url = (f'{CIPD_DOWNLOAD_URL}/{OPENSSL_CIPD_WIN_AMD_PATH}'
f'/+/version:2@{OPENSSL_CIPD_WIN_AMD_VERSION}')
else:
ssl_url = (f'{CIPD_DOWNLOAD_URL}/{OPENSSL_CIPD_LINUX_AMD_PATH}'
f'/+/version:2@{OPENSSL_CIPD_LINUX_AMD_VERSION}')
if os.path.exists(ssl_dir):
RmTree(ssl_dir)
DownloadAndUnpack(ssl_url, ssl_dir, is_known_zip=True)
os.environ['OPENSSL_DIR'] = ssl_dir
return ssl_dir
def VerifyStage0JsonHash():
hasher = hashlib.sha256()
with open(STAGE0_JSON_PATH, 'rb') as input:
hasher.update(input.read())
actual_hash = hasher.hexdigest()
if actual_hash == STAGE0_JSON_SHA256:
return
print('src/stage0.json hash is different than expected!')
print('Expected hash: ' + STAGE0_JSON_SHA256)
print('Actual hash: ' + actual_hash)
sys.exit(1)
def FetchBetaPackage(name, rust_git_hash, triple=None):
'''Downloads the beta package specified for the compiler build
If `triple` is not specified, it downloads a package for the current
machine's architecture.
Unpacks the package and returns the path to root of the package.
'''
triple = triple if triple else RustTargetTriple()
filename = f'{name}-beta-{triple}'
# Pull the stage0 JSON to find the package intended to be used to
# build this version of the Rust compiler.
STAGE0_JSON_URL = (
'https://chromium.googlesource.com/external/github.com/'
'rust-lang/rust/+/{GIT_HASH}/src/stage0.json?format=TEXT')
base64_text = urllib.request.urlopen(
STAGE0_JSON_URL.format(GIT_HASH=rust_git_hash)).read().decode("utf-8")
stage0 = json.loads(base64.b64decode(base64_text))
# The stage0 JSON contains the path to all tarballs it uses binaries from.
for k in stage0['checksums_sha256'].keys():
if k.endswith(filename + '.tar.gz'):
package_tgz = k
server = stage0['config']['dist_server']
DownloadAndUnpack(f'{server}/{package_tgz}', LLVM_BUILD_TOOLS_DIR)
return os.path.join(LLVM_BUILD_TOOLS_DIR, filename)
def InstallBetaPackage(package_dir, install_dir):
args = [
f'--destdir={install_dir}',
f'--prefix=',
]
if sys.platform.startswith('linux'):
# Avoid warnings due to not running as root.
args += ['--disable-ldconfig']
RunCommand([os.path.join(package_dir, 'install.sh')] + args)
def CargoVendor(cargo_bin):
'''Runs `cargo vendor` to pull down dependencies.'''
os.chdir(RUST_SRC_DIR)
# Some Submodules are part of the workspace and need to exist (so we can
# read their Cargo.toml files) before we can vendor their deps.
submod_cmd = [
'git', 'submodule', 'update', '--init', '--recursive', '--depth', '1'
]
RunWithRetry(submod_cmd, 'git submodule')
vendor_env = os.environ
# The Cargo.toml files in the Rust toolchain may use nightly Cargo
# features, but the cargo binary is beta. This env var enables the
# beta cargo binary to allow nightly features anyway.
# https://github.com/rust-lang/rust/commit/2e52f4deb0544480b6aefe2c0cc1e6f3c893b081
vendor_env['RUSTC_BOOTSTRAP'] = '1'
vendor_cmd = [
cargo_bin,
'vendor',
'--locked',
'--versioned-dirs',
]
RunWithRetry(vendor_cmd, 'cargo vendor')
os.chdir(CHROMIUM_DIR)
def RunWithRetry(command, name):
'''Run a command, retrying a few times then aborting if it fails.'''
for i in range(0, 3):
if RunCommand(command, fail_hard=False):
return
elif i < 2:
print(f'failed {name}, retrying...')
else:
sys.exit(1)
class XPy:
''' Runner for x.py, Rust's build script. Holds shared state between x.py
runs. '''
def __init__(self, zlib_path, libxml2_dirs, build_mac_arm, debian_sysroot,
verbose):
self._env = collections.defaultdict(str, os.environ)
self._build_mac_arm = build_mac_arm
self._verbose = verbose
self._llvm_bins_path = os.path.join(RUST_HOST_LLVM_INSTALL_DIR, 'bin')
# We append to these flags, make sure they exist.
ENV_FLAGS = [
'CFLAGS',
'CXXFLAGS',
'LDFLAGS',
'RUSTFLAGS_BOOTSTRAP',
'RUSTFLAGS_NOT_BOOTSTRAP',
'RUSTDOCFLAGS',
]
self._env = collections.defaultdict(str, os.environ)
for f in ENV_FLAGS:
self._env.setdefault(f, '')
# The AR, CC, CXX flags control the C/C++ compiler used through the `cc`
# crate. There are also C/C++ targets that are part of the Rust
# toolchain build for which the tool is controlled from `config.toml`,
# so these must be duplicated there.
if sys.platform == 'win32':
self._env['AR'] = os.path.join(self._llvm_bins_path,
'llvm-lib.exe')
self._env['CC'] = os.path.join(self._llvm_bins_path,
'clang-cl.exe')
self._env['CXX'] = os.path.join(self._llvm_bins_path,
'clang-cl.exe')
self._env['LD'] = os.path.join(self._llvm_bins_path,
'lld-link.exe')
else:
self._env['AR'] = os.path.join(self._llvm_bins_path, 'llvm-ar')
self._env['CC'] = os.path.join(self._llvm_bins_path, 'clang')
self._env['CXX'] = os.path.join(self._llvm_bins_path, 'clang++')
self._env['LD'] = os.path.join(self._llvm_bins_path, 'clang')
if sys.platform == 'darwin':
# The system/xcode compiler would find system SDK correctly, but
# the Clang we've built does not. See
# https://github.com/llvm/llvm-project/issues/45225
sdk_path = subprocess.check_output(['xcrun', '--show-sdk-path'],
text=True).rstrip()
self._env['CFLAGS'] += f' -isysroot {sdk_path}'
self._env['CXXFLAGS'] += f' -isysroot {sdk_path}'
self._env['LDFLAGS'] += f' -isysroot {sdk_path}'
self._env['RUSTFLAGS_BOOTSTRAP'] += (
f' -Clink-arg=-isysroot -Clink-arg={sdk_path}')
self._env['RUSTFLAGS_NOT_BOOTSTRAP'] += (
f' -Clink-arg=-isysroot -Clink-arg={sdk_path}')
# Rust compiletests don't get any of the RUSTFLAGS that we set here
# and then the clang linker can't find `-lSystem`, unless we set the
# `SDKROOT`.
self._env['SDKROOT'] = sdk_path
if zlib_path:
self._env['CFLAGS'] += f' -I{zlib_path}'
self._env['CXXFLAGS'] += f' -I{zlib_path}'
self._env['LDFLAGS'] += f' {LD_PATH_FLAG}{zlib_path}'
self._env['RUSTFLAGS_BOOTSTRAP'] += (
f' -Clink-arg={LD_PATH_FLAG}{zlib_path}')
self._env['RUSTFLAGS_NOT_BOOTSTRAP'] += (
f' -Clink-arg={LD_PATH_FLAG}{zlib_path}')
if libxml2_dirs:
self._env['CFLAGS'] += f' -I{libxml2_dirs.include_dir}'
self._env['CXXFLAGS'] += f' -I{libxml2_dirs.include_dir}'
self._env['LDFLAGS'] += f' {LD_PATH_FLAG}{libxml2_dirs.lib_dir}'
self._env['RUSTFLAGS_BOOTSTRAP'] += (
f' -Clink-arg={LD_PATH_FLAG}{libxml2_dirs.lib_dir}')
self._env['RUSTFLAGS_NOT_BOOTSTRAP'] += (
f' -Clink-arg={LD_PATH_FLAG}{libxml2_dirs.lib_dir}')
if debian_sysroot:
# This mainly influences the glibc version that rustc itself needs.
sysroot_cflag = f'--sysroot={debian_sysroot}'
self._env['CFLAGS'] += f' {sysroot_cflag}'
self._env['CXXFLAGS'] += f' {sysroot_cflag}'
self._env['LDFLAGS'] += f' {sysroot_cflag}'
self._env['RUSTFLAGS'] += f' -Clink-arg={sysroot_cflag}'
# pkg-config will by default look for system-wide libs. This tells
# it to look exclusively in the sysroot instead.
self._env['PKG_CONFIG_SYSROOT_DIR'] = debian_sysroot
# Due to an interaction with the above flags, we must tell lzma-sys
# explicitly to build it from source.
self._env['LZMA_API_STATIC'] = '1'
# TODO(danakj): On windows we point the to lld-link in config.toml so
# we don't (and can't) specify -fuse-ld=lld, as lld-link doesn't know
# that argument, and it's already using lld. Should we do the same for
# other platforms and remove this link argument?
if sys.platform != 'win32':
# Direct rustc to use Chromium's lld instead of the system linker.
# This is critical for stage 1 onward since we need to link libs
# with LLVM bitcode. It is also good for a hermetic build in
# general.
self._env['RUSTFLAGS_BOOTSTRAP'] += ' -Clink-arg=-fuse-ld=lld'
self._env['RUSTFLAGS_NOT_BOOTSTRAP'] += ' -Clink-arg=-fuse-ld=lld'
# The `--undefined-version` flag is needed due to a bug in libtest:
# https://github.com/rust-lang/rust/issues/105967. The flag does
# not exist on Mac or Windows.
if sys.platform.startswith('linux'):
self._env['RUSTFLAGS_BOOTSTRAP'] += (
' -Clink-arg=-Wl,--undefined-version')
self._env['RUSTFLAGS_NOT_BOOTSTRAP'] += (
' -Clink-arg=-Wl,--undefined-version')
# Rustdoc should use our clang linker as well, as we pass flags that
# the system linker may not understand.
self._env['RUSTDOCFLAGS'] += f' -Clinker={self._env["LD"]}'
# Cargo normally stores files in $HOME. Override this.
self._env['CARGO_HOME'] = CARGO_HOME_DIR
# https://crbug.com/1441182
# Pretend we're a CI, since some path length reduction features are
# only enabled for CI. Otherwise some tests fail when the rust src
# directory name length is too long.
self._env['GITHUB_ACTIONS'] = 'true'
def configure(self, build_mac_arm, x86_64_llvm_config,
aarch64_llvm_config):
# Read the config.toml template file...
with open(RUST_CONFIG_TEMPLATE_PATH, 'r') as input:
template = string.Template(input.read())
def quote_string(s: str):
return s.replace('\\', '\\\\').replace('"', '\\"')
subs = {}
subs['INSTALL_DIR'] = quote_string(str(RUST_TOOLCHAIN_OUT_DIR))
subs['LLVM_BIN'] = quote_string(str(self._llvm_bins_path))
subs['PACKAGE_VERSION'] = GetRustClangRevision()
subs["LLVM_CONFIG_WINDOWS_x86_64"] = quote_string(
str(x86_64_llvm_config))
subs["LLVM_CONFIG_APPLE_AARCH64"] = quote_string(
str(aarch64_llvm_config))
subs["LLVM_CONFIG_APPLE_X86_64"] = quote_string(
str(x86_64_llvm_config))
subs["LLVM_CONFIG_LINUX_X86_64"] = quote_string(
str(x86_64_llvm_config))
# ...and apply substitutions, writing to config.toml in Rust tree.
with open(os.path.join(RUST_SRC_DIR, 'config.toml'), 'w') as output:
output.write(template.substitute(subs))
def run(self, sub, args):
''' Run x.py subcommand with specified args. '''
os.chdir(RUST_SRC_DIR)
cmd = [sys.executable, 'x.py', sub]
if self._verbose and self._verbose > 0:
cmd.append('-' + self._verbose * 'v')
RunCommand(cmd + args, setenv=True, env=self._env)
os.chdir(CHROMIUM_DIR)
def get_env(self):
''' The environment variables set for x.py invocations, as a dict. '''
return self._env
# Get arguments to run desired test suites, minus disabled tests.
def GetTestArgs():
args = TEST_SUITES
for excluded in EXCLUDED_TESTS:
args.append('--exclude')
args.append(excluded)
if sys.platform == 'win32':
for excluded in EXCLUDED_TESTS_WINDOWS:
args.append('--exclude')
args.append(excluded)
if sys.platform == 'darwin':
for excluded in EXCLUDED_TESTS_MAC:
args.append('--exclude')
args.append(excluded)
return args
def MakeVersionStamp(git_hash):
# We must generate a version stamp that contains the full version of the
# built Rust compiler:
# * The version number returned from `rustc --version`.
# * The git hash.
# * The chromium revision name of the compiler build, which includes the
# associated clang/llvm version.
with open(RUST_SRC_VERSION_FILE_PATH) as version_file:
rust_version = version_file.readline().rstrip()
return (f'rustc {rust_version} {git_hash}'
f' ({GetRustClangRevision()} chromium)\n')
def GetLatestRustCommit():
"""Get the latest commit hash in the LLVM monorepo."""
url = (
'https://chromium.googlesource.com/external/' +
'github.com/rust-lang/rust/+/refs/heads/master?format=JSON' # nocheck
)
main = json.loads(
urllib.request.urlopen(url).read().decode("utf-8").replace(")]}'", ""))
return main['commit']
def RustTargetTriple(build_mac_arm=False):
if sys.platform == 'darwin':
if platform.machine() == 'arm64' or build_mac_arm:
return 'aarch64-apple-darwin'
else:
return 'x86_64-apple-darwin'
elif sys.platform == 'win32':
return 'x86_64-pc-windows-msvc'
else:
return 'x86_64-unknown-linux-gnu'
# Fetch or build the LLVM libraries, for the host machine and when
# cross-compiling for the target machine.
#
# Returns the path to llvm-config for x86_64 and aarch64 targets.
def BuildLLVMLibraries(skip_build, build_mac_arm):
# LLVM libraries for the target machine. Usually the same as the host,
# unless we are cross-compiling.
if build_mac_arm:
target_llvm_build_dir = RUST_CROSS_TARGET_LLVM_BUILD_DIR
target_llvm_install_dir = RUST_CROSS_TARGET_LLVM_INSTALL_DIR
else:
target_llvm_build_dir = RUST_HOST_LLVM_BUILD_DIR
target_llvm_install_dir = RUST_HOST_LLVM_INSTALL_DIR
if not skip_build:
print(f'Building the host LLVM in {RUST_HOST_LLVM_BUILD_DIR}...')
build_cmd = [
sys.executable,
os.path.join(CLANG_SCRIPTS_DIR, 'build.py'),
'--disable-asserts',
'--no-tools',
# PIC needed for Rust build (links LLVM into shared object)
'--pic',
'--with-ml-inliner-model=',
# Not using this in Rust yet, see also crbug.com/1476464.
'--without-zstd',
]
if sys.platform.startswith('linux'):
build_cmd.append('--without-android')
build_cmd.append('--without-fuchsia')
if sys.platform == 'darwin':
build_cmd.append('--without-fuchsia')
RunCommand(build_cmd + [
'--build-dir', RUST_HOST_LLVM_BUILD_DIR, '--install-dir',
RUST_HOST_LLVM_INSTALL_DIR
])
# Build target compiler.
if build_mac_arm:
print(f'Building the target cross-compile LLVM in '
f'{target_llvm_build_dir}...')
build_cmd.append('--build-mac-arm')
if not skip_build:
RunCommand(build_cmd + [
'--build-dir', target_llvm_build_dir, '--install-dir',
target_llvm_install_dir
])
print(f'Copying the target-but-native llvm-config to LLVM '
f'install dir...')
shutil.copy(
os.path.join(target_llvm_install_dir, 'bin', 'llvm-config'),
os.path.join(target_llvm_install_dir, 'bin',
'llvm-config.bak'))
shutil.copy(
os.path.join(target_llvm_build_dir, 'NATIVE', 'bin',
'llvm-config'),
os.path.join(target_llvm_install_dir, 'bin', 'llvm-config'))
# Default to pointing all architectures at the host machine on the
# assumption we are not cross-compiling.
x86_64_llvm_config = os.path.join(RUST_HOST_LLVM_BUILD_DIR, 'bin',
'llvm-config')
aarch64_llvm_config = os.path.join(RUST_HOST_LLVM_BUILD_DIR, 'bin',
'llvm-config')
# Cross-compiling options follow.
if build_mac_arm:
# If we're building for aarch64 on an x86_64 machine, then we have a
# separate `target_llvm_install_dir` which has the aarch64 things.
# Inside a cross-compiled LLVM build, in the NATIVE directory, is an
# llvm-config that can run on the host machine.
aarch64_llvm_config = os.path.join(target_llvm_install_dir, 'bin',
'llvm-config')
return (x86_64_llvm_config, aarch64_llvm_config, target_llvm_install_dir)
def GitCherryPick(git_repository, git_remote, commit):
print(f'Cherry-picking {commit} in {git_repository} from {git_remote}')
RunCommand(
['git', '-C', git_repository, 'remote', 'add', 'github', git_remote],
fail_hard=False)
RunCommand(['git', '-C', git_repository, 'fetch', 'github', commit])
if RunCommand([
'git', '-C', git_repository, 'merge-base', '--is-ancestor', commit,
'HEAD'
],
fail_hard=False):
print('Commit already an ancestor; skipping.')
return
RunCommand(['git', '-C', git_repository, 'cherry-pick', commit])
def main():
parser = argparse.ArgumentParser(
description='Build and package Rust toolchain')
parser.add_argument('-v',
'--verbose',
action='count',
help='run subcommands with verbosity')
parser.add_argument('--build-mac-arm',
action='store_true',
help='Build arm binaries. Only valid on macOS.')
parser.add_argument(
'--verify-stage0-hash',
action='store_true',
help=
'checkout Rust, verify the stage0 hash, then quit without building. '
'Will print the actual hash if different than expected.')
parser.add_argument(
'--dump-env',
action='store_true',
help=
'dump all environment variables set for x.py to a file `rust-build-env`'
)
parser.add_argument('--skip-checkout',
action='store_true',
help='do not create or update any checkouts')
parser.add_argument('--sync-for-gnrt',
action='store_true',
help='sync checkout and deps for gnrt run then quit.')
parser.add_argument('--skip-clean',
action='store_true',
help='skip x.py clean step')
parser.add_argument(
'--skip-llvm-build',
action='store_true',
help='do not build LLVM, presuming build_rust.py was '
'already run and the LLVM libs are thus already present.')
parser.add_argument('--skip-test',
action='store_true',
help='skip running rustc and libstd tests')
parser.add_argument('--skip-install',
action='store_true',
help='do not install to RUST_TOOLCHAIN_OUT_DIR')
parser.add_argument('--rust-force-head-revision',
action='store_true',
help='build the latest revision')
parser.add_argument(
'--prepare-run-xpy',
action='store_true',
help='set up the build directory to use --run-xpy subsequently. For '
'debugging.')
parser.add_argument(
'--run-xpy',
action='store_true',
help='run x.py command in configured Rust checkout. Quits after '
'running specified command, skipping all normal build steps. For '
'debugging. Running x.py directly will not set the appropriate env '
'variables nor update config.toml')
args, rest = parser.parse_known_args()
if args.build_mac_arm and sys.platform != 'darwin':
print('--build-mac-arm only valid on macOS')
return 1
if args.build_mac_arm and platform.machine() == 'arm64':
print('--build-mac-arm only valid on intel to cross-build arm')
return 1
debian_sysroot = None
if sys.platform.startswith('linux') and not args.sync_for_gnrt:
# Fetch sysroot we build rustc against. This ensures a minimum supported
# host (not Chromium target). Since the rustc linux package is for
# x86_64 only, that is the sole needed sysroot.
debian_sysroot = DownloadDebianSysroot('amd64')
# Require zlib compression.
if sys.platform == 'win32':
zlib_path = AddZlibToPath()
else:
zlib_path = None
# libxml2 is built when building LLVM, so we use that.
if sys.platform == 'win32':
libxml2_dirs = GetLibXml2Dirs()
else:
libxml2_dirs = None
# TODO(crbug.com/1271215): OpenSSL is somehow already present on the Windows
# builder, but we should change to using a package from 3pp when it is
# available.
if (sys.platform != 'win32' and not args.build_mac_arm
and not args.sync_for_gnrt):
# Building cargo depends on OpenSSL.
AddOpenSSLToEnv(args.build_mac_arm)
xpy = XPy(zlib_path, libxml2_dirs, args.build_mac_arm, debian_sysroot,
args.verbose)
if args.dump_env:
with open('rust-build-env', 'w') as f:
for name, val in xpy.get_env().items():
print(f'{name}={val}', file=f)
# Assume the checkout has already been prepared. A full build or a
# --prepare-run-xpy run will set it up.
if args.run_xpy:
# Ensure the config.toml was previously generated.
config_path = os.path.join(RUST_SRC_DIR, 'config.toml')
assert os.path.exists(config_path)
assert os.path.isfile(config_path)
if rest[0] == '--':
rest = rest[1:]
xpy.run(rest[0], rest[1:])
return 0
else:
assert not rest
if sys.platform == 'win32':
# Use curl to prime Windows's root cert store (crbug.com/1448442).
RunCommand(['curl', '-I', 'https://static.rust-lang.org'])
if args.rust_force_head_revision:
assert not args.skip_checkout
checkout_revision = GetLatestRustCommit()
else:
checkout_revision = RUST_REVISION
if not args.skip_checkout:
CheckoutGitRepo('Rust', RUST_GIT_URL, checkout_revision, RUST_SRC_DIR)
# TODO(crbug.com/1493085): remove once
# https://github.com/rust-lang/rust/pull/116672 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'4f1e147849d89f4c2324fd569a48e6be66fae5d8')
# TODO: Remove once
# https://github.com/rust-lang/rust/pull/118410 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'81cd7c5b11766ed1e3214a2233371fb6d72ed89c')
# TODO: Remove once
# https://github.com/rust-lang/rust/pull/118610 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'b378059e6b2573c5356423fa31d184a89a3b6029')
# TODO: Remove once
# https://github.com/rust-lang/rust/pull/118818 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'a0c5079889b1f86dd9e246d8863a5c8b44fbdb78')
# TODO: Remove once
# https://github.com/rust-lang/rust/pull/118866 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'46a801559127441675f2341bd1d684809a47def1')
# TODO: Remove once
# https://github.com/rust-lang/rust/pull/118941 has been merged.
GitCherryPick(RUST_SRC_DIR, 'https://github.com/rust-lang/rust.git',
'0a285e8de7eec36e1de68c83764d23f2522a4274')
path = FetchBetaPackage('cargo', checkout_revision)
if sys.platform == 'win32':
cargo_bin = os.path.join(path, 'cargo', 'bin', 'cargo.exe')
else:
cargo_bin = os.path.join(path, 'cargo', 'bin', 'cargo')
CargoVendor(cargo_bin)
# Gnrt needs the checkout to be up-to-date, workspace submodules to be
# synced for cargo to work, and the cargo binary itself. All this is done,
# so quit.
if args.sync_for_gnrt:
return 0
VerifyStage0JsonHash()
if args.verify_stage0_hash:
# The above function exits and prints the actual hash if
# verification failed so we just quit here; if we reach this point,
# the hash is valid.
return 0
(x86_64_llvm_config, aarch64_llvm_config,
target_llvm_dir) = BuildLLVMLibraries(args.skip_llvm_build,
args.build_mac_arm)
AddCMakeToPath()
# Set up config.toml in Rust source tree.
xpy.configure(args.build_mac_arm, x86_64_llvm_config, aarch64_llvm_config)
# Deps are updated, so we're done now. All steps needed for --run-xpy to
# work should be above this.
if args.prepare_run_xpy:
return 0
building_on_host_triple = RustTargetTriple()
xpy_args = ['--build', building_on_host_triple]
if args.build_mac_arm:
building_for_host_triple = RustTargetTriple(build_mac_arm=True)
xpy_args.extend([
# The compiler will run on ARM.
'--host',
building_for_host_triple,
# The compiler will build stuff for ARM.
'--target',
building_for_host_triple
])
if not args.skip_clean:
print('Cleaning build artifacts...')
xpy.run('clean', xpy_args)
# When cross-compiling, the bootstrap does not reuse previous artifacts,
# so by splitting the build up into steps, we end up building the compiler
# multiple times. As such, when cross-compiling, we skip right to the
# install step, which will build what is needed.
if not args.build_mac_arm and not args.skip_install:
if not args.skip_test and not args.build_mac_arm:
print(f'Running stage 2 tests...')
xpy.run('test', ['--stage', '2'] + xpy_args + GetTestArgs())
# Build stage 2 compiler, tools, and libraries. This should reuse
# earlier stages from the test command (if run).
print('Building stage 2 artifacts...')
xpy.run('build', xpy_args + ['--stage', '2'] + BUILD_TARGETS)
if args.skip_install:
# Rust is fully built. We can quit.
return 0
print(f'Installing Rust to {RUST_TOOLCHAIN_OUT_DIR} ...')
# Clean output directory.
if os.path.exists(RUST_TOOLCHAIN_OUT_DIR):
RmTree(RUST_TOOLCHAIN_OUT_DIR)
artifacts = DISTRIBUTION_ARTIFACTS
if args.build_mac_arm:
for a in DISTRIBUTION_ARTIFACTS_SKIPPED_CROSS_COMPILE:
artifacts.remove(a)
# TODO(crbug.com/1504532): remove once
# https://github.com/rust-lang/rust/blob/master/.gitmodules#L33-L37 has
# been updated to include
# https://github.com/llvm/llvm-project/commit/7939ce39dac0078fef7183d6198598b99c652c88
rust_llvm_dir = os.path.join(RUST_SRC_DIR, 'src', 'llvm-project')
GitCherryPick(rust_llvm_dir, 'https://github.com/llvm/llvm-project.git',
'7939ce39dac0078fef7183d6198598b99c652c88')
# Remove .git in llvm so x.py install will not sync it.
if (os.path.exists(os.path.join(rust_llvm_dir, '.git'))):
os.remove(os.path.join(rust_llvm_dir, '.git'))
xpy.run('install', xpy_args + artifacts)
# Copy additional vendored crates required for building stdlib.
print(f'Copying vendored dependencies to {RUST_TOOLCHAIN_OUT_DIR} ...')
shutil.copytree(RUST_SRC_VENDOR_DIR, RUST_TOOLCHAIN_SRC_DIST_VENDOR_DIR)
with open(VERSION_SRC_PATH, 'w') as stamp:
stamp.write(MakeVersionStamp(checkout_revision))
return 0
if __name__ == '__main__':
sys.exit(main())