| #!/usr/bin/env python3 | |
| # Copyright 2023 The Chromium Authors | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """ | |
| This script is used to build Debian sysroot images for building Chromium. | |
| """ | |
| import argparse | |
| import collections | |
| import hashlib | |
| import lzma | |
| import os | |
| import re | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| import time | |
| import requests | |
| import reversion_glibc | |
| DISTRO = "debian" | |
| RELEASES = { | |
| "amd64": "bullseye", | |
| "i386": "bullseye", | |
| "armhf": "bullseye", | |
| "arm64": "bullseye", | |
| "mipsel": "bullseye", | |
| "mips64el": "bullseye", | |
| "ppc64el": "bullseye", | |
| "riscv64": "trixie", | |
| "s390x": "bullseye", | |
| } | |
| GCC_VERSIONS = { | |
| "bullseye": 10, | |
| "trixie": 12, | |
| } | |
| # This number is appended to the sysroot key to cause full rebuilds. It | |
| # should be incremented when removing packages or patching existing packages. | |
| # It should not be incremented when adding packages. | |
| SYSROOT_RELEASE = 3 | |
| SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| CHROME_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) | |
| # gpg keyring file generated using generate_keyring.sh | |
| KEYRING_FILE = os.path.join(SCRIPT_DIR, "keyring.gpg") | |
| ARCHIVE_TIMESTAMP = "20250129T203412Z" | |
| ARCHIVE_URL = f"https://snapshot.debian.org/archive/debian/{ARCHIVE_TIMESTAMP}/" | |
| APT_SOURCES_LIST = [ | |
| # Debian 12 (Bookworm) is needed for GTK4. It should be kept before | |
| # bullseye so that bullseye takes precedence. | |
| ("bookworm", ["main"]), | |
| ("bookworm-updates", ["main"]), | |
| # This mimics a sources.list from bullseye. | |
| ("bullseye", ["main", "contrib", "non-free"]), | |
| ("bullseye-updates", ["main", "contrib", "non-free"]), | |
| ("bullseye-backports", ["main", "contrib", "non-free"]), | |
| ] | |
| APT_SOURCES_LIST_RISCV = [("trixie", ["main", "contrib"])] | |
| APT_SOURCES_LISTS = { | |
| "amd64": APT_SOURCES_LIST, | |
| "i386": APT_SOURCES_LIST, | |
| "armhf": APT_SOURCES_LIST, | |
| "arm64": APT_SOURCES_LIST, | |
| "mipsel": APT_SOURCES_LIST, | |
| "mips64el": APT_SOURCES_LIST, | |
| "ppc64el": APT_SOURCES_LIST, | |
| "riscv64": APT_SOURCES_LIST_RISCV, | |
| "s390x": APT_SOURCES_LIST, | |
| } | |
| TRIPLES = { | |
| "amd64": "x86_64-linux-gnu", | |
| "i386": "i386-linux-gnu", | |
| "armhf": "arm-linux-gnueabihf", | |
| "arm64": "aarch64-linux-gnu", | |
| "mipsel": "mipsel-linux-gnu", | |
| "mips64el": "mips64el-linux-gnuabi64", | |
| "ppc64el": "powerpc64le-linux-gnu", | |
| "riscv64": "riscv64-linux-gnu", | |
| "s390x": "s390x-linux-gnu", | |
| } | |
| LIB_DIRS = { | |
| "bullseye": "lib", | |
| "trixie": "usr/lib", | |
| } | |
| REQUIRED_TOOLS = [ | |
| "dpkg-deb", | |
| "file", | |
| "gpgv", | |
| "readelf", | |
| "tar", | |
| "xz", | |
| ] | |
| # Package configuration | |
| PACKAGES_EXT = "xz" | |
| RELEASE_FILE = "Release" | |
| RELEASE_FILE_GPG = "Release.gpg" | |
| # List of development packages. Dependencies are automatically included. | |
| DEBIAN_PACKAGES = [ | |
| "libasound2-dev", | |
| "libbluetooth-dev", | |
| "libc6-dev", | |
| "libcap-dev", | |
| "libcolord-dev", | |
| "libcups2-dev", | |
| "libcupsimage2-dev", | |
| "libcurl4-gnutls-dev", | |
| "libdbusmenu-glib-dev", | |
| "libdeflate-dev", | |
| "libelf-dev", | |
| "libflac-dev", | |
| "libgbm-dev", | |
| "libgcrypt20-dev", | |
| "libgnutls28-dev", | |
| "libgtk-3-dev", | |
| "libgtk-4-dev", | |
| "libinput-dev", | |
| "libjbig-dev", | |
| "libjpeg-dev", | |
| "libjsoncpp-dev", | |
| "libkrb5-dev", | |
| "liblcms2-dev", | |
| "liblzma-dev", | |
| "libminizip-dev", | |
| "libmtdev-dev", | |
| "libncurses-dev", | |
| "libnss3-dev", | |
| "libopus-dev", | |
| "libpam0g-dev", | |
| "libpci-dev", | |
| "libpipewire-0.3-dev", | |
| "libpulse-dev", | |
| "libre2-dev", | |
| "libsnappy-dev", | |
| "libspeechd-dev", | |
| "libssl-dev", | |
| "libsystemd-dev", | |
| "libtiff-dev", | |
| "libutempter-dev", | |
| "libva-dev", | |
| "libwayland-egl-backend-dev", | |
| "libwebp-dev", | |
| "libx11-xcb-dev", | |
| "libxcb-dri2-0-dev", | |
| "libxcb-dri3-dev", | |
| "libxcb-glx0-dev", | |
| "libxcb-image0-dev", | |
| "libxcb-present-dev", | |
| "libxcb-render-util0-dev", | |
| "libxcb-util-dev", | |
| "libxkbregistry-dev", | |
| "libxshmfence-dev", | |
| "libxslt1-dev", | |
| "libxss-dev", | |
| "libxt-dev", | |
| "libxxf86vm-dev", | |
| "mesa-common-dev", | |
| "qt6-base-dev", | |
| "qtbase5-dev", | |
| "valgrind-if-available", | |
| ] | |
| def banner(message: str) -> None: | |
| print("#" * 70) | |
| print(message) | |
| print("#" * 70) | |
| def sub_banner(message: str) -> None: | |
| print("-" * 70) | |
| print(message) | |
| print("-" * 70) | |
| def hash_file(hasher, file_name: str) -> str: | |
| with open(file_name, "rb") as f: | |
| while chunk := f.read(8192): | |
| hasher.update(chunk) | |
| return hasher.hexdigest() | |
| def atomic_copyfile(source: str, destination: str) -> None: | |
| dest_dir = os.path.dirname(destination) | |
| with tempfile.NamedTemporaryFile( | |
| mode="wb", delete=False, dir=dest_dir | |
| ) as temp_file: | |
| temp_filename = temp_file.name | |
| shutil.copyfile(source, temp_filename) | |
| os.rename(temp_filename, destination) | |
| def download_or_copy_non_unique_filename(url: str, dest: str) -> None: | |
| """ | |
| Downloads a file from a given URL to a destination with a unique filename, | |
| based on the SHA-256 hash of the URL. | |
| """ | |
| hash_digest = hashlib.sha256(url.encode()).hexdigest() | |
| unique_dest = f"{dest}.{hash_digest}" | |
| download_or_copy(url, unique_dest) | |
| atomic_copyfile(unique_dest, dest) | |
| def download_or_copy(source: str, destination: str) -> None: | |
| """ | |
| Downloads a file from the given URL or copies it from a local path to the | |
| specified destination. | |
| """ | |
| if os.path.exists(destination): | |
| print(f"{destination} already in place") | |
| return | |
| if source.startswith(("http://", "https://")): | |
| download_file(source, destination) | |
| else: | |
| atomic_copyfile(source, destination) | |
| def download_file(url: str, dest: str, retries=5) -> None: | |
| """ | |
| Downloads a file from a URL to a specified destination with retry logic, | |
| directory creation, and atomic write. | |
| """ | |
| print(f"Downloading from {url} -> {dest}") | |
| # Create directories if they don't exist | |
| os.makedirs(os.path.dirname(dest), exist_ok=True) | |
| for attempt in range(retries): | |
| try: | |
| with requests.get(url, stream=True) as response: | |
| response.raise_for_status() | |
| # Use a temporary file to write data | |
| with tempfile.NamedTemporaryFile( | |
| mode="wb", delete=False, dir=os.path.dirname(dest) | |
| ) as temp_file: | |
| for chunk in response.iter_content(chunk_size=8192): | |
| temp_file.write(chunk) | |
| # Rename temporary file to destination file | |
| os.rename(temp_file.name, dest) | |
| print(f"Downloaded {dest}") | |
| break | |
| except requests.RequestException as e: | |
| print(f"Attempt {attempt} failed: {e}") | |
| # Exponential back-off | |
| time.sleep(2**attempt) | |
| else: | |
| raise Exception(f"Failed to download file after {retries} attempts") | |
| def get_build_dir(arch: str) -> str: | |
| return os.path.join(CHROME_DIR, "out", "sysroot-build", RELEASES[arch]) | |
| def sanity_check(build_dir: str) -> None: | |
| """ | |
| Performs sanity checks to ensure the environment is correctly set up. | |
| """ | |
| banner("Sanity Checks") | |
| # Determine the Chrome build directory | |
| os.makedirs(build_dir, exist_ok=True) | |
| print(f"Using build directory: {build_dir}") | |
| # Check for required tools | |
| missing = [tool for tool in REQUIRED_TOOLS if not shutil.which(tool)] | |
| if missing: | |
| raise Exception(f"Required tools not found: {', '.join(missing)}") | |
| def clear_install_dir(install_root: str) -> None: | |
| if os.path.exists(install_root): | |
| shutil.rmtree(install_root) | |
| os.makedirs(install_root) | |
| def create_tarball(install_root: str, arch: str, build_dir: str) -> None: | |
| tarball_path = os.path.join( | |
| build_dir, f"{DISTRO}_{RELEASES[arch]}_{arch}_sysroot.tar.xz" | |
| ) | |
| banner("Creating tarball " + tarball_path) | |
| command = [ | |
| "tar", | |
| "--owner=0", | |
| "--group=0", | |
| "--numeric-owner", | |
| "--sort=name", | |
| "--no-xattrs", | |
| "-I", | |
| "xz -z9 -T0 --lzma2='dict=256MiB'", | |
| "-cf", | |
| tarball_path, | |
| "-C", | |
| install_root, | |
| ".", | |
| ] | |
| subprocess.run(command, check=True) | |
| def generate_package_list_dist_repo( | |
| arch: str, dist: str, repo_name: str, build_dir: str | |
| ) -> list[dict[str, str]]: | |
| repo_basedir = f"{ARCHIVE_URL}/dists/{dist}" | |
| package_list = f"{build_dir}/Packages.{dist}_{repo_name}_{arch}" | |
| package_list = f"{package_list}.{PACKAGES_EXT}" | |
| package_file_arch = f"{repo_name}/binary-{arch}/Packages.{PACKAGES_EXT}" | |
| package_list_arch = f"{repo_basedir}/{package_file_arch}" | |
| download_or_copy_non_unique_filename(package_list_arch, package_list) | |
| verify_package_listing(package_file_arch, package_list, dist, build_dir) | |
| with lzma.open(package_list, "rt") as src: | |
| return [ | |
| dict( | |
| line.split(": ", 1) | |
| for line in package_meta.splitlines() | |
| if not line.startswith(" ") | |
| ) | |
| for package_meta in src.read().split("\n\n") | |
| if package_meta | |
| ] | |
| def generate_package_list(arch: str, build_dir: str) -> dict[str, str]: | |
| # Workaround for some misconfigured package dependencies. | |
| BROKEN_DEPS = { | |
| "libgcc1", | |
| "qt6-base-abi", | |
| "libc-dev", # pulls in a newer libc6-dev | |
| } | |
| package_meta = {} | |
| sources = APT_SOURCES_LISTS[arch] | |
| for dist, repos in sources: | |
| for repo_name in repos: | |
| for meta in generate_package_list_dist_repo( | |
| arch, dist, repo_name, build_dir | |
| ): | |
| package_meta[meta["Package"]] = meta | |
| if "Provides" not in meta: | |
| continue | |
| for provides in meta["Provides"].split(", "): | |
| # Strip version requirements | |
| provides = provides.split()[0] | |
| if provides in package_meta: | |
| continue | |
| package_meta[provides] = meta | |
| def add_package_dependencies(package: str) -> None: | |
| if package in BROKEN_DEPS: | |
| return | |
| meta = package_meta[package] | |
| url = ARCHIVE_URL + meta["Filename"] | |
| if url in package_dict: | |
| return | |
| package_dict[url] = meta["SHA256"] | |
| if "Depends" in meta: | |
| for dep in meta["Depends"].split(", "): | |
| add_package_dependencies(dep.split()[0].split(":")[0]) | |
| # Read the input file and create a dictionary mapping package names to URLs | |
| # and checksums. | |
| missing = set(DEBIAN_PACKAGES) | |
| # Add corresponding libstdc++-dev package (needed for trixie) | |
| missing.add(f"libstdc++-{GCC_VERSIONS[RELEASES[arch]]}-dev") | |
| package_dict: dict[str, str] = {} | |
| for package in package_meta: | |
| if package in missing: | |
| missing.remove(package) | |
| add_package_dependencies(package) | |
| if missing: | |
| raise Exception(f"Missing packages: {', '.join(missing)}") | |
| # Write the URLs and checksums of the requested packages to the output file | |
| output_file = os.path.join( | |
| SCRIPT_DIR, "generated_package_lists", f"{RELEASES[arch]}.{arch}" | |
| ) | |
| with open(output_file, "w") as f: | |
| f.write("\n".join(sorted(package_dict)) + "\n") | |
| return package_dict | |
| def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None: | |
| banner("Misc Hacks & Patches") | |
| debian_dir = os.path.join(install_root, "debian") | |
| control_file = os.path.join(debian_dir, "control") | |
| # Create an empty control file | |
| open(control_file, "a").close() | |
| # Remove an unnecessary dependency on qtchooser. | |
| qtchooser_conf = os.path.join( | |
| install_root, | |
| "usr", | |
| "lib", | |
| TRIPLES[arch], | |
| "qt-default/qtchooser/default.conf", | |
| ) | |
| if os.path.exists(qtchooser_conf): | |
| os.remove(qtchooser_conf) | |
| # libxcomposite1 is missing a symbols file. | |
| atomic_copyfile( | |
| os.path.join(script_dir, "libxcomposite1-symbols"), | |
| os.path.join( | |
| install_root, "debian", "libxcomposite1", "DEBIAN", "symbols" | |
| ), | |
| ) | |
| # __GLIBC_MINOR__ is used as a feature test macro. Replace it with the | |
| # earliest supported version of glibc (2.26). | |
| features_h = os.path.join(install_root, "usr", "include", "features.h") | |
| replace_in_file(features_h, r"(#define\s+__GLIBC_MINOR__)", r"\1 26 //") | |
| # glibc >= 2.33 marks mallinfo as deprecated, which fails warning-as-error | |
| # build as mallinfo is still used by process_metrics_posix.cc and partition | |
| # allocator unit tests. Remove the attribute from mallinfo only. | |
| malloc_h = os.path.join(install_root, "usr", "include", "malloc.h") | |
| replace_in_file( | |
| malloc_h, | |
| r"(extern\s+struct\s+mallinfo\s+mallinfo\s+\(void\)\s+__THROW) " | |
| r"__MALLOC_DEPRECATED;", | |
| r"\1;", | |
| ) | |
| # C23 STRTOL requires glibc >= 2.38 | |
| replace_in_file( | |
| features_h, r"(#\s?define\s+__GLIBC_USE_C23_STRTOL)", r"\1 0 //" | |
| ) | |
| # riscv_hwprobe requires glibc >= 2.40 | |
| if arch == "riscv64": | |
| os.remove( | |
| os.path.join( | |
| install_root, | |
| "usr", | |
| "include", | |
| "riscv64-linux-gnu", | |
| "sys", | |
| "hwprobe.h", | |
| ) | |
| ) | |
| # fcntl64() was introduced in glibc 2.28. Make sure to use fcntl() instead. | |
| fcntl_h = os.path.join(install_root, "usr", "include", "fcntl.h") | |
| replace_in_file( | |
| fcntl_h, | |
| r"#ifndef __USE_FILE_OFFSET64(\nextern int fcntl)", | |
| r"#if 1\1", | |
| ) | |
| # Do not use pthread_cond_clockwait as it was introduced in glibc 2.30. | |
| cppconfig_h = os.path.join( | |
| install_root, | |
| "usr", | |
| "include", | |
| TRIPLES[arch], | |
| "c++", | |
| str(GCC_VERSIONS[RELEASES[arch]]), | |
| "bits", | |
| "c++config.h", | |
| ) | |
| replace_in_file( | |
| cppconfig_h, | |
| r"(#define\s+_GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT)", | |
| r"// \1", | |
| ) | |
| # Include limits.h in stdlib.h to fix an ODR issue. | |
| stdlib_h = os.path.join(install_root, "usr", "include", "stdlib.h") | |
| replace_in_file( | |
| stdlib_h, r"(#include <stddef.h>)", r"\1\n#include <limits.h>" | |
| ) | |
| # Glibc < 2.34 (like in our Bullseye sysroot) doesn't support | |
| # _FORTIFY_SOURCE=3. We can "upgrade" it by redefining the internal macros | |
| # used by the fortified headers to use __builtin_dynamic_object_size instead | |
| # of __builtin_object_size. This can be removed when the sysroot is upgraded | |
| # from bullseye to bookworm. | |
| # | |
| # First, allow __USE_FORTIFY_LEVEL to be 3. | |
| replace_in_file( | |
| features_h, | |
| r"(#\s?if\s+_FORTIFY_SOURCE\s?>\s?1)", | |
| r"# if _FORTIFY_SOURCE > 2\n" | |
| r"# define __USE_FORTIFY_LEVEL 3\n" | |
| r"# elif _FORTIFY_SOURCE > 1", | |
| ) | |
| # Second, redefine __bos and __bos0 to use __builtin_dynamic_object_size | |
| # when __USE_FORTIFY_LEVEL is 3. | |
| cdefs_h = os.path.join( | |
| install_root, "usr", "include", TRIPLES[arch], "sys", "cdefs.h" | |
| ) | |
| replace_in_file( | |
| cdefs_h, | |
| r"(#define\s+__bos\(ptr\)\s+__builtin_object_size\s+" | |
| r"\(ptr,\s+__USE_FORTIFY_LEVEL\s+>\s+1\))", | |
| r"#if defined(__clang__) && defined(__USE_FORTIFY_LEVEL) && " | |
| r"__USE_FORTIFY_LEVEL > 2\n" | |
| r"# define __bos(ptr) __builtin_dynamic_object_size (ptr, 1)\n" | |
| r"# define __bos0(ptr) __builtin_dynamic_object_size (ptr, 0)\n" | |
| r"#else\n" | |
| r"\1", | |
| ) | |
| replace_in_file( | |
| cdefs_h, | |
| r"(#define\s+__bos0\(ptr\)\s+__builtin_object_size\s+\(ptr,\s+0\))", | |
| r"\1\n#endif", | |
| ) | |
| # Move pkgconfig scripts. | |
| pkgconfig_dir = os.path.join(install_root, "usr", "lib", "pkgconfig") | |
| os.makedirs(pkgconfig_dir, exist_ok=True) | |
| triple_pkgconfig_dir = os.path.join( | |
| install_root, "usr", "lib", TRIPLES[arch], "pkgconfig" | |
| ) | |
| if os.path.exists(triple_pkgconfig_dir): | |
| for file in os.listdir(triple_pkgconfig_dir): | |
| shutil.move(os.path.join(triple_pkgconfig_dir, file), pkgconfig_dir) | |
| # Avoid requiring unsupported glibc versions. | |
| for lib in ["libc.so.6", "libm.so.6", "libcrypt.so.1"]: | |
| lib_path = os.path.join(install_root, "lib", TRIPLES[arch], lib) | |
| reversion_glibc.reversion_glibc(lib_path, arch) | |
| # GTK4 is provided by bookworm (12), but pango is provided by bullseye | |
| # (11). Fix the GTK4 pkgconfig file to relax the pango version | |
| # requirement. | |
| gtk4_pc = os.path.join(pkgconfig_dir, "gtk4.pc") | |
| replace_in_file(gtk4_pc, r"pango [>=0-9. ]*", "pango") | |
| replace_in_file(gtk4_pc, r"pangocairo [>=0-9. ]*", "pangocairo") | |
| # Remove a cyclic symlink: /usr/bin/X11 -> /usr/bin | |
| os.remove(os.path.join(install_root, "usr/bin/X11")) | |
| def create_extra_symlinks(install_root: str, arch: str): | |
| if RELEASES[arch] != "bullseye": | |
| # Recent debian releases no longer symlink lib{dl,pthread,rt,util}.so | |
| # because they are integrated into libc.so.6 itself | |
| for lib in [ | |
| "libdl.so.2", | |
| "librt.so.1", | |
| "libpthread.so.0", | |
| "libutil.so.1", | |
| ]: | |
| os.symlink( | |
| lib, | |
| os.path.join( | |
| install_root, "lib", TRIPLES[arch], lib.rpartition(".")[0] | |
| ), | |
| ) | |
| def replace_in_file( | |
| file_path: str, search_pattern: str, replace_pattern: str | |
| ) -> None: | |
| with open(file_path, "r") as file: | |
| content = file.read() | |
| with open(file_path, "w") as file: | |
| file.write(re.sub(search_pattern, replace_pattern, content)) | |
| def install_into_sysroot( | |
| build_dir: str, install_root: str, packages: dict[str, str] | |
| ) -> None: | |
| """ | |
| Installs libraries and headers into the sysroot environment. | |
| """ | |
| banner("Install Libs And Headers Into Jail") | |
| debian_packages_dir = os.path.join(build_dir, "debian-packages") | |
| os.makedirs(debian_packages_dir, exist_ok=True) | |
| debian_dir = os.path.join(install_root, "debian") | |
| os.makedirs(debian_dir, exist_ok=True) | |
| for package, sha256sum in packages.items(): | |
| package_name = os.path.basename(package) | |
| package_path = os.path.join(debian_packages_dir, package_name) | |
| banner(f"Installing {package_name}") | |
| download_or_copy(package, package_path) | |
| if hash_file(hashlib.sha256(), package_path) != sha256sum: | |
| raise ValueError(f"SHA256 mismatch for {package_path}") | |
| sub_banner(f"Extracting to {install_root}") | |
| subprocess.run( | |
| ["dpkg-deb", "-x", package_path, install_root], check=True | |
| ) | |
| base_package = get_base_package_name(package_path) | |
| debian_package_dir = os.path.join(debian_dir, base_package, "DEBIAN") | |
| # Extract the control file | |
| os.makedirs(debian_package_dir, exist_ok=True) | |
| with subprocess.Popen( | |
| ["dpkg-deb", "-e", package_path, debian_package_dir], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| ) as proc: | |
| _, err = proc.communicate() | |
| if proc.returncode != 0: | |
| message = "Failed to extract control from" | |
| raise Exception( | |
| f"{message} {package_path}: {err.decode('utf-8')}" | |
| ) | |
| # Prune /usr/share, leaving only allowlisted directories. | |
| USR_SHARE_ALLOWLIST = { | |
| "fontconfig", | |
| "pkgconfig", | |
| "wayland", | |
| "wayland-protocols", | |
| } | |
| usr_share = os.path.join(install_root, "usr", "share") | |
| for item in os.listdir(usr_share): | |
| full_path = os.path.join(usr_share, item) | |
| if os.path.isdir(full_path) and item not in USR_SHARE_ALLOWLIST: | |
| shutil.rmtree(full_path) | |
| def get_base_package_name(package_path: str) -> str: | |
| """ | |
| Retrieves the base package name from a Debian package. | |
| """ | |
| result = subprocess.run( | |
| ["dpkg-deb", "--field", package_path, "Package"], | |
| capture_output=True, | |
| text=True, | |
| ) | |
| if result.returncode != 0: | |
| raise Exception( | |
| f"Failed to get package name from {package_path}: {result.stderr}" | |
| ) | |
| return result.stdout.strip() | |
| def cleanup_jail_symlinks(install_root: str) -> None: | |
| """ | |
| Cleans up jail symbolic links by converting absolute symlinks | |
| into relative ones. | |
| """ | |
| for root, dirs, files in os.walk(install_root): | |
| for name in files + dirs: | |
| full_path = os.path.join(root, name) | |
| if os.path.islink(full_path): | |
| target_path = os.readlink(full_path) | |
| if target_path == "/dev/null": | |
| # Some systemd services get masked by symlinking them to | |
| # /dev/null. It's safe to remove these. | |
| os.remove(full_path) | |
| continue | |
| # If the link's target does not exist, remove this broken link. | |
| if os.path.isabs(target_path): | |
| absolute_target = os.path.join( | |
| install_root, target_path.strip("/") | |
| ) | |
| else: | |
| absolute_target = os.path.join( | |
| os.path.dirname(full_path), target_path | |
| ) | |
| if not os.path.exists(absolute_target): | |
| os.remove(full_path) | |
| continue | |
| if os.path.isabs(target_path): | |
| # Compute the relative path from the symlink to the target. | |
| relative_path = os.path.relpath( | |
| os.path.join(install_root, target_path.strip("/")), | |
| os.path.dirname(full_path), | |
| ) | |
| # Verify that the target exists inside the install_root. | |
| joined_path = os.path.join( | |
| os.path.dirname(full_path), relative_path | |
| ) | |
| if not os.path.exists(joined_path): | |
| raise Exception( | |
| f"Link target doesn't exist: {joined_path}" | |
| ) | |
| os.remove(full_path) | |
| os.symlink(relative_path, full_path) | |
| def removing_unnecessary_files(install_root, arch): | |
| """ | |
| Minimizes the sysroot by removing unnecessary files. | |
| """ | |
| # Preserve these files. | |
| gcc_triple = "i686-linux-gnu" if arch == "i386" else TRIPLES[arch] | |
| gcc_version = GCC_VERSIONS[RELEASES[arch]] | |
| ALLOWLIST = { | |
| "usr/bin/cups-config", | |
| f"usr/lib/gcc/{gcc_triple}/{gcc_version}/libgcc.a", | |
| f"usr/lib/{TRIPLES[arch]}/libc_nonshared.a", | |
| f"usr/lib/{TRIPLES[arch]}/libffi_pic.a", | |
| } | |
| for file in ALLOWLIST: | |
| assert os.path.exists(os.path.join(install_root, file)), ( | |
| f"{file} does not exist" | |
| ) | |
| # Remove all executables and static libraries, and any symlinks that | |
| # were pointing to them. | |
| reverse_links = collections.defaultdict(list) | |
| remove = [] | |
| for root, _, files in os.walk(install_root): | |
| for filename in files: | |
| filepath = os.path.join(root, filename) | |
| if os.path.relpath(filepath, install_root) in ALLOWLIST: | |
| continue | |
| if os.path.islink(filepath): | |
| target_path = os.readlink(filepath) | |
| if not os.path.isabs(target_path): | |
| target_path = os.path.join(root, target_path) | |
| reverse_links[os.path.realpath(target_path)].append(filepath) | |
| elif "so" in filepath.split(".")[-3:]: | |
| continue | |
| elif os.access(filepath, os.X_OK) or filepath.endswith(".a"): | |
| remove.append(filepath) | |
| for filepath in remove: | |
| os.remove(filepath) | |
| for link in reverse_links[filepath]: | |
| os.remove(link) | |
| def strip_sections(install_root: str, arch: str): | |
| """ | |
| Strips all sections from ELF files except for dynamic linking and | |
| essential sections. Skips static libraries (.a) and object files (.o). | |
| """ | |
| PRESERVED_SECTIONS = { | |
| ".dynamic", | |
| ".dynstr", | |
| ".dynsym", | |
| ".gnu.version", | |
| ".gnu.version_d", | |
| ".gnu.version_r", | |
| ".hash", | |
| ".note.ABI-tag", | |
| ".note.gnu.build-id", | |
| } | |
| for root, _, files in os.walk(install_root): | |
| for file in files: | |
| file_path = os.path.join(root, file) | |
| if ( | |
| os.access(file, os.X_OK) | |
| or file.endswith((".a", ".o")) | |
| or os.path.islink(file_path) | |
| ): | |
| continue | |
| # Verify this is an ELF file | |
| with open(file_path, "rb") as f: | |
| magic = f.read(4) | |
| if magic != b"\x7fELF": | |
| continue | |
| # Get section headers | |
| objdump_cmd = ["objdump", "-h", file_path] | |
| result = subprocess.run( | |
| objdump_cmd, check=True, text=True, capture_output=True | |
| ) | |
| section_lines = result.stdout.splitlines() | |
| # Parse section names | |
| sections = set() | |
| for line in section_lines: | |
| parts = line.split() | |
| if len(parts) > 1 and parts[0].isdigit(): | |
| sections.add(parts[1]) | |
| sections_to_remove = sections - PRESERVED_SECTIONS | |
| if sections_to_remove: | |
| objcopy_arch = "amd64" if arch == "i386" else arch | |
| objcopy_bin = TRIPLES[objcopy_arch] + "-objcopy" | |
| if not shutil.which(objcopy_bin): | |
| objcopy_bin = "objcopy" | |
| objcopy_cmd = ( | |
| [objcopy_bin] | |
| + [ | |
| f"--remove-section={section}" | |
| for section in sections_to_remove | |
| ] | |
| + [file_path] | |
| ) | |
| subprocess.run(objcopy_cmd, check=True, stderr=subprocess.PIPE) | |
| def record_metadata(install_root: str) -> dict[str, tuple[float, float]]: | |
| """ | |
| Recursively walk the install_root directory and record the metadata of all | |
| files. Symlinks are not followed. Returns a dictionary mapping each path | |
| (relative to install_root) to its original metadata. | |
| """ | |
| metadata = {} | |
| for root, dirs, files in os.walk(install_root): | |
| for name in dirs + files: | |
| full_path = os.path.join(root, name) | |
| rel_path = os.path.relpath(full_path, install_root) | |
| st = os.lstat(full_path) | |
| metadata[rel_path] = (st.st_atime, st.st_mtime) | |
| return metadata | |
| def restore_metadata( | |
| install_root: str, old_meta: dict[str, tuple[float, float]] | |
| ) -> None: | |
| """ | |
| 1. Restore the metadata of any file that exists in old_meta. | |
| 2. For all other files, set their timestamp to ARCHIVE_TIMESTAMP. | |
| 3. For all directories (including install_root), set the timestamp | |
| to ARCHIVE_TIMESTAMP. | |
| """ | |
| # Convert the timestamp to a UNIX epoch time. | |
| archive_time = time.mktime( | |
| time.strptime(ARCHIVE_TIMESTAMP, "%Y%m%dT%H%M%SZ") | |
| ) | |
| # Walk through the install_root, applying old_meta where available; | |
| # otherwise set times to archive_time. | |
| for root, dirs, files in os.walk(install_root): | |
| # Directories get archive_time. | |
| os.utime(root, (archive_time, archive_time)) | |
| # Files: old_meta if available, else archive_time. | |
| for file_name in files: | |
| file_path = os.path.join(root, file_name) | |
| if os.path.lexists(file_path): | |
| rel_path = os.path.relpath(file_path, install_root) | |
| if rel_path in old_meta: | |
| restore_time = old_meta[rel_path] | |
| else: | |
| restore_time = (archive_time, archive_time) | |
| os.utime(file_path, restore_time, follow_symlinks=False) | |
| def build_sysroot(arch: str) -> None: | |
| build_dir = get_build_dir(arch) | |
| install_root = os.path.join(build_dir, f"{RELEASES[arch]}_{arch}_staging") | |
| clear_install_dir(install_root) | |
| packages = generate_package_list(arch, build_dir) | |
| install_into_sysroot(build_dir, install_root, packages) | |
| old_metadata = record_metadata(install_root) | |
| hacks_and_patches(install_root, SCRIPT_DIR, arch) | |
| create_extra_symlinks(install_root, arch) | |
| cleanup_jail_symlinks(install_root) | |
| removing_unnecessary_files(install_root, arch) | |
| strip_sections(install_root, arch) | |
| restore_metadata(install_root, old_metadata) | |
| create_tarball(install_root, arch, build_dir) | |
| def upload_sysroot(arch: str) -> str: | |
| build_dir = get_build_dir(arch) | |
| tarball_path = os.path.join( | |
| build_dir, f"{DISTRO}_{RELEASES[arch]}_{arch}_sysroot.tar.xz" | |
| ) | |
| command = [ | |
| "upload_to_google_storage_first_class.py", | |
| "--bucket", | |
| "chrome-linux-sysroot", | |
| tarball_path, | |
| ] | |
| return subprocess.check_output(command).decode("utf-8") | |
| def verify_package_listing( | |
| file_path: str, output_file: str, dist: str, build_dir: str | |
| ) -> None: | |
| """ | |
| Verifies the downloaded Packages.xz file against its checksum and GPG keys. | |
| """ | |
| # Paths for Release and Release.gpg files | |
| repo_basedir = f"{ARCHIVE_URL}/dists/{dist}" | |
| release_list = f"{repo_basedir}/{RELEASE_FILE}" | |
| release_list_gpg = f"{repo_basedir}/{RELEASE_FILE_GPG}" | |
| release_file = os.path.join(build_dir, f"{dist}-{RELEASE_FILE}") | |
| release_file_gpg = os.path.join(build_dir, f"{dist}-{RELEASE_FILE_GPG}") | |
| if not os.path.exists(KEYRING_FILE): | |
| raise Exception(f"KEYRING_FILE not found: {KEYRING_FILE}") | |
| # Download Release and Release.gpg files | |
| download_or_copy_non_unique_filename(release_list, release_file) | |
| download_or_copy_non_unique_filename(release_list_gpg, release_file_gpg) | |
| # Verify Release file with GPG | |
| subprocess.run( | |
| ["gpgv", "--keyring", KEYRING_FILE, release_file_gpg, release_file], | |
| check=True, | |
| ) | |
| # Find the SHA256 checksum for the specific file in the Release file | |
| sha256sum_pattern = re.compile( | |
| r"([a-f0-9]{64})\s+\d+\s+" + re.escape(file_path) + r"$" | |
| ) | |
| sha256sum_match = None | |
| with open(release_file, "r") as f: | |
| for line in f: | |
| if match := sha256sum_pattern.search(line): | |
| sha256sum_match = match.group(1) | |
| break | |
| if not sha256sum_match: | |
| raise Exception(f"Checksum for {file_path} not found in {release_file}") | |
| if hash_file(hashlib.sha256(), output_file) != sha256sum_match: | |
| raise Exception(f"Checksum mismatch for {output_file}") | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Build and upload Debian sysroot images for Chromium." | |
| ) | |
| parser.add_argument("command", choices=["build", "upload"]) | |
| parser.add_argument("architecture", choices=list(TRIPLES)) | |
| args = parser.parse_args() | |
| sanity_check(get_build_dir(args.architecture)) | |
| if args.command == "build": | |
| build_sysroot(args.architecture) | |
| elif args.command == "upload": | |
| upload_sysroot(args.architecture) | |
| if __name__ == "__main__": | |
| main() |