| #!/usr/bin/env python3 |
| # Copyright 2015 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """This script will check out llvm and clang, and then package the results up |
| to a number of tarballs.""" |
| |
| import argparse |
| import fnmatch |
| import itertools |
| import json |
| import lzma |
| import multiprocessing.dummy |
| import os |
| import platform |
| import shutil |
| import subprocess |
| import sys |
| import tarfile |
| import time |
| |
| # Path constants. |
| THIS_DIR = os.path.dirname(__file__) |
| CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) |
| THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') |
| BUILDTOOLS_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'buildtools') |
| LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') |
| LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') |
| LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join( |
| THIRD_PARTY_DIR, 'llvm-bootstrap-install' |
| ) |
| LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') |
| LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') |
| |
| DEFAULT_GCS_BUCKET = 'chromium-browser-clang-staging' |
| |
| recorded_artifacts = [] |
| |
| |
| def Tee(output, logfile): |
| logfile.write(output) |
| print(output, end='') |
| |
| |
| def TeeCmd(cmd, logfile, fail_hard=True): |
| """Runs cmd and writes the output to both stdout and logfile.""" |
| # Reading from PIPE can deadlock if one buffer is full but we wait on a |
| # different one. To work around this, pipe the subprocess's stderr to |
| # its stdout buffer and don't give it a stdin. |
| # shell=True is required in cmd.exe since depot_tools has an svn.bat, and |
| # bat files only work with shell=True set. |
| with subprocess.Popen( |
| cmd, |
| shell=sys.platform == 'win32', |
| stdin=subprocess.DEVNULL, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT, |
| ) as proc: |
| for line in iter(proc.stdout.readline, b''): |
| Tee(line.decode(), logfile) |
| exit_code = proc.wait() |
| if exit_code != 0 and fail_hard: |
| print('Failed:', cmd) |
| sys.exit(1) |
| |
| |
| def PrintTarProgress(tarinfo): |
| print('Adding', tarinfo.name) |
| return tarinfo |
| |
| |
| def GetGsutilPath(): |
| if 'find_depot_tools' not in sys.modules: |
| sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build')) |
| global find_depot_tools |
| import find_depot_tools |
| depot_path = find_depot_tools.add_depot_tools_to_path() |
| if depot_path is None: |
| print( |
| 'depot_tools are not found in PATH. ' |
| 'Follow the instructions in this document ' |
| 'http://dev.chromium.org/developers/how-tos/install-depot-tools' |
| ' to install depot_tools and then try again.' |
| ) |
| sys.exit(1) |
| gsutil_path = os.path.join(depot_path, 'gsutil.py') |
| return gsutil_path |
| |
| |
| def RunGsutil(args): |
| return subprocess.call([sys.executable, GetGsutilPath()] + args) |
| |
| |
| def PackageInArchive(directory_path, archive_path): |
| bin_dir_path = os.path.join(directory_path, 'bin') |
| if sys.platform != 'win32' and os.path.exists(bin_dir_path): |
| for f in os.listdir(bin_dir_path): |
| file_path = os.path.join(bin_dir_path, f) |
| if not os.path.islink(file_path): |
| if sys.platform == 'darwin': |
| subprocess.call(['strip', '-x', file_path]) |
| else: |
| subprocess.call(['strip', file_path]) |
| |
| with tarfile.open( |
| archive_path + '.tar.xz', 'w:xz', preset=9 | lzma.PRESET_EXTREME |
| ) as tar_xz: |
| for f in sorted(os.listdir(directory_path)): |
| tar_xz.add( |
| os.path.join(directory_path, f), arcname=f, filter=PrintTarProgress |
| ) |
| |
| |
| def VerifyPackageDoesntExist(gcs_bucket, filename, gcs_platform): |
| """Verifies that the package doesn't already exist on GCS, exiting if so.""" |
| gcs_path = f'gs://{gcs_bucket}/{gcs_platform}/{filename}' |
| print(f'Checking if {gcs_path} already exists...') |
| if RunGsutil(['stat', gcs_path]) == 0: |
| print(f'Package {gcs_path} already exists!') |
| print('Did you forget to update the sub-revision?') |
| sys.exit(1) |
| |
| |
| def MaybeUpload( |
| do_upload, gcs_bucket, filename, gcs_platform, extra_gsutil_args=[] |
| ): |
| recorded_artifacts.append( |
| { |
| 'local_file': filename, |
| 'gcs_bucket': gcs_bucket, |
| 'gcs_path': '%s/%s' % (gcs_platform, filename), |
| } |
| ) |
| gsutil_args = ( |
| # Fail if the item generation is not 0 (i.e. if it already exists). |
| ['-h', 'x-goog-if-generation-match:0', 'cp'] |
| + extra_gsutil_args |
| + ['-n', filename, 'gs://%s/%s/' % (gcs_bucket, gcs_platform)] |
| ) |
| if do_upload: |
| print('Uploading %s to Google Cloud Storage...' % filename) |
| exit_code = RunGsutil(gsutil_args) |
| if exit_code != 0: |
| print("gsutil failed, exit_code: %s" % exit_code) |
| sys.exit(exit_code) |
| else: |
| print('To upload, run:') |
| print('gsutil %s' % ' '.join(gsutil_args)) |
| |
| |
| def UploadPDBsToSymbolServer(binaries): |
| assert sys.platform == 'win32' |
| # Upload PDB and binary to the symbol server on Windows. Put them into the |
| # chromium-browser-symsrv bucket, since chrome devs have that in their |
| # _NT_SYMBOL_PATH already. Executable and PDB must be at paths following a |
| # certain pattern for the Microsoft debuggers to be able to load them. |
| # Executable: |
| # chromium-browser-symsrv/clang-cl.exe/ABCDEFAB01234/clang-cl.ex_ |
| # ABCDEFAB is the executable's timestamp in %08X format, 01234 is the |
| # executable's image size in %x format. tools/symsrc/img_fingerprint.py |
| # can compute this ABCDEFAB01234 string for us, so use that. |
| # The .ex_ instead of .exe at the end means that the file is compressed. |
| # PDB: |
| # gs://chromium-browser-symsrv/clang-cl.exe.pdb/AABBCCDD/clang-cl.exe.pd_ |
| # AABBCCDD here is computed from the output of |
| # dumpbin /all mybinary.exe | find "Format: RSDS" |
| # but tools/symsrc/pdb_fingerprint_from_img.py can compute it already, so |
| # again just use that. |
| sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'tools', 'symsrc')) |
| import img_fingerprint |
| import pdb_fingerprint_from_img |
| |
| files = [] |
| for binary_path in binaries: |
| binary_path = os.path.join(LLVM_RELEASE_DIR, binary_path) |
| binary_id = img_fingerprint.GetImgFingerprint(binary_path) |
| (pdb_id, pdb_path) = pdb_fingerprint_from_img.GetPDBInfoFromImg(binary_path) |
| files += [(binary_path, binary_id), (pdb_path, pdb_id)] |
| |
| # The build process builds clang.exe and then copies it to clang-cl.exe |
| # (both are the same binary and they behave differently on what their |
| # filename is). Hence, the pdb is at clang.pdb, not at clang-cl.pdb. |
| # Likewise, lld-link.exe's PDB file is called lld.pdb. |
| |
| # Compress and upload. |
| def compress(t): |
| subprocess.check_call( |
| [ |
| 'makecab', |
| '/D', |
| 'CompressionType=LZX', |
| '/D', |
| 'CompressionMemory=21', |
| t[0], |
| '/L', |
| os.path.dirname(t[0]), |
| ], |
| stdout=open(os.devnull, 'w'), |
| ) |
| |
| multiprocessing.dummy.Pool().map(compress, files) |
| for f, f_id in files: |
| f_cab = f[:-1] + '_' |
| dest = '%s/%s/%s' % (os.path.basename(f), f_id, os.path.basename(f_cab)) |
| print('Uploading %s to Google Cloud Storage...' % dest) |
| gsutil_args = [ |
| 'cp', |
| '-n', |
| '-a', |
| 'public-read', |
| f_cab, |
| 'gs://chromium-browser-symsrv/' + dest, |
| ] |
| exit_code = RunGsutil(gsutil_args) |
| if exit_code != 0: |
| print("gsutil failed, exit_code: %s" % exit_code) |
| print( |
| "If a precondition did not hold, a package at this revision likely aready exists." |
| ) |
| sys.exit(exit_code) |
| |
| |
| def replace_version(want, version): |
| """Replaces $V with provided version.""" |
| return set([w.replace('$V', version) for w in want]) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='build and package clang') |
| parser.add_argument( |
| '--upload', |
| action='store_true', |
| help='Upload the target archive to Google Cloud Storage.', |
| ) |
| parser.add_argument( |
| '--bucket', |
| default=DEFAULT_GCS_BUCKET, |
| help='Google Cloud Storage bucket where the target archive is uploaded', |
| ) |
| parser.add_argument( |
| '--revision', help='LLVM revision to use. Default: based on update.py' |
| ) |
| parser.add_argument( |
| '--output-artifacts-json', help='Write JSON list of artifacts to this file.' |
| ) |
| parser.add_argument( |
| '--gcs-dir-name', help='Override the GCS platform directory name.' |
| ) |
| args = parser.parse_args() |
| |
| if args.revision: |
| # Use upload_revision.py to set the revision first. |
| cmd = [ |
| sys.executable, |
| os.path.join(THIS_DIR, 'upload_revision.py'), |
| '--no-git', # Just run locally, don't upload anything. |
| '--clang-git-hash=' + args.revision, |
| ] |
| subprocess.run(cmd, check=True) |
| |
| # This needs to happen after upload_revision.py modifies update.py. |
| from update import ( |
| PACKAGE_VERSION, |
| RELEASE_VERSION, |
| STAMP_FILE, |
| STAMP_FILENAME, |
| ) |
| |
| expected_stamp = PACKAGE_VERSION |
| pdir = 'clang-' + expected_stamp |
| print(pdir) |
| |
| if args.gcs_dir_name: |
| gcs_platform = args.gcs_dir_name |
| elif sys.platform == 'darwin': |
| # When we need to run this script on an arm machine, we need to add a |
| # --build-mac-intel switch to pick which clang to build, pick the |
| # 'Mac_arm64' here when there's no flag and 'Mac' when --build-mac-intel is |
| # passed. Also update the build script to explicitly pass a default triple |
| # then. |
| if platform.machine() == 'arm64': |
| gcs_platform = 'Mac_arm64' |
| else: |
| gcs_platform = 'Mac' |
| elif sys.platform == 'win32': |
| gcs_platform = 'Win' |
| else: |
| gcs_platform = 'Linux_x64' |
| |
| if args.upload: |
| VerifyPackageDoesntExist(args.bucket, pdir + '.tar.xz', gcs_platform) |
| |
| with open('buildlog.txt', 'w', encoding='utf-8') as log: |
| Tee('Starting build\n', log) |
| |
| # Do a clobber build. |
| shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) |
| shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) |
| shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) |
| |
| build_cmd = [ |
| sys.executable, |
| os.path.join(THIS_DIR, 'build.py'), |
| '--bootstrap', |
| '--disable-asserts', |
| '--run-tests', |
| ] |
| # PGO drastically increases packaging time and x86-64 macs are almost |
| # obsolete while being slow, so don't PGO-optimize x86-64 mac toolchains. |
| if sys.platform != 'darwin' or platform.machine() == 'arm64': |
| build_cmd.append('--pgo') |
| if sys.platform != 'darwin': |
| build_cmd.append('--thinlto') |
| if sys.platform.startswith('linux'): |
| build_cmd.append('--bolt') |
| |
| TeeCmd(build_cmd, log) |
| |
| stamp = open(STAMP_FILE).read().rstrip() |
| if stamp != expected_stamp: |
| print('Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp)) |
| return 1 |
| |
| shutil.rmtree(pdir, ignore_errors=True) |
| |
| # Track of all files that should be part of runtime package. '$V' is replaced |
| # by RELEASE_VERSION, and no glob is supported. |
| runtime_packages = None |
| runtime_package_name = None |
| |
| android_runtime_packages = None |
| android_runtime_package_name = None |
| |
| # Copy a list of files to the directory we're going to tar up. |
| # This supports the same patterns that the fnmatch module understands. |
| # '$V' is replaced by RELEASE_VERSION further down. |
| exe_ext = '.exe' if sys.platform == 'win32' else '' |
| want = set( |
| [ |
| STAMP_FILENAME, |
| 'bin/llvm-pdbutil' + exe_ext, |
| 'bin/llvm-symbolizer' + exe_ext, |
| 'bin/llvm-undname' + exe_ext, |
| # Copy built-in headers (lib/clang/3.x.y/include). |
| 'lib/clang/$V/include/*', |
| 'lib/clang/$V/share/asan_ignorelist.txt', |
| 'lib/clang/$V/share/cfi_ignorelist.txt', |
| ] |
| ) |
| if sys.platform == 'win32': |
| want.update( |
| [ |
| 'bin/clang-cl.exe', |
| 'bin/lld-link.exe', |
| 'bin/llvm-ml.exe', |
| ] |
| ) |
| else: |
| want.update( |
| [ |
| 'bin/clang', |
| # Add LLD. |
| 'bin/lld', |
| # Add llvm-ar for LTO. |
| 'bin/llvm-ar', |
| # llvm-ml for Windows cross builds. |
| 'bin/llvm-ml', |
| # Add llvm-readobj (symlinked from llvm-readelf) for extracting SONAMEs. |
| 'bin/llvm-readobj', |
| ] |
| ) |
| if sys.platform != 'darwin': |
| # The Fuchsia runtimes are only built on non-Mac platforms. |
| want.update( |
| [ |
| 'lib/clang/$V/lib/aarch64-unknown-fuchsia/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/x86_64-unknown-fuchsia/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/x86_64-unknown-fuchsia/libclang_rt.profile.a', |
| 'lib/clang/$V/lib/x86_64-unknown-fuchsia/libclang_rt.asan.so', |
| 'lib/clang/$V/lib/x86_64-unknown-fuchsia/libclang_rt.asan-preinit.a', |
| 'lib/clang/$V/lib/x86_64-unknown-fuchsia/libclang_rt.asan_static.a', |
| ] |
| ) |
| if sys.platform == 'darwin': |
| runtime_package_name = 'clang-mac-runtime-library' |
| runtime_packages = set( |
| [ |
| # AddressSanitizer runtime. |
| 'lib/clang/$V/lib/darwin/libclang_rt.asan_ios_dynamic.dylib', |
| 'lib/clang/$V/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', |
| 'lib/clang/$V/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', |
| # Builtin libraries for the _IsOSVersionAtLeast runtime function. |
| 'lib/clang/$V/lib/darwin/libclang_rt.ios.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.iossim.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.osx.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.tvos.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.tvossim.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.watchos.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.watchossim.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.xros.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.xrossim.a', |
| # Profile runtime (used by profiler and code coverage). |
| 'lib/clang/$V/lib/darwin/libclang_rt.profile_iossim.a', |
| 'lib/clang/$V/lib/darwin/libclang_rt.profile_osx.a', |
| # UndefinedBehaviorSanitizer runtime. |
| 'lib/clang/$V/lib/darwin/libclang_rt.ubsan_iossim_dynamic.dylib', |
| 'lib/clang/$V/lib/darwin/libclang_rt.ubsan_osx_dynamic.dylib', |
| ] |
| ) |
| want.update(runtime_packages) |
| want.update( |
| [ |
| # Add llvm-objcopy for its use as install_name_tool. |
| 'bin/llvm-objcopy', |
| ] |
| ) |
| elif sys.platform.startswith('linux'): |
| runtime_package_name = 'clang-linux-runtime-library' |
| runtime_packages = set( |
| [ |
| # pylint: disable=line-too-long |
| # AddressSanitizer C runtime (pure C won't link with *_cxx). |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.asan.a', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.asan.a.syms', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.asan.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.asan.a.syms', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.asan.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.asan.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.asan.a.syms', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.a.syms', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.asan_static.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.asan_static.a', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.asan_static.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.asan_static.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.asan_static.a', |
| # AddressSanitizer C++ runtime. |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.asan_cxx.a', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.asan_cxx.a.syms', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.asan_cxx.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.asan_cxx.a.syms', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.asan_cxx.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.asan_cxx.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.asan_cxx.a.syms', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.asan_cxx.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.asan_cxx.a.syms', |
| # Builtins for Linux. |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.builtins.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.builtins.a', |
| # crtstart/crtend for Linux. |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/clang_rt.crtbegin.o', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/clang_rt.crtend.o', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/clang_rt.crtbegin.o', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/clang_rt.crtend.o', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/clang_rt.crtbegin.o', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/clang_rt.crtend.o', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/clang_rt.crtbegin.o', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/clang_rt.crtend.o', |
| # MemorySanitizer C runtime (pure C won't link with *_cxx). |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.msan.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.msan.a.syms', |
| # MemorySanitizer C++ runtime. |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.msan_cxx.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.msan_cxx.a.syms', |
| # Profile runtime (used by profiler and code coverage). |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.profile.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.profile.a', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.profile.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.profile.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a', |
| # ThreadSanitizer C runtime (pure C won't link with *_cxx). |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.tsan.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.tsan.a.syms', |
| # ThreadSanitizer C++ runtime. |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.tsan_cxx.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.tsan_cxx.a.syms', |
| # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a.syms', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.ubsan_standalone.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.ubsan_standalone.a.syms', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.ubsan_standalone.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a.syms', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.ubsan_standalone.a.syms', |
| # UndefinedBehaviorSanitizer C++ runtime. |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a', |
| 'lib/clang/$V/lib/aarch64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a.syms', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.ubsan_standalone_cxx.a', |
| 'lib/clang/$V/lib/armv7-unknown-linux-gnueabihf/libclang_rt.ubsan_standalone_cxx.a.syms', |
| 'lib/clang/$V/lib/i386-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a', |
| 'lib/clang/$V/lib/riscv64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a.syms', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a', |
| 'lib/clang/$V/lib/x86_64-unknown-linux-gnu/libclang_rt.ubsan_standalone_cxx.a.syms', |
| # Ignorelist for MemorySanitizer (used on Linux only). |
| 'lib/clang/$V/share/msan_ignorelist.txt', |
| # pylint: enable=line-too-long |
| ] |
| ) |
| want.update(runtime_packages) |
| want.update( |
| [ |
| # Required for use_debug_fissions=true to not break symbolization on swarming. |
| 'bin/llvm-dwp', |
| # Add llvm-objcopy for partition extraction on Android. |
| 'bin/llvm-objcopy', |
| # Add llvm-nm. |
| 'bin/llvm-nm', |
| ] |
| ) |
| # The Android compiler-rt runtimes are cross-compiled target libraries |
| # (host-independent). Ship them only as a standalone package that every |
| # host building for Android (Linux included) overlays onto its host clang, |
| # rather than bundling them into each host's clang package. |
| android_runtime_package_name = 'clang-android-runtime-library' |
| android_runtime_packages = set( |
| [ |
| # pylint: disable=line-too-long |
| # AddressSanitizer Android runtime. |
| 'lib/clang/$V/lib/linux/libclang_rt.asan-aarch64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan-arm-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan-i686-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan-x86_64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan-riscv64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan_static-aarch64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan_static-arm-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan_static-i686-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan_static-x86_64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.asan_static-riscv64-android.a', |
| # Builtins for Android. |
| 'lib/clang/$V/lib/linux/libclang_rt.builtins-aarch64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.builtins-arm-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.builtins-i686-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.builtins-x86_64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.builtins-riscv64-android.a', |
| # HWASAN Android runtime. |
| 'lib/clang/$V/lib/linux/libclang_rt.hwasan-aarch64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.hwasan-preinit-aarch64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.hwasan-riscv64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.hwasan-preinit-riscv64-android.a', |
| # Profile runtime (used by profiler and code coverage). |
| 'lib/clang/$V/lib/linux/libclang_rt.profile-i686-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.profile-x86_64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.profile-aarch64-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.profile-arm-android.a', |
| 'lib/clang/$V/lib/linux/libclang_rt.profile-riscv64-android.a', |
| # UndefinedBehaviorSanitizer Android runtime, needed for CFI. |
| 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-aarch64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-arm-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-i686-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-x86_64-android.so', |
| 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-riscv64-android.so', |
| # pylint: enable=line-too-long |
| ] |
| ) |
| elif sys.platform == 'win32': |
| runtime_package_name = 'clang-win-runtime-library' |
| |
| runtime_packages = set( |
| [ |
| # pylint: disable=line-too-long |
| 'bin/llvm-symbolizer.exe', |
| # AddressSanitizer runtime. |
| 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.dll', |
| 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.lib', |
| # Thunk for AddressSanitizer for component builds. |
| 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic_runtime_thunk-x86_64.lib', |
| # Thunk for AddressSanitizer for static builds. |
| 'lib/clang/$V/lib/windows/clang_rt.asan_static_runtime_thunk-x86_64.lib', |
| # Builtins for C/C++. |
| 'lib/clang/$V/lib/windows/clang_rt.builtins-i386.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.builtins-x86_64.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.builtins-aarch64.lib', |
| # Profile runtime (used by profiler and code coverage). |
| 'lib/clang/$V/lib/windows/clang_rt.profile-i386.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.profile-x86_64.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.profile-aarch64.lib', |
| # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). |
| 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', |
| # UndefinedBehaviorSanitizer C++ runtime. |
| 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', |
| # pylint: enable=line-too-long |
| ] |
| ) |
| want.update(runtime_packages) |
| |
| # reclient is a tool for executing programs remotely. When uploading the |
| # binary to be executed, it needs to know which other files the binary depends |
| # on. This can include shared libraries, as well as other dependencies not |
| # explicitly mentioned in the source code (those would be found by reclient's |
| # include scanner) such as sanitizer ignore lists. |
| # |
| # These paths are written relative to the package root, and will be rebased |
| # to wherever the reclient config file is written when added to the file. |
| reclient_inputs = { |
| 'clang': set( |
| [ |
| # Note: These have to match the `want` list exactly. |
| 'lib/clang/$V/share/asan_ignorelist.txt', |
| 'lib/clang/$V/share/cfi_ignorelist.txt', |
| ] |
| ), |
| } |
| if sys.platform == 'win32': |
| # TODO(crbug.com/335997052): Remove this again once we have a compiler |
| # flag that tells clang-cl to not auto-add it (and then explicitly pass |
| # it via GN). |
| reclient_inputs['clang'].update( |
| [ |
| 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.profile-i386.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.profile-x86_64.lib', |
| 'lib/clang/$V/lib/windows/clang_rt.profile-aarch64.lib', |
| ] |
| ) |
| |
| # Check that all non-glob wanted files exist on disk. |
| want = replace_version(want, RELEASE_VERSION) |
| found_all_wanted_files = True |
| for w in want: |
| if '*' in w: |
| continue |
| if os.path.exists(os.path.join(LLVM_RELEASE_DIR, w)): |
| continue |
| print('wanted file "%s" but it did not exist' % w, file=sys.stderr) |
| found_all_wanted_files = False |
| |
| if not found_all_wanted_files: |
| return 1 |
| |
| # Check that all reclient inputs are in the package. |
| for tool in reclient_inputs: |
| reclient_inputs[tool] = set( |
| [i.replace('$V', RELEASE_VERSION) for i in reclient_inputs[tool]] |
| ) |
| missing = reclient_inputs[tool] - want |
| if missing: |
| print('reclient inputs not part of package: ', missing, file=sys.stderr) |
| return 1 |
| |
| reclient_input_strings = {t: '' for t in reclient_inputs} |
| |
| # TODO(thakis): Try walking over want and copying the files in there instead |
| # of walking the directory and doing fnmatch() against want. |
| for root, dirs, files in os.walk(LLVM_RELEASE_DIR): |
| dirs.sort() # Walk dirs in sorted order. |
| # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/... |
| rel_root = root[len(LLVM_RELEASE_DIR) + 1 :] |
| rel_files = [os.path.join(rel_root, f) for f in files] |
| wanted_files = list( |
| set( |
| itertools.chain.from_iterable( |
| fnmatch.filter(rel_files, p) for p in want |
| ) |
| ) |
| ) |
| if wanted_files: |
| # Guaranteed to not yet exist at this point: |
| os.makedirs(os.path.join(pdir, rel_root)) |
| llvm_strip = os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-strip') |
| for f in sorted(wanted_files): |
| src = os.path.join(LLVM_RELEASE_DIR, f) |
| dest = os.path.join(pdir, f) |
| shutil.copy(src, dest) |
| # Strip libraries. |
| if 'libclang_rt.builtins' in f and 'android' in f: |
| # Keep the builtins' DWARF info for unwinding. |
| pass |
| elif sys.platform == 'darwin' and f.endswith('.dylib'): |
| subprocess.call(['strip', '-x', dest]) |
| elif sys.platform.startswith('linux') and os.path.splitext(f)[1] in [ |
| '.so', |
| '.a', |
| ]: |
| subprocess.call([llvm_strip, '--keep-file-symbols', '-g', dest]) |
| # If this is an reclient input, add it to the inputs file(s). |
| for tool, inputs in reclient_inputs.items(): |
| if any(fnmatch.fnmatch(f, i) for i in inputs): |
| rel_input = os.path.relpath(dest, os.path.join(pdir, 'bin')) |
| reclient_input_strings[tool] += '%s\n' % rel_input |
| |
| # Write the reclient inputs files. |
| if sys.platform != 'win32': |
| reclient_input_strings['clang++'] = reclient_input_strings['clang'] |
| reclient_input_strings['clang-cl'] = reclient_input_strings['clang'] |
| else: |
| reclient_input_strings['clang-cl.exe'] = reclient_input_strings.pop('clang') |
| for tool, string in reclient_input_strings.items(): |
| filename = os.path.join(pdir, 'bin', '%s_remote_toolchain_inputs' % tool) |
| print('%s:\n%s' % (filename, string)) |
| with open(filename, 'w') as f: |
| f.write(string) |
| |
| # Set up symlinks. |
| if sys.platform != 'win32': |
| os.symlink('clang', os.path.join(pdir, 'bin', 'clang++')) |
| os.symlink('clang', os.path.join(pdir, 'bin', 'clang-cl')) |
| os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) |
| os.symlink('lld', os.path.join(pdir, 'bin', 'ld64.lld')) |
| os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) |
| os.symlink('lld', os.path.join(pdir, 'bin', 'wasm-ld')) |
| os.symlink('llvm-readobj', os.path.join(pdir, 'bin', 'llvm-readelf')) |
| |
| if sys.platform.startswith('linux') or sys.platform == 'darwin': |
| os.symlink('llvm-objcopy', os.path.join(pdir, 'bin', 'llvm-strip')) |
| os.symlink( |
| 'llvm-objcopy', os.path.join(pdir, 'bin', 'llvm-install-name-tool') |
| ) |
| |
| # Make `--target=*-cros-linux-gnu` work with |
| # LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON. |
| for arch, abi in [ |
| ('armv7', 'gnueabihf'), |
| ('aarch64', 'gnu'), |
| ('x86_64', 'gnu'), |
| ]: |
| old = '%s-unknown-linux-%s' % (arch, abi) |
| new = old.replace('unknown', 'cros').replace('armv7', 'armv7a') |
| os.symlink( |
| old, os.path.join(pdir, 'lib', 'clang', RELEASE_VERSION, 'lib', new) |
| ) |
| |
| # Create main archive. |
| PackageInArchive(pdir, pdir) |
| MaybeUpload(args.upload, args.bucket, pdir + '.tar.xz', gcs_platform) |
| |
| # Upload build log next to it. |
| os.rename('buildlog.txt', pdir + '-buildlog.txt') |
| MaybeUpload( |
| args.upload, |
| args.bucket, |
| pdir + '-buildlog.txt', |
| gcs_platform, |
| extra_gsutil_args=['-z', 'txt'], |
| ) |
| os.remove(pdir + '-buildlog.txt') |
| |
| # Zip up runtime libraries, if specified. |
| if runtime_package_name and len(runtime_packages) > 0: |
| runtime_dir = f'{runtime_package_name}-{stamp}' |
| shutil.rmtree(runtime_dir, ignore_errors=True) |
| for f in sorted(replace_version(runtime_packages, RELEASE_VERSION)): |
| os.makedirs(os.path.dirname(os.path.join(runtime_dir, f)), exist_ok=True) |
| shutil.copy( |
| os.path.join(pdir, f), |
| os.path.join(runtime_dir, f), |
| ) |
| PackageInArchive(runtime_dir, runtime_dir) |
| MaybeUpload(args.upload, args.bucket, f'{runtime_dir}.tar.xz', gcs_platform) |
| |
| if android_runtime_package_name and len(android_runtime_packages) > 0: |
| runtime_dir = f'{android_runtime_package_name}-{stamp}' |
| shutil.rmtree(runtime_dir, ignore_errors=True) |
| for f in sorted(replace_version(android_runtime_packages, RELEASE_VERSION)): |
| os.makedirs(os.path.dirname(os.path.join(runtime_dir, f)), exist_ok=True) |
| # Use the build output for runtimes that are not bundled into the |
| # host package (i.e. the Android target runtimes, which ship only |
| # in this standalone package). |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, f), |
| os.path.join(runtime_dir, f), |
| ) |
| PackageInArchive(runtime_dir, runtime_dir) |
| MaybeUpload(args.upload, args.bucket, f'{runtime_dir}.tar.xz', gcs_platform) |
| |
| # Zip up llvm-code-coverage for code coverage. |
| code_coverage_dir = 'llvm-code-coverage-' + stamp |
| shutil.rmtree(code_coverage_dir, ignore_errors=True) |
| os.makedirs(os.path.join(code_coverage_dir, 'bin')) |
| for filename in ['llvm-cov', 'llvm-profdata']: |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', filename + exe_ext), |
| os.path.join(code_coverage_dir, 'bin'), |
| ) |
| PackageInArchive(code_coverage_dir, code_coverage_dir) |
| MaybeUpload( |
| args.upload, args.bucket, code_coverage_dir + '.tar.xz', gcs_platform |
| ) |
| |
| # Zip up llvm-objdump and related tools for sanitizer coverage and Supersize. |
| objdumpdir = 'llvmobjdump-' + stamp |
| shutil.rmtree(objdumpdir, ignore_errors=True) |
| os.makedirs(os.path.join(objdumpdir, 'bin')) |
| for filename in [ |
| 'llvm-bcanalyzer', |
| 'llvm-cxxfilt', |
| 'llvm-dwarfdump', |
| 'llvm-nm', |
| 'llvm-objdump', |
| ]: |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', filename + exe_ext), |
| os.path.join(objdumpdir, 'bin'), |
| ) |
| llvmobjdump_stamp_file_base = 'llvmobjdump_build_revision' |
| llvmobjdump_stamp_file = os.path.join(objdumpdir, llvmobjdump_stamp_file_base) |
| with open(llvmobjdump_stamp_file, 'w') as f: |
| f.write(expected_stamp) |
| f.write('\n') |
| if sys.platform != 'win32': |
| os.symlink('llvm-objdump', os.path.join(objdumpdir, 'bin', 'llvm-otool')) |
| PackageInArchive(objdumpdir, objdumpdir) |
| MaybeUpload(args.upload, args.bucket, objdumpdir + '.tar.xz', gcs_platform) |
| |
| # Zip up clang-tidy for users who opt into it, and Tricium. |
| clang_tidy_dir = 'clang-tidy-' + stamp |
| shutil.rmtree(clang_tidy_dir, ignore_errors=True) |
| os.makedirs(os.path.join(clang_tidy_dir, 'bin')) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'clang-tidy' + exe_ext), |
| os.path.join(clang_tidy_dir, 'bin'), |
| ) |
| PackageInArchive(clang_tidy_dir, clang_tidy_dir) |
| MaybeUpload( |
| args.upload, args.bucket, clang_tidy_dir + '.tar.xz', gcs_platform |
| ) |
| |
| # Zip up clangd and related tools for users who opt into it. |
| clangd_dir = 'clangd-' + stamp |
| shutil.rmtree(clangd_dir, ignore_errors=True) |
| os.makedirs(os.path.join(clangd_dir, 'bin')) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'clangd' + exe_ext), |
| os.path.join(clangd_dir, 'bin'), |
| ) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'clang-include-cleaner' + exe_ext), |
| os.path.join(clangd_dir, 'bin'), |
| ) |
| PackageInArchive(clangd_dir, clangd_dir) |
| MaybeUpload(args.upload, args.bucket, clangd_dir + '.tar.xz', gcs_platform) |
| |
| # Zip up clang-format so we can update it (separately from the clang roll). |
| clang_format_dir = 'clang-format-' + stamp |
| shutil.rmtree(clang_format_dir, ignore_errors=True) |
| os.makedirs(os.path.join(clang_format_dir, 'bin')) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'clang-format' + exe_ext), |
| os.path.join(clang_format_dir, 'bin'), |
| ) |
| PackageInArchive(clang_format_dir, clang_format_dir) |
| MaybeUpload( |
| args.upload, args.bucket, clang_format_dir + '.tar.xz', gcs_platform |
| ) |
| |
| if sys.platform == 'darwin': |
| # dsymutil isn't part of the main zip, and it gets periodically |
| # deployed to CIPD (manually, not as part of clang rolls) for use in the |
| # Mac build toolchain. |
| dsymdir = 'dsymutil-' + stamp |
| shutil.rmtree(dsymdir, ignore_errors=True) |
| os.makedirs(os.path.join(dsymdir, 'bin')) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'dsymutil'), |
| os.path.join(dsymdir, 'bin'), |
| ) |
| PackageInArchive(dsymdir, dsymdir) |
| MaybeUpload(args.upload, args.bucket, dsymdir + '.tar.xz', gcs_platform) |
| |
| # Zip up the translation_unit tool. |
| translation_unit_dir = 'translation_unit-' + stamp |
| shutil.rmtree(translation_unit_dir, ignore_errors=True) |
| os.makedirs(os.path.join(translation_unit_dir, 'bin')) |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'translation_unit' + exe_ext), |
| os.path.join(translation_unit_dir, 'bin'), |
| ) |
| PackageInArchive(translation_unit_dir, translation_unit_dir) |
| MaybeUpload( |
| args.upload, args.bucket, translation_unit_dir + '.tar.xz', gcs_platform |
| ) |
| |
| # Zip up the libclang binaries. |
| libclang_dir = 'libclang-' + stamp |
| shutil.rmtree(libclang_dir, ignore_errors=True) |
| os.makedirs(os.path.join(libclang_dir, 'bin')) |
| os.makedirs(os.path.join(libclang_dir, 'bindings', 'python', 'clang')) |
| if sys.platform == 'win32': |
| shutil.copy( |
| os.path.join(LLVM_RELEASE_DIR, 'bin', 'libclang.dll'), |
| os.path.join(libclang_dir, 'bin'), |
| ) |
| py_bindings_dir = os.path.join( |
| LLVM_DIR, 'clang', 'bindings', 'python', 'clang' |
| ) |
| for filename in os.listdir(py_bindings_dir): |
| if filename == "__pycache__": |
| continue |
| shutil.copy( |
| os.path.join(py_bindings_dir, filename), |
| os.path.join(libclang_dir, 'bindings', 'python', 'clang'), |
| ) |
| PackageInArchive(libclang_dir, libclang_dir) |
| MaybeUpload(args.upload, args.bucket, libclang_dir + '.tar.xz', gcs_platform) |
| |
| if sys.platform == 'win32' and args.upload: |
| binaries = [f for f in want if f.endswith('.exe') or f.endswith('.dll')] |
| assert 'bin/clang-cl.exe' in binaries |
| assert 'bin/lld-link.exe' in binaries |
| start = time.time() |
| UploadPDBsToSymbolServer(binaries) |
| end = time.time() |
| print('symbol upload took', end - start, 'seconds') |
| |
| if args.output_artifacts_json: |
| with open(args.output_artifacts_json, 'w') as f: |
| json.dump(recorded_artifacts, f, indent=2) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
