diff --git a/tools/clang/scripts/build.py b/tools/clang/scripts/build.py index c9d1d29..354a3e4 100755 --- a/tools/clang/scripts/build.py +++ b/tools/clang/scripts/build.py
@@ -14,6 +14,7 @@ import argparse import glob +import json import os import pipes import re @@ -31,17 +32,14 @@ # Path constants. (All of these should be absolute paths.) THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party') LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') +COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt') LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap-install') -CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') -THREADS_ENABLED_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'threads_enabled') +CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'llvm', 'tools', 'chrometools') +THREADS_ENABLED_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, + 'llvm-threads-enabled') COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') -CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang') -LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld') -COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') -LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx') -LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi') LLVM_BUILD_TOOLS_DIR = os.path.abspath( os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) ANDROID_NDK_DIR = os.path.join( @@ -49,29 +47,10 @@ FUCHSIA_SDK_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'fuchsia-sdk', 'sdk') -LLVM_REPO_URL = os.environ.get('LLVM_REPO_URL', - 'https://llvm.org/svn/llvm-project') - BUG_REPORT_URL = ('https://crbug.com and run tools/clang/scripts/upload_crash.py' ' (only works inside Google) which will upload a report') -def GetSvnRevision(svn_repo): - """Returns current revision of the svn repo at svn_repo.""" - svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True) - m = re.search(r'Revision: (\d+)', svn_info) - return m.group(1) - - -def RmCmakeCache(dir): - """Delete CMake cache related files from dir.""" - for dirpath, dirs, files in os.walk(dir): - if 'CMakeCache.txt' in files: - os.remove(os.path.join(dirpath, 'CMakeCache.txt')) - if 'CMakeFiles' in dirs: - RmTree(os.path.join(dirpath, 'CMakeFiles')) - - def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): """Run command and return success (True) or failure; or if fail_hard is True, exit on failure. If msvc_arch is set, runs the command in a @@ -113,41 +92,64 @@ CopyFile(os.path.join(src, f), dst) -def Checkout(name, url, dir): - """Checkout the SVN module at url into dir. Use name for the log message.""" - print("Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)) +def CheckoutLLVM(commit, dir): + """Checkout the LLVM monorepo at a certain git commit in dir. Any local + modifications in dir will be lost.""" + print("Checking out LLVM monorepo %s into '%s'" % (commit, dir)) - command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir] - # The checkout command usually succeeds and produces lots of unininteresting - # output. Hence, pass `--quiet` on the first run and run an explicit command - # to print the revision we got afterwards on the first attempt. - if RunCommand(command + ['--quiet'], fail_hard=False): - RunCommand(['svnversion', dir]) - return + git_dir = os.path.join(dir, '.git') + fetch_cmd = ['git', '--git-dir', git_dir, 'fetch'] + checkout_cmd = ['git', 'checkout', commit] + # Do a somewhat shallow clone to save on bandwidth for GitHub and for us. + # The depth was whosen to be deep enough to contain the version we're + # building, and shallow enough to save significantly on bandwidth compared to + # a full clone. + clone_cmd = ['git', 'clone', '--depth', '10000', + 'https://github.com/llvm/llvm-project/', dir] + + # Try updating the current repo. + if RunCommand(fetch_cmd, fail_hard=False): + os.chdir(dir) + if RunCommand(checkout_cmd, fail_hard=False): + return + + # Otherwise, do a fresh clone. if os.path.isdir(dir): print("Removing %s." % dir) RmTree(dir) + if RunCommand(clone_cmd, fail_hard=False): + os.chdir(dir) + if RunCommand(checkout_cmd, fail_hard=False): + return - print("Retrying.") - RunCommand(command) + print('CheckoutLLVM failed.') + sys.exit(1) -def CheckoutRepos(args): - if args.skip_checkout: - return +def UrlOpen(url): + # Normally we'd use urllib, but on our bots it can't connect to the GitHub API + # due to using too old TLS (see crbug.com/897796#c56). As a horrible + # workaround, shell out to curl instead. It seems curl is recent enough on all + # our machines that it can connect. On Windows it's in our gnuwin package. + # TODO(crbug.com/965937): Use urllib once our Python is recent enough. + return subprocess.check_output(['curl', '--silent', url]) - Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR) - Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR) - Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR) - # Remove compiler-rt at old location. - if os.path.exists(os.path.join(LLVM_DIR, 'compiler-rt')): - RmTree(os.path.join(LLVM_DIR, 'compiler-rt')) - Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) - if sys.platform == 'darwin': - # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes - # (i.e. this is needed for bootstrap builds). - Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR) + +def GetLatestLLVMCommit(): + """Get the latest commit hash in the LLVM monorepo.""" + ref = json.loads(UrlOpen(('https://api.github.com/repos/' + 'llvm/llvm-project/git/refs/heads/master'))) + assert ref['object']['type'] == 'commit' + return ref['object']['sha'] + + +def GetSvnRevision(commit): + """Get the svn revision corresponding to a git commit in the LLVM repo.""" + commit = json.loads(UrlOpen(('https://api.github.com/repos/llvm/' + 'llvm-project/git/commits/' + commit))) + revision = re.search("llvm-svn: ([0-9]+)$", commit['message']).group(1) + return revision def DeleteChromeToolsShim(): @@ -180,17 +182,6 @@ f.write('endif (CHROMIUM_TOOLS_SRC)\n') -def AddSvnToPathOnWin(): - """Download svn.exe and add it to PATH.""" - if sys.platform != 'win32': - return - svn_ver = 'svn-1.6.6-win' - svn_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, svn_ver) - if not os.path.exists(svn_dir): - DownloadAndUnpack(CDS_URL + '/tools/%s.zip' % svn_ver, LLVM_BUILD_TOOLS_DIR) - os.environ['PATH'] = svn_dir + os.pathsep + os.environ.get('PATH', '') - - def AddCMakeToPath(args): """Download CMake and add it to PATH.""" if args.use_system_cmake: @@ -218,7 +209,7 @@ return gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin') - GNUWIN_VERSION = '10' + GNUWIN_VERSION = '11' GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp') if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION: print('GNU Win tools already up to date.') @@ -284,7 +275,7 @@ help='build with asserts disabled') parser.add_argument('--gcc-toolchain', help='(no longer used)') parser.add_argument('--lto-lld', action='store_true', - help='build lld with LTO') + help='build lld with LTO (only applies on Linux)') parser.add_argument('--llvm-force-head-revision', action='store_true', help='build the latest revision') parser.add_argument('--run-tests', action='store_true', @@ -314,6 +305,7 @@ print('--lto-lld requires --bootstrap') return 1 if args.lto_lld and not sys.platform.startswith('linux'): + # TODO(hans): Use it on Windows too. print('--lto-lld is only effective on Linux. Ignoring the option.') args.lto_lld = False if args.with_android and not os.path.exists(ANDROID_NDK_DIR): @@ -339,19 +331,17 @@ return 1 - # DEVELOPER_DIR needs to be set when Xcode isn't in a standard location - # and xcode-select wasn't run. This is needed for running clang and ld - # for the build done by this script, but it's also needed for running - # macOS system svn, so this needs to happen before calling functions using - # svn. - SetMacXcodePath() - - AddSvnToPathOnWin() + # The gnuwin package also includes curl, which is needed to interact with the + # github API below. + # TODO(crbug.com/965937): Use urllib once our Python is recent enough, and + # move this down to where we fetch other build tools. + AddGnuWinToPath() global CLANG_REVISION, PACKAGE_VERSION if args.llvm_force_head_revision: - CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL) - PACKAGE_VERSION = CLANG_REVISION + '-0' + CLANG_REVISION = GetLatestLLVMCommit() + PACKAGE_VERSION = '%s-%s-0' % (GetSvnRevision(CLANG_REVISION), + CLANG_REVISION[:8]) # Don't buffer stdout, so that print statements are immediately flushed. sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) @@ -360,15 +350,18 @@ WriteStampFile('', STAMP_FILE) WriteStampFile('', FORCE_HEAD_REVISION_FILE) - AddCMakeToPath(args) - AddGnuWinToPath() + # DEVELOPER_DIR needs to be set when Xcode isn't in a standard location + # and xcode-select wasn't run. + SetMacXcodePath() + AddCMakeToPath(args) DeleteChromeToolsShim() - CheckoutRepos(args) + if not args.skip_checkout: + CheckoutLLVM(CLANG_REVISION, LLVM_DIR); if args.skip_build: - return + return 0 cc, cxx = None, None @@ -377,34 +370,64 @@ ldflags = [] targets = 'AArch64;ARM;Mips;PowerPC;SystemZ;WebAssembly;X86' + + projects = 'clang;compiler-rt;lld;chrometools' + + if sys.platform == 'darwin': + # clang needs libc++, else -stdlib=libc++ won't find includes + # (this is needed for bootstrap builds and for building the fuchsia runtime) + projects += ';libcxx' + base_cmake_args = ['-GNinja', '-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=%s' % ('OFF' if args.disable_asserts else 'ON'), + '-DLLVM_ENABLE_PROJECTS=' + projects, + '-DLLVM_TARGETS_TO_BUILD=' + targets, '-DLLVM_ENABLE_PIC=OFF', '-DLLVM_ENABLE_UNWIND_TABLES=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', - '-DLLVM_TARGETS_TO_BUILD=' + targets, - # Statically link MSVCRT to avoid DLL dependencies. - '-DLLVM_USE_CRT_RELEASE=MT', '-DCLANG_PLUGIN_SUPPORT=OFF', '-DCLANG_ENABLE_STATIC_ANALYZER=OFF', '-DCLANG_ENABLE_ARCMT=OFF', # TODO(crbug.com/929645): Use newer toolchain to host. '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON', '-DBUG_REPORT_URL=' + BUG_REPORT_URL, + # See PR41956: Don't link libcxx into libfuzzer. + '-DCOMPILER_RT_USE_LIBCXX=NO', ] + if sys.platform == 'win32': + base_cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT') + + if sys.platform == 'darwin': + # Use the system libc++abi. + # TODO(hans): use https://reviews.llvm.org/D62060 instead + base_cmake_args.append('-DLIBCXX_CXX_ABI=libcxxabi') + base_cmake_args.append('-DLIBCXX_CXX_ABI_SYSTEM=1') + if sys.platform != 'win32': # libxml2 is required by the Win manifest merging tool used in cross-builds. base_cmake_args.append('-DLLVM_ENABLE_LIBXML2=FORCE_ON') if args.bootstrap: print('Building bootstrap compiler') + if os.path.exists(LLVM_BOOTSTRAP_DIR): + RmTree(LLVM_BOOTSTRAP_DIR) EnsureDirExists(LLVM_BOOTSTRAP_DIR) os.chdir(LLVM_BOOTSTRAP_DIR) + + projects = 'clang' + if args.lto_lld: + projects += ';lld' + if sys.platform == 'darwin': + # Need libc++ and compiler-rt for the bootstrap compiler on mac. + projects += ';libcxx;compiler-rt' + bootstrap_args = base_cmake_args + [ + # Need ARM and AArch64 for building the ios clang_rt. '-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64', + '-DLLVM_ENABLE_PROJECTS=' + projects, '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, '-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), @@ -428,15 +451,10 @@ "-DCOMPILER_RT_ENABLE_WATCHOS=OFF", "-DCOMPILER_RT_ENABLE_TVOS=OFF", ]) - else: - # On non-darwin, the bootstrap toolchain doesn't need compiler-rt, - # so don't build it during bootstrap. If we ever do need it, - # consider setting COMPILER_RT_DEFAULT_TARGET_ONLY. - bootstrap_args.append("-DLLVM_TOOL_COMPILER_RT_BUILD=OFF") if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) - RmCmakeCache('.') - RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') + RunCommand(['cmake'] + bootstrap_args + [os.path.join(LLVM_DIR, 'llvm')], + msvc_arch='x64') RunCommand(['ninja'], msvc_arch='x64') if args.run_tests: if sys.platform == 'win32': @@ -455,7 +473,7 @@ cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') - print('Building final compiler') + print('Bootstrap compiler installed; building final compiler.') # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is # needed, on OS X it requires libc++. clang only automatically links to libc++ @@ -529,20 +547,16 @@ '-DLLVM_ENABLE_LTO=thin', '-DLLVM_USE_LINKER=lld'] - RmCmakeCache('.') - RunCommand(['cmake'] + threads_enabled_cmake_args + [LLVM_DIR], + RunCommand(['cmake'] + threads_enabled_cmake_args + + [os.path.join(LLVM_DIR, 'llvm')], msvc_arch='x64', env=deployment_env) RunCommand(['ninja'] + tools_with_threading, msvc_arch='x64') - # Build clang and other tools. - CreateChromeToolsShim() - - cmake_args = [] - if cc is not None: base_cmake_args.append('-DCMAKE_C_COMPILER=' + cc) - if cxx is not None: base_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx) default_tools = ['plugins', 'blink_gc_plugin', 'translation_unit'] chrome_tools = list(set(default_tools + args.extra_tools)) - cmake_args += base_cmake_args + [ + if cc is not None: base_cmake_args.append('-DCMAKE_C_COMPILER=' + cc) + if cxx is not None: base_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx) + cmake_args = base_cmake_args + [ '-DLLVM_ENABLE_THREADS=OFF', '-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), @@ -556,10 +570,14 @@ cmake_args += ['-DCOMPILER_RT_ENABLE_IOS=ON', '-DSANITIZER_MIN_OSX_VERSION=10.7'] + # TODO(crbug.com/41866): Use -DLLVM_EXTERNAL_PROJECTS instead. + CreateChromeToolsShim() + + if os.path.exists(LLVM_BUILD_DIR): + RmTree(LLVM_BUILD_DIR) EnsureDirExists(LLVM_BUILD_DIR) os.chdir(LLVM_BUILD_DIR) - RmCmakeCache('.') - RunCommand(['cmake'] + cmake_args + [LLVM_DIR], + RunCommand(['cmake'] + cmake_args + [os.path.join(LLVM_DIR, 'llvm')], msvc_arch='x64', env=deployment_env) RunCommand(['ninja'], msvc_arch='x64') @@ -590,9 +608,8 @@ rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', RELEASE_VERSION, 'lib', platform) - # Do an out-of-tree build of compiler-rt. - # On Windows, this is used to get the 32-bit ASan run-time. - # TODO(hans): Remove once the regular build above produces this. + # Do an out-of-tree build of compiler-rt for 32-bit Win ASan. + # TODO(hans): Do we use 32-bit ASan? Can we drop it? if sys.platform == 'win32': if os.path.isdir(COMPILER_RT_BUILD_DIR): RmTree(COMPILER_RT_BUILD_DIR) @@ -606,14 +623,12 @@ '-DLLVM_ENABLE_THREADS=OFF', '-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] - RmCmakeCache('.') - RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR], + RunCommand(['cmake'] + compiler_rt_args + + [os.path.join(LLVM_DIR, 'llvm')], msvc_arch='x86', env=deployment_env) RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') # Copy select output to the main tree. - # TODO(hans): Make this (and the .gypi and .isolate files) version number - # independent. rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', RELEASE_VERSION, 'lib', platform) # Blacklists: @@ -673,7 +688,6 @@ '-DSANITIZER_CXX_ABI=libcxxabi', '-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-u__cxa_demangle', '-DANDROID=1'] - RmCmakeCache('.') RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR]) # We use ASan i686 build for fuzzing. @@ -728,7 +742,6 @@ '-DCMAKE_C_COMPILER_WORKS=ON', '-DCMAKE_ASM_COMPILER_WORKS=ON', ] - RmCmakeCache('.') RunCommand(['cmake'] + fuchsia_args + [os.path.join(COMPILER_RT_DIR, 'lib', 'builtins')]) @@ -745,17 +758,16 @@ # Run tests. if args.run_tests or args.llvm_force_head_revision: - os.chdir(LLVM_BUILD_DIR) - RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64') + RunCommand(['ninja', '-C', LLVM_BUILD_DIR, 'cr-check-all'], msvc_arch='x64') + if args.run_tests: if sys.platform == 'win32': CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin')) - os.chdir(LLVM_BUILD_DIR) test_targets = [ 'check-all' ] if sys.platform == 'darwin': # TODO(thakis): Run check-all on Darwin too, https://crbug.com/959361 test_targets = [ 'check-llvm', 'check-clang', 'check-lld' ] - RunCommand(['ninja'] + test_targets, msvc_arch='x64') + RunCommand(['ninja', '-C', LLVM_BUILD_DIR] + test_targets, msvc_arch='x64') if sys.platform == 'darwin': for dylib in glob.glob(os.path.join(rt_lib_dst_dir, '*.dylib')): @@ -770,7 +782,6 @@ WriteStampFile(PACKAGE_VERSION, STAMP_FILE) WriteStampFile(PACKAGE_VERSION, FORCE_HEAD_REVISION_FILE) - print('Clang build was successful.') return 0
diff --git a/tools/clang/scripts/package.py b/tools/clang/scripts/package.py index ebf486641..ae8a0fd 100755 --- a/tools/clang/scripts/package.py +++ b/tools/clang/scripts/package.py
@@ -15,6 +15,8 @@ import sys import tarfile +from update import RELEASE_VERSION + # Path constants. THIS_DIR = os.path.dirname(__file__) CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) @@ -175,28 +177,15 @@ platform = 'Linux_x64' with open('buildlog.txt', 'w') as log: - Tee('Diff in llvm:\n', log) - TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) - TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) - Tee('Diff in llvm/tools/clang:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], - log, fail_hard=False) - # TODO(thakis): compiler-rt is in projects/compiler-rt on Windows but - # llvm/compiler-rt elsewhere. So this diff call is currently only right on - # Windows. - Tee('Diff in llvm/compiler-rt:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], - log, fail_hard=False) - Tee('Diff in llvm/projects/libcxx:\n', log) - TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')], - log, fail_hard=False) - TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')], - log, fail_hard=False) - + if os.path.exists(LLVM_DIR): + Tee('Diff in llvm:\n', log) + cwd = os.getcwd() + os.chdir(LLVM_DIR) + TeeCmd(['git', 'status'], log, fail_hard=False) + TeeCmd(['git', 'diff'], log, fail_hard=False) + os.chdir(cwd) + else: + Tee('No previous llvm checkout.\n', log) Tee('Starting build\n', log) # Do a clobber build. @@ -221,149 +210,165 @@ # Copy a whitelist of files to the directory we're going to tar up. # This supports the same patterns that the fnmatch module understands. + # '$V' is replaced by RELEASE_VERSION further down. exe_ext = '.exe' if sys.platform == 'win32' else '' - want = ['bin/llvm-pdbutil' + exe_ext, - 'bin/llvm-symbolizer' + exe_ext, - 'bin/llvm-undname' + exe_ext, - # Copy built-in headers (lib/clang/3.x.y/include). - 'lib/clang/*/include/*', - 'lib/clang/*/share/asan_blacklist.txt', - 'lib/clang/*/share/cfi_blacklist.txt', - ] + want = [ + 'bin/llvm-pdbutil' + exe_ext, + 'bin/llvm-symbolizer' + exe_ext, + 'bin/llvm-undname' + exe_ext, + # Copy built-in headers (lib/clang/3.x.y/include). + 'lib/clang/$V/include/*', + 'lib/clang/$V/share/asan_blacklist.txt', + 'lib/clang/$V/share/cfi_blacklist.txt', + ] if sys.platform == 'win32': - want.append('bin/clang-cl.exe') - want.append('bin/lld-link.exe') + want.extend([ + 'bin/clang-cl.exe', + 'bin/lld-link.exe', + ]) else: - want.append('bin/clang') + want.extend([ + 'bin/clang', + + # Include libclang_rt.builtins.a for Fuchsia targets. + 'lib/clang/$V/aarch64-fuchsia/lib/libclang_rt.builtins.a', + 'lib/clang/$V/x86_64-fuchsia/lib/libclang_rt.builtins.a', + ]) if sys.platform == 'darwin': want.extend([ - # AddressSanitizer runtime. - 'lib/clang/*/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', - 'lib/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', + # AddressSanitizer runtime. + 'lib/clang/$V/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', + 'lib/clang/$V/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/darwin/libclang_rt.fuzzer_no_main_osx.a', + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/darwin/libclang_rt.fuzzer_no_main_osx.a', - # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the - # _IsOSVersionAtLeast runtime function. - 'lib/clang/*/lib/darwin/libclang_rt.ios.a', - 'lib/clang/*/lib/darwin/libclang_rt.osx.a', + # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the + # _IsOSVersionAtLeast runtime function. + 'lib/clang/$V/lib/darwin/libclang_rt.ios.a', + 'lib/clang/$V/lib/darwin/libclang_rt.osx.a', - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/darwin/libclang_rt.profile_iossim.a', - 'lib/clang/*/lib/darwin/libclang_rt.profile_osx.a', + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/darwin/libclang_rt.profile_iossim.a', + 'lib/clang/$V/lib/darwin/libclang_rt.profile_osx.a', ]) elif sys.platform.startswith('linux'): - # Add llvm-ar and lld for LTO. - want.append('bin/llvm-ar') - want.append('bin/lld') want.extend([ - # AddressSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.asan-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.asan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.asan-x86_64.a.syms', + 'bin/lld', - # AddressSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.asan_cxx-x86_64.a.syms', + # Add llvm-ar for LTO. + 'bin/llvm-ar', - # AddressSanitizer Android runtime. - 'lib/clang/*/lib/linux/libclang_rt.asan-aarch64-android.so', - 'lib/clang/*/lib/linux/libclang_rt.asan-arm-android.so', - 'lib/clang/*/lib/linux/libclang_rt.asan-i686-android.so', + # AddressSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.asan-i386.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan-x86_64.a.syms', - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a', + # AddressSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.asan_cxx-i386.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.asan_cxx-x86_64.a.syms', - # HWASAN Android runtime. - 'lib/clang/*/lib/linux/libclang_rt.hwasan-aarch64-android.so', + # AddressSanitizer Android runtime. + 'lib/clang/$V/lib/linux/libclang_rt.asan-aarch64-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.asan-arm-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.asan-i686-android.so', - # MemorySanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.msan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.msan-x86_64.a.syms', + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a', - # MemorySanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.msan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.msan_cxx-x86_64.a.syms', + # HWASAN Android runtime. + 'lib/clang/$V/lib/linux/libclang_rt.hwasan-aarch64-android.so', - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/linux/libclang_rt.profile-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-aarch64-android.a', - 'lib/clang/*/lib/linux/libclang_rt.profile-arm-android.a', + # MemorySanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.msan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.msan-x86_64.a.syms', - # ThreadSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.tsan-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.tsan-x86_64.a.syms', + # MemorySanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.msan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.msan_cxx-x86_64.a.syms', - # ThreadSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.tsan_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.tsan_cxx-x86_64.a.syms', + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/linux/libclang_rt.profile-i386.a', + 'lib/clang/$V/lib/linux/libclang_rt.profile-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.profile-aarch64-android.a', + 'lib/clang/$V/lib/linux/libclang_rt.profile-arm-android.a', - # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-x86_64.a.syms', + # ThreadSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.tsan-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.tsan-x86_64.a.syms', - # UndefinedBehaviorSanitizer C++ runtime. - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-i386.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a.syms', + # ThreadSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.tsan_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.tsan_cxx-x86_64.a.syms', - # UndefinedBehaviorSanitizer Android runtime, needed for CFI. - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-aarch64-android.so', - 'lib/clang/*/lib/linux/libclang_rt.ubsan_standalone-arm-android.so', + # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-i386.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-x86_64.a.syms', - # Blacklist for MemorySanitizer (used on Linux only). - 'lib/clang/*/share/msan_blacklist.txt', + # UndefinedBehaviorSanitizer C++ runtime. + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone_cxx-i386.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone_cxx-x86_64.a.syms', + + # UndefinedBehaviorSanitizer Android runtime, needed for CFI. + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-aarch64-android.so', + 'lib/clang/$V/lib/linux/libclang_rt.ubsan_standalone-arm-android.so', + + # Blacklist for MemorySanitizer (used on Linux only). + 'lib/clang/$V/share/msan_blacklist.txt', ]) elif sys.platform == 'win32': want.extend([ - # AddressSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/windows/clang_rt.asan-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan-x86_64.lib', + # AddressSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/windows/clang_rt.asan-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.asan-x86_64.lib', - # AddressSanitizer C++ runtime. - 'lib/clang/*/lib/windows/clang_rt.asan_cxx-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_cxx-x86_64.lib', + # AddressSanitizer C++ runtime. + 'lib/clang/$V/lib/windows/clang_rt.asan_cxx-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.asan_cxx-x86_64.lib', - # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). - 'lib/clang/*/lib/windows/clang_rt.fuzzer_no_main-x86_64.lib', + # Fuzzing instrumentation (-fsanitize=fuzzer-no-link). + 'lib/clang/$V/lib/windows/clang_rt.fuzzer_no_main-x86_64.lib', - # Thunk for AddressSanitizer needed for static build of a shared lib. - 'lib/clang/*/lib/windows/clang_rt.asan_dll_thunk-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dll_thunk-x86_64.lib', + # Thunk for AddressSanitizer needed for static build of a shared lib. + 'lib/clang/$V/lib/windows/clang_rt.asan_dll_thunk-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.asan_dll_thunk-x86_64.lib', - # AddressSanitizer runtime for component build. - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-i386.dll', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-x86_64.dll', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic-x86_64.lib', + # AddressSanitizer runtime for component build. + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-i386.dll', + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.dll', + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic-x86_64.lib', - # Thunk for AddressSanitizer for component build of a shared lib. - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic_runtime_thunk-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.asan_dynamic_runtime_thunk-x86_64.lib', + # Thunk for AddressSanitizer for component build of a shared lib. + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic_runtime_thunk-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.asan_dynamic_runtime_thunk-x86_64.lib', - # Profile runtime (used by profiler and code coverage). - 'lib/clang/*/lib/windows/clang_rt.profile-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.profile-x86_64.lib', + # Profile runtime (used by profiler and code coverage). + 'lib/clang/$V/lib/windows/clang_rt.profile-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.profile-x86_64.lib', - # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', + # UndefinedBehaviorSanitizer C runtime (pure C won't link with *_cxx). + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone-x86_64.lib', - # UndefinedBehaviorSanitizer C++ runtime. - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone_cxx-i386.lib', - 'lib/clang/*/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', + # UndefinedBehaviorSanitizer C++ runtime. + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone_cxx-i386.lib', + 'lib/clang/$V/lib/windows/clang_rt.ubsan_standalone_cxx-x86_64.lib', ]) - if sys.platform in ('linux2', 'darwin'): - # Include libclang_rt.builtins.a for Fuchsia targets. - want.extend(['lib/clang/*/aarch64-fuchsia/lib/libclang_rt.builtins.a', - 'lib/clang/*/x86_64-fuchsia/lib/libclang_rt.builtins.a', - ]) + # Check all non-glob wanted files exist on disk. + want = [w.replace('$V', RELEASE_VERSION) for w in want] + for w in want: + if '*' in w: continue + if os.path.exists(os.path.join(LLVM_RELEASE_DIR, w)): continue + print >>sys.stderr, 'wanted file "%s" but it did not exist' % w + return 1 + # TODO(thakis): Try walking over want and copying the files in there instead + # of walking the directory and doing fnmatch() against want. for root, dirs, files in os.walk(LLVM_RELEASE_DIR): # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/... rel_root = root[len(LLVM_RELEASE_DIR)+1:]
diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py index 9d45020..876fb66d1 100755 --- a/tools/clang/scripts/update.py +++ b/tools/clang/scripts/update.py
@@ -36,12 +36,15 @@ # Do NOT CHANGE this if you don't know what you're doing -- see # https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md # Reverting problematic clang rolls is safe, though. -CLANG_REVISION = '361212' -CLANG_SUB_REVISION = 1 +CLANG_REVISION = '67510fac36d27b2e22c7cd955fc167136b737b93' +CLANG_SVN_REVISION = '361212' +CLANG_SUB_REVISION = 2 -PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) +PACKAGE_VERSION = '%s-%s-%s' % (CLANG_SVN_REVISION, CLANG_REVISION[:8], + CLANG_SUB_REVISION) RELEASE_VERSION = '9.0.0' + CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', 'https://commondatastorage.googleapis.com/chromium-browser-clang')
diff --git a/tools/clang/scripts/upload_revision.py b/tools/clang/scripts/upload_revision.py index 22dfeda0..33f1ea7 100755 --- a/tools/clang/scripts/upload_revision.py +++ b/tools/clang/scripts/upload_revision.py
@@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""This script takes a Clang revision as an argument, it then +"""This script takes a Clang git revision as an argument, it then creates a feature branch, puts this revision into update.py, uploads a CL, triggers Clang Upload try bots, and tells what to do next""" @@ -16,6 +16,8 @@ import subprocess import sys +from build import GetSvnRevision + # Path constants. THIS_DIR = os.path.dirname(__file__) UPDATE_PY_PATH = os.path.join(THIS_DIR, "update.py") @@ -23,13 +25,17 @@ is_win = sys.platform.startswith('win32') -def PatchRevision(clang_revision, clang_sub_revision): +def PatchRevision(clang_git_revision, clang_svn_revision, clang_sub_revision): with open(UPDATE_PY_PATH, 'rb') as f: content = f.read() - m = re.search("CLANG_REVISION = '([0-9]+)'", content) + m = re.search("CLANG_SVN_REVISION = '([0-9]+)'", content) clang_old_revision = m.group(1) - content = re.sub("CLANG_REVISION = '[0-9]+'", - "CLANG_REVISION = '{}'".format(clang_revision), + + content = re.sub("CLANG_REVISION = '[0-9a-f]+'", + "CLANG_REVISION = '{}'".format(clang_git_revision), + content, count=1) + content = re.sub("CLANG_SVN_REVISION = '[0-9]+'", + "CLANG_SVN_REVISION = '{}'".format(clang_svn_revision), content, count=1) content = re.sub("CLANG_SUB_REVISION = [0-9]+", "CLANG_SUB_REVISION = {}".format(clang_sub_revision), @@ -45,32 +51,38 @@ def main(): parser = argparse.ArgumentParser(description='upload new clang revision') - parser.add_argument('clang_revision', metavar='CLANG_REVISION', - type=int, nargs=1, - help='Clang revision to build the toolchain for.') - parser.add_argument('clang_sub_revision', metavar='CLANG_SUB_REVISION', + parser.add_argument('clang_git_revision', nargs=1, + help='Clang git revision to build the toolchain for.') + parser.add_argument('clang_sub_revision', type=int, nargs='?', default=1, help='Clang sub-revision to build the toolchain for.') args = parser.parse_args() - clang_revision = args.clang_revision[0] + clang_git_revision = args.clang_git_revision[0] + clang_svn_revision = GetSvnRevision(clang_git_revision) clang_sub_revision = args.clang_sub_revision + # Needs shell=True on Windows due to git.bat in depot_tools. git_revision = subprocess.check_output( ["git", "rev-parse", "origin/master"], shell=is_win).strip() - print "Making a patch for Clang revision r{}-{}".format( - clang_revision, clang_sub_revision) - print "Chrome revision: {}".format(git_revision) - clang_old_revision = PatchRevision(clang_revision, clang_sub_revision) - Git(["checkout", "-b", "clang-{}-{}".format( - clang_revision, clang_sub_revision)]) + print "Making a patch for Clang {}-{}-{}".format(clang_svn_revision, + clang_git_revision[:8], + clang_sub_revision) + print "Chrome revision: {}".format(git_revision) + + clang_old_revision = PatchRevision(clang_git_revision, clang_svn_revision, + clang_sub_revision) + + Git(["checkout", "-b", "clang-{}-{}-{}".format(clang_svn_revision, + clang_git_revision[:8], + clang_sub_revision)]) Git(["add", UPDATE_PY_PATH]) commit_message = 'Ran `{}`.'.format(' '.join(sys.argv)) Git(["commit", "-m", "Roll clang {}:{}.\n\n{}".format( - clang_old_revision, clang_revision, commit_message)]) + clang_old_revision, clang_svn_revision, commit_message)]) Git(["cl", "upload", "-f"]) Git(["cl", "try", "-b", "linux_upload_clang", "-r", git_revision])