| #!/usr/bin/python |
| # Copyright (c) 2012 The Native Client Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Recipes for NativeClient toolchain packages. |
| |
| The real entry plumbing is in toolchain_main.py. |
| """ |
| |
| import collections |
| import fnmatch |
| import platform |
| import os |
| import re |
| import sys |
| |
| sys.path.append(os.path.join(os.path.dirname(__file__), '..')) |
| import pynacl.gsd_storage |
| import pynacl.platform |
| |
| import command |
| import toolchain_main |
| |
| |
| SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| NACL_DIR = os.path.dirname(SCRIPT_DIR) |
| |
| |
| # Used here and in toolchain_build_bionic.py |
| GCC_VERSION = '4.9.2' |
| |
| |
| # See command.GenerateGitPatches for the schema of entries in this dict. |
| # Additionally, each may contain a 'repo' key whose value is the name |
| # to use in place of the package name when calling GitUrl (below). |
| GIT_REVISIONS = { |
| 'binutils': { |
| 'rev': '68b975af7ef47a9d28f21f4c93431f35777a5109', |
| 'upstream-branch': 'upstream/binutils-2_25-branch', |
| 'upstream-name': 'binutils-2.25', |
| # This is tag binutils-2_25, but Gerrit won't let us push |
| # non-annotated tags, and the upstream tag is not annotated. |
| 'upstream-base': '68b975af7ef47a9d28f21f4c93431f35777a5109', |
| }, |
| 'gcc': { |
| 'rev': 'b23dd79950a5453d3b3b5a0030d7a1894cafcffe', |
| 'upstream-branch': 'upstream/gcc-4_9-branch', |
| 'upstream-name': 'gcc-' + GCC_VERSION, |
| # Upstream tag gcc-<GCC_VERSION>-release: |
| 'upstream-base': 'c1283af40b65f1ad862cf5b27e2d9ed10b2076b6', |
| }, |
| 'newlib': { |
| 'rev': 'bf66148d14c7fca26b9198dd5dc81e743893bb66', |
| 'upstream-branch': 'upstream/master', |
| 'upstream-name': 'newlib-2.1.0', |
| # Upstream tag newlib_2_1_0: |
| 'upstream-base': '99fc6c167467b41466ec90e8260e9c49cbe3d13c', |
| }, |
| 'gdb': { |
| 'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8', |
| 'repo': 'binutils', |
| 'upstream-branch': 'upstream/gdb-7.7-branch', |
| 'upstream-name': 'gdb-7.7.1', |
| # Upstream tag gdb-7.7-release: |
| 'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b', |
| }, |
| } |
| |
| TAR_FILES = { |
| 'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'), |
| 'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'), |
| 'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'), |
| 'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'), |
| 'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'), |
| 'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'), |
| } |
| |
| GIT_BASE_URL = 'https://chromium.googlesource.com/native_client' |
| GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client' |
| |
| ALT_GIT_BASE_URL = 'https://chromium.googlesource.com/a/native_client' |
| |
| KNOWN_MIRRORS = [('http://git.chromium.org/native_client', GIT_BASE_URL)] |
| PUSH_MIRRORS = [('http://git.chromium.org/native_client', GIT_PUSH_URL), |
| (ALT_GIT_BASE_URL, GIT_PUSH_URL), |
| (GIT_BASE_URL, GIT_PUSH_URL)] |
| |
| |
| def GitUrl(package, push_url=False): |
| repo = GIT_REVISIONS[package].get('repo', package) |
| if push_url: |
| base_url = GIT_PUSH_URL |
| else: |
| base_url = GIT_BASE_URL |
| |
| return '%s/nacl-%s.git' % (base_url, repo) |
| |
| |
| def CollectSources(): |
| sources = {} |
| |
| for package in TAR_FILES: |
| tar_file = TAR_FILES[package] |
| if fnmatch.fnmatch(tar_file, '*.bz2'): |
| extract = EXTRACT_STRIP_TBZ2 |
| elif fnmatch.fnmatch(tar_file, '*.gz'): |
| extract = EXTRACT_STRIP_TGZ |
| else: |
| raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package) |
| sources[package] = { |
| 'type': 'source', |
| 'commands': [ |
| command.Command(extract + [command.path.join('%(abs_top_srcdir)s', |
| '..', 'third_party', |
| tar_file)], |
| cwd='%(output)s'), |
| ], |
| } |
| |
| patch_packages = [] |
| patch_commands = [] |
| for package, info in GIT_REVISIONS.iteritems(): |
| sources[package] = { |
| 'type': 'source', |
| 'commands': command.SyncGitRepoCmds(GitUrl(package), '%(output)s', |
| info['rev'], |
| git_cache='%(git_cache_dir)s', |
| push_url=GitUrl(package, True), |
| known_mirrors=KNOWN_MIRRORS, |
| push_mirrors=PUSH_MIRRORS), |
| } |
| patch_packages.append(package) |
| patch_info = {'name': package} |
| patch_info.update(info) |
| patch_commands.append( |
| command.GenerateGitPatches('%(' + package + ')s/.git', patch_info)) |
| |
| sources['patches'] = { |
| 'type': 'build', |
| 'dependencies': patch_packages, |
| 'commands': patch_commands, |
| } |
| |
| # The gcc_libs component gets the whole GCC source tree. |
| sources['gcc_libs'] = sources['gcc'] |
| |
| # The gcc component omits all the source directories that are used solely |
| # for building target libraries. We don't want those included in the |
| # input hash calculation so that we don't rebuild the compiler when the |
| # the only things that have changed are target libraries. |
| sources['gcc'] = { |
| 'type': 'source', |
| 'dependencies': ['gcc_libs'], |
| 'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [ |
| 'boehm-gc', |
| 'libada', |
| 'libatomic', |
| 'libffi', |
| 'libgcc', |
| 'libgfortran', |
| 'libgo', |
| 'libgomp', |
| 'libitm', |
| 'libjava', |
| 'libmudflap', |
| 'libobjc', |
| 'libquadmath', |
| 'libsanitizer', |
| 'libssp', |
| 'libstdc++-v3', |
| ])] |
| } |
| |
| # We have to populate the newlib source tree with the "exported" form of |
| # some headers from the native_client source tree. The newlib build |
| # needs these to be in the expected place. By doing this in the source |
| # target, these files will be part of the input hash and so we don't need |
| # to do anything else to keep track of when they might have changed in |
| # the native_client source tree. |
| newlib_sys_nacl = command.path.join('%(output)s', |
| 'newlib', 'libc', 'sys', 'nacl') |
| newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl, |
| dirname)) |
| for dirname in ['bits', 'sys', 'machine']] |
| newlib_unpack.append(command.Command([ |
| 'python', |
| command.path.join('%(top_srcdir)s', 'src', |
| 'trusted', 'service_runtime', 'export_header.py'), |
| command.path.join('%(top_srcdir)s', 'src', |
| 'trusted', 'service_runtime', 'include'), |
| newlib_sys_nacl, |
| ])) |
| sources['newlib']['commands'] += newlib_unpack |
| |
| return sources |
| |
| |
| # Canonical tuples we use for hosts. |
| WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-32') |
| MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64') |
| ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm') |
| LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32') |
| LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64') |
| |
| # Map of native host tuple to extra tuples that it cross-builds for. |
| EXTRA_HOSTS_MAP = { |
| LINUX_X86_64_TUPLE: [ |
| LINUX_X86_32_TUPLE, |
| ARM_HOST_TUPLE, |
| WINDOWS_HOST_TUPLE, |
| ], |
| } |
| |
| # Map of native host tuple to host tuples that are "native enough". |
| # For these hosts, we will do a native-style build even though it's |
| # not the native tuple, just passing some extra compiler flags. |
| NATIVE_ENOUGH_MAP = { |
| LINUX_X86_64_TUPLE: { |
| LINUX_X86_32_TUPLE: ['-m32'], |
| }, |
| } |
| |
| # The list of targets to build toolchains for. |
| TARGET_LIST = ['arm', 'i686'] |
| |
| # List upload targets for each host we want to upload packages for. |
| TARGET = collections.namedtuple('TARGET', ['name', 'pkg_prefix']) |
| HOST_TARGET = collections.namedtuple('HOST_TARGET', |
| ['os', 'arch', 'differ3264', 'targets']) |
| |
| STANDARD_TARGETS = [TARGET('arm', '')] |
| LINUX_X86_64_TARGETS = [TARGET('arm', ''), TARGET('i686', 'ng_')] |
| |
| UPLOAD_HOST_TARGETS = [ |
| HOST_TARGET('win', 'x86-32', False, STANDARD_TARGETS), |
| HOST_TARGET('darwin', 'x86-64', False, STANDARD_TARGETS), |
| HOST_TARGET('linux', 'arm', False, STANDARD_TARGETS), |
| HOST_TARGET('linux', 'x86-32', False, STANDARD_TARGETS), |
| HOST_TARGET('linux', 'x86-64', True, LINUX_X86_64_TARGETS), |
| ] |
| |
| # GDB is built by toolchain_build but injected into package targets built by |
| # other means. List out what package targets, packages, and the tar file we are |
| # injecting on top of here. |
| GDB_INJECT_HOSTS = [ |
| ('win', 'x86-32'), |
| ('darwin', 'x86-64'), |
| ('linux', 'x86-32'), |
| ] |
| |
| GDB_INJECT_PACKAGES = [ |
| ('nacl_x86_newlib', ['core_sdk.tgz', 'naclsdk.tgz']), |
| ('nacl_x86_glibc', ['core_sdk.tar.bz2', 'toolchain.tar.bz2']), |
| ('nacl_x86_newlib_raw', ['naclsdk.tgz']), |
| ('nacl_x86_glibc_raw', ['toolchain.tar.bz2']), |
| ] |
| |
| # These are extra arguments to pass gcc's configure that vary by target. |
| TARGET_GCC_CONFIG = { |
| 'arm': ['--with-tune=cortex-a15'], |
| } |
| |
| PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]' |
| BUG_URL = 'http://gonacl.com/reportissue' |
| |
| TAR_XV = ['tar', '-x', '-v'] |
| EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f'] |
| EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f'] |
| CONFIGURE_CMD = ['sh', '%(src)s/configure'] |
| MAKE_PARALLEL_CMD = ['make', '-j%(cores)s'] |
| MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check'] |
| MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s'] |
| |
| # This file gets installed by multiple packages' install steps, but it is |
| # never useful when installed in isolation. So we remove it from the |
| # installation directories before packaging up. |
| REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s', |
| 'share', 'info', 'dir')) |
| |
| def ConfigureHostArch(host): |
| configure_args = [] |
| |
| is_cross = CrossCompiling(host) |
| |
| if is_cross: |
| extra_cc_args = [] |
| configure_args.append('--host=' + host) |
| else: |
| extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, []) |
| if extra_cc_args: |
| # The host we've chosen is "native enough", such as x86-32 on x86-64. |
| # But it's not what config.guess will yield, so we need to supply |
| # a --build switch to ensure things build correctly. |
| configure_args.append('--build=' + host) |
| |
| extra_cxx_args = list(extra_cc_args) |
| if fnmatch.fnmatch(host, '*-linux*'): |
| # Avoid shipping binaries with a runtime dependency on |
| # a particular version of the libstdc++ shared library. |
| # TODO(mcgrathr): Do we want this for MinGW and/or Mac too? |
| extra_cxx_args.append('-static-libstdc++') |
| |
| if extra_cc_args: |
| # These are the defaults when there is no setting, but we will add |
| # additional switches, so we must supply the command name too. |
| if is_cross: |
| cc = host + '-gcc' |
| else: |
| cc = 'gcc' |
| configure_args.append('CC=' + ' '.join([cc] + extra_cc_args)) |
| |
| if extra_cxx_args: |
| # These are the defaults when there is no setting, but we will add |
| # additional switches, so we must supply the command name too. |
| if is_cross: |
| cxx = host + '-g++' |
| else: |
| cxx = 'g++' |
| configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args)) |
| |
| if HostIsWindows(host): |
| # The i18n support brings in runtime dependencies on MinGW DLLs |
| # that we don't want to have to distribute alongside our binaries. |
| # So just disable it, and compiler messages will always be in US English. |
| configure_args.append('--disable-nls') |
| |
| return configure_args |
| |
| |
| def ConfigureHostCommon(host): |
| return ConfigureHostArch(host) + [ |
| '--prefix=', |
| '--disable-silent-rules', |
| '--without-gcc-arch', |
| ] |
| |
| |
| def ConfigureHostLib(host): |
| return ConfigureHostCommon(host) + [ |
| '--disable-shared', |
| ] |
| |
| |
| def ConfigureHostTool(host): |
| return ConfigureHostCommon(host) + [ |
| '--with-pkgversion=' + PACKAGE_NAME, |
| '--with-bugurl=' + BUG_URL, |
| '--without-zlib', |
| ] |
| |
| |
| def MakeCommand(host, extra_args=[]): |
| if HostIsWindows(host): |
| # There appears to be nothing we can pass at top-level configure time |
| # that will prevent the configure scripts from finding MinGW's libiconv |
| # and using it. We have to force this variable into the environment |
| # of the sub-configure runs, which are run via make. |
| make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no'] |
| else: |
| make_command = MAKE_PARALLEL_CMD |
| return make_command + extra_args |
| |
| |
| # Return the 'make check' command to run. |
| # When cross-compiling, don't try to run test suites. |
| def MakeCheckCommand(host): |
| if CrossCompiling(host): |
| return ['true'] |
| return MAKE_CHECK_CMD |
| |
| |
| def InstallDocFiles(subdir, files): |
| doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir) |
| dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file)) |
| for file in files])) |
| commands = ([command.Mkdir(dir, parents=True) for dir in dirs] + |
| [command.Copy(command.path.join('%(' + subdir + ')s', file), |
| command.path.join(doc_dir, file)) |
| for file in files]) |
| return commands |
| |
| |
| def NewlibLibcScript(arch, elfclass_x86_64='elf32'): |
| template = """/* |
| * This is a linker script that gets installed as libc.a for the |
| * newlib-based NaCl toolchain. It brings in the constituent |
| * libraries that make up what -lc means semantically. |
| */ |
| OUTPUT_FORMAT(%s) |
| GROUP ( libnacl.a libcrt_common.a ) |
| """ |
| if arch == 'arm': |
| # Listing three formats instead of one makes -EL/-EB switches work |
| # for the endian-switchable ARM backend. |
| format_list = ['elf32-littlearm-nacl', |
| 'elf32-bigarm-nacl', |
| 'elf32-littlearm-nacl'] |
| elif arch == 'i686': |
| format_list = ['elf32-i386-nacl'] |
| elif arch == 'x86_64': |
| format_list = ['%s-x86-64-nacl' % elfclass_x86_64] |
| else: |
| raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch) |
| return template % ', '.join(['"' + fmt + '"' for fmt in format_list]) |
| |
| |
| # The default strip behavior removes debugging and symbol table |
| # sections, but it leaves the .comment section. This contains the |
| # compiler version string, and so it changes when the compiler changes |
| # even if the actual machine code it produces is completely identical. |
| # Hence, the target library packages will always change when the |
| # compiler changes unless these sections are removed. Doing this |
| # requires somehow teaching the makefile rules to pass the |
| # --remove-section=.comment switch to TARGET-strip. For the GCC |
| # target libraries, setting STRIP_FOR_TARGET is sufficient. But |
| # quoting nightmares make it difficult to pass a command with a space |
| # in it as the STRIP_FOR_TARGET value. So the build writes a little |
| # script that can be invoked with a simple name. |
| # |
| # Though the gcc target libraries' makefiles are smart enough to obey |
| # STRIP_FOR_TARGET for library files, the newlib makefiles just |
| # blindly use $(INSTALL_DATA) for both header (text) files and library |
| # files. Hence it's necessary to override its INSTALL_DATA setting to |
| # one that will do stripping using this script, and thus the script |
| # must silently do nothing to non-binary files. |
| def ConfigureTargetPrep(arch): |
| script_file = 'strip_for_target' |
| |
| config_target = arch + '-nacl' |
| script_contents = """\ |
| #!/bin/sh |
| mode=--strip-all |
| for arg; do |
| case "$arg" in |
| -*) ;; |
| *) |
| type=`file --brief --mime-type "$arg"` |
| case "$type" in |
| application/x-executable|application/x-sharedlib) ;; |
| application/x-archive|application/x-object) mode=--strip-debug ;; |
| *) exit 0 ;; |
| esac |
| ;; |
| esac |
| done |
| exec %s-strip $mode --remove-section=.comment "$@" |
| """ % config_target |
| |
| return [ |
| command.WriteData(script_contents, script_file), |
| command.Command(['chmod', '+x', script_file]), |
| ] |
| |
| |
| def ConfigureTargetArgs(arch): |
| config_target = arch + '-nacl' |
| return [ |
| '--target=' + config_target, |
| '--with-sysroot=/' + config_target, |
| 'STRIP_FOR_TARGET=%(cwd)s/strip_for_target', |
| ] |
| |
| |
| def CommandsInBuild(command_lines): |
| return [ |
| command.RemoveDirectory('build'), |
| command.Mkdir('build'), |
| ] + [command.Command(cmd, cwd='build') |
| for cmd in command_lines] |
| |
| |
| def PopulateDeps(dep_dirs): |
| commands = [command.RemoveDirectory('all_deps'), |
| command.Mkdir('all_deps')] |
| commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True) |
| for dirname in dep_dirs] |
| return commands |
| |
| |
| def WithDepsOptions(options, component=None): |
| if component is None: |
| directory = command.path.join('%(cwd)s', 'all_deps') |
| else: |
| directory = '%(abs_' + component + ')s' |
| return ['--with-' + option + '=' + directory |
| for option in options] |
| |
| |
| # Return the component name we'll use for a base component name and |
| # a host tuple. The component names cannot contain dashes or other |
| # non-identifier characters, because the names of the files uploaded |
| # to Google Storage are constrained. GNU configuration tuples contain |
| # dashes, which we translate to underscores. |
| def ForHost(component_name, host): |
| return component_name + '_' + pynacl.gsd_storage.LegalizeName(host) |
| |
| |
| # These are libraries that go into building the compiler itself. |
| def HostGccLibs(host): |
| def H(component_name): |
| return ForHost(component_name, host) |
| host_gcc_libs = { |
| H('gmp'): { |
| 'type': 'build', |
| 'dependencies': ['gmp'], |
| 'commands': [ |
| command.Command(ConfigureCommand('gmp') + |
| ConfigureHostLib(host) + [ |
| '--with-sysroot=%(abs_output)s', |
| '--enable-cxx', |
| # Without this, the built library will |
| # assume the instruction set details |
| # available on the build machine. With |
| # this, it dynamically chooses what code |
| # to use based on the details of the |
| # actual host CPU at runtime. |
| '--enable-fat', |
| ]), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| ], |
| }, |
| H('mpfr'): { |
| 'type': 'build', |
| 'dependencies': ['mpfr', H('gmp')], |
| 'commands': [ |
| command.Command(ConfigureCommand('mpfr') + |
| ConfigureHostLib(host) + |
| WithDepsOptions(['sysroot', 'gmp'], H('gmp'))), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| ], |
| }, |
| H('mpc'): { |
| 'type': 'build', |
| 'dependencies': ['mpc', H('gmp'), H('mpfr')], |
| 'commands': PopulateDeps(['%(' + H('gmp') + ')s', |
| '%(' + H('mpfr') + ')s']) + [ |
| command.Command(ConfigureCommand('mpc') + |
| ConfigureHostLib(host) + |
| WithDepsOptions(['sysroot', 'gmp', 'mpfr'])), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| ], |
| }, |
| H('isl'): { |
| 'type': 'build', |
| 'dependencies': ['isl', H('gmp')], |
| 'commands': [ |
| command.Command(ConfigureCommand('isl') + |
| ConfigureHostLib(host) + |
| WithDepsOptions(['sysroot', 'gmp-prefix'], |
| H('gmp'))), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| # The .pc files wind up containing some absolute paths |
| # that make the output depend on the build directory name. |
| # The dependents' configure scripts don't need them anyway. |
| command.RemoveDirectory(command.path.join( |
| '%(output)s', 'lib', 'pkgconfig')), |
| ], |
| }, |
| H('cloog'): { |
| 'type': 'build', |
| 'dependencies': ['cloog', H('gmp'), H('isl')], |
| 'commands': PopulateDeps(['%(' + H('gmp') + ')s', |
| '%(' + H('isl') + ')s']) + [ |
| command.Command(ConfigureCommand('cloog') + |
| ConfigureHostLib(host) + [ |
| '--with-bits=gmp', |
| '--with-isl=system', |
| ] + WithDepsOptions(['sysroot', |
| 'gmp-prefix', |
| 'isl-prefix'])), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| # The .pc files wind up containing some absolute paths |
| # that make the output depend on the build directory name. |
| # The dependents' configure scripts don't need them anyway. |
| command.RemoveDirectory(command.path.join( |
| '%(output)s', 'lib', 'pkgconfig')), |
| ], |
| }, |
| H('expat'): { |
| 'type': 'build', |
| 'dependencies': ['expat'], |
| 'commands': [ |
| command.Command(ConfigureCommand('expat') + |
| ConfigureHostLib(host)), |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + [ |
| # expat does not support the install-strip target. |
| 'installlib', |
| 'INSTALL=%(expat)s/conftools/install-sh -c -s', |
| 'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644', |
| ]), |
| ], |
| }, |
| } |
| return host_gcc_libs |
| |
| |
| HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog'] |
| |
| def HostGccLibsDeps(host): |
| return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS] |
| |
| |
| def SDKLibs(host, target): |
| def H(component_name): |
| return ForHost(component_name, host) |
| components = ['newlib_%s' % target, |
| 'gcc_libs_%s' % target, |
| H('binutils_%s' % target), |
| H('gcc_%s' % target), |
| ] |
| sdk_compiler = { |
| H('sdk_compiler_%s' % target): { |
| 'type': 'work', |
| 'dependencies': components, |
| 'commands': [command.CopyRecursive('%(' + item + ')s', '%(output)s') |
| for item in components], |
| }, |
| } |
| sdk_libs = { |
| 'sdk_libs_%s' % target: { |
| 'type': 'build', |
| 'dependencies': [H('sdk_compiler_%s' % target)], |
| 'inputs': { |
| 'src_untrusted': os.path.join(NACL_DIR, 'src', 'untrusted'), |
| 'src_include': os.path.join(NACL_DIR, 'src', 'include'), |
| 'scons.py': os.path.join(NACL_DIR, 'scons.py'), |
| 'site_scons': os.path.join(NACL_DIR, 'site_scons'), |
| 'prep_nacl_sdk': |
| os.path.join(NACL_DIR, 'build', 'prep_nacl_sdk.py'), |
| }, |
| 'commands': [ |
| command.Command( |
| [sys.executable, '%(scons.py)s', |
| '--verbose', 'MODE=nacl', '-j%(cores)s', 'naclsdk_validate=0', |
| 'platform=%s' % target, |
| 'nacl_newlib_dir=%(abs_' + H('sdk_compiler_%s' % target) + ')s', |
| 'DESTINATION_ROOT=%(work_dir)s', |
| 'includedir=' + command.path.join('%(output)s', |
| target + '-nacl', 'include'), |
| 'libdir=' + command.path.join('%(output)s', |
| target + '-nacl', 'lib'), |
| 'install'], |
| cwd=NACL_DIR), |
| ], |
| }, |
| } |
| |
| return dict(sdk_compiler.items() + sdk_libs.items()) |
| |
| |
| def ConfigureCommand(source_component): |
| return [command % {'src': '%(' + source_component + ')s'} |
| for command in CONFIGURE_CMD] |
| |
| |
| # When doing a Canadian cross, we need native-hosted cross components |
| # to do the GCC build. |
| def GccDeps(host, target): |
| components = ['binutils_' + target] |
| if CrossCompiling(host): |
| components.append('gcc_' + target) |
| host = NATIVE_TUPLE |
| return [ForHost(component, host) for component in components] |
| |
| |
| def GccCommand(host, target, cmd): |
| components_for_path = GccDeps(host, target) |
| return command.Command( |
| cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin') |
| for component in components_for_path]) |
| |
| |
| def ConfigureGccCommand(source_component, host, target, extra_args=[]): |
| return GccCommand( |
| host, |
| target, |
| ConfigureCommand(source_component) + |
| ConfigureHostTool(host) + |
| ConfigureTargetArgs(target) + |
| TARGET_GCC_CONFIG.get(target, []) + [ |
| '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s', |
| '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s', |
| '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s', |
| '--with-isl=%(abs_' + ForHost('isl', host) + ')s', |
| '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s', |
| '--enable-cloog-backend=isl', |
| '--disable-dlopen', |
| '--disable-shared', |
| '--with-newlib', |
| '--with-linker-hash-style=gnu', |
| '--enable-linker-build-id', |
| '--enable-languages=c,c++,lto', |
| ] + extra_args) |
| |
| |
| |
| def HostTools(host, target): |
| def H(component_name): |
| return ForHost(component_name, host) |
| |
| def WindowsAlternate(if_windows, if_not_windows, if_mac=None): |
| if if_mac is not None and HostIsMac(host): |
| return if_mac |
| elif HostIsWindows(host): |
| return if_windows |
| else: |
| return if_not_windows |
| |
| # Return the file name with the appropriate suffix for an executable file. |
| def Exe(file): |
| return file + WindowsAlternate('.exe', '') |
| |
| # The binutils git checkout includes all the directories in the |
| # upstream binutils-gdb.git repository, but some of these |
| # directories are not included in a binutils release tarball. The |
| # top-level Makefile will try to build whichever of the whole set |
| # exist, but we don't want these extra directories built. So we |
| # stub them out by creating dummy <subdir>/Makefile files; having |
| # these exist before the configure-<subdir> target in the |
| # top-level Makefile runs prevents it from doing anything. |
| binutils_dummy_dirs = ['gdb', 'libdecnumber', 'readline', 'sim'] |
| def DummyDirCommands(dirs): |
| dummy_makefile = """\ |
| .DEFAULT:;@echo Ignoring $@ |
| """ |
| commands = [] |
| for dir in dirs: |
| commands.append(command.Mkdir(command.path.join('%(cwd)s', dir))) |
| commands.append(command.WriteData( |
| dummy_makefile, command.path.join('%(cwd)s', dir, 'Makefile'))) |
| return commands |
| |
| tools = { |
| H('binutils_' + target): { |
| 'type': 'build', |
| 'dependencies': ['binutils'], |
| 'commands': ConfigureTargetPrep(target) + [ |
| command.Command( |
| ConfigureCommand('binutils') + |
| ConfigureHostTool(host) + |
| ConfigureTargetArgs(target) + [ |
| '--enable-deterministic-archives', |
| '--enable-gold', |
| ] + WindowsAlternate([], ['--enable-plugins'])) |
| ] + DummyDirCommands(binutils_dummy_dirs) + [ |
| command.Command(MakeCommand(host)), |
| command.Command(MakeCheckCommand(host)), |
| command.Command(MAKE_DESTDIR_CMD + ['install-strip']), |
| REMOVE_INFO_DIR, |
| ] + InstallDocFiles('binutils', |
| ['COPYING3'] + |
| [command.path.join(subdir, 'NEWS') |
| for subdir in |
| ['binutils', 'gas', 'ld', 'gold']]) + |
| # The top-level lib* directories contain host libraries |
| # that we don't want to include in the distribution. |
| [command.RemoveDirectory(command.path.join('%(output)s', name)) |
| for name in ['lib', 'lib32', 'lib64']], |
| }, |
| |
| H('gcc_' + target): { |
| 'type': 'build', |
| 'dependencies': (['gcc'] + HostGccLibsDeps(host) + |
| GccDeps(host, target)), |
| 'commands': ConfigureTargetPrep(target) + [ |
| ConfigureGccCommand('gcc', host, target), |
| # GCC's configure step writes configargs.h with some strings |
| # including the configure command line, which get embedded |
| # into the gcc driver binary. The build only works if we use |
| # absolute paths in some of the configure switches, but |
| # embedding those paths makes the output differ in repeated |
| # builds done in different directories, which we do not want. |
| # So force the generation of that file early and then edit it |
| # in place to replace the absolute paths with something that |
| # never varies. Note that the 'configure-gcc' target will |
| # actually build some components before running gcc/configure. |
| GccCommand(host, target, |
| MakeCommand(host, ['configure-gcc'])), |
| command.Command(['sed', '-i', '-e', |
| ';'.join(['s@%%(abs_%s)s@.../%s_install@g' % |
| (component, component) |
| for component in |
| HostGccLibsDeps(host)] + |
| ['s@%(cwd)s@...@g']), |
| command.path.join('gcc', 'configargs.h')]), |
| # gcc/Makefile's install rules ordinarily look at the |
| # installed include directory for a limits.h to decide |
| # whether the lib/gcc/.../include-fixed/limits.h header |
| # should be made to expect a libc-supplied limits.h or not. |
| # Since we're doing this build in a clean environment without |
| # any libc installed, we need to force its hand here. |
| GccCommand(host, target, |
| MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])), |
| # gcc/Makefile's install targets populate this directory |
| # only if it already exists. |
| command.Mkdir(command.path.join('%(output)s', |
| target + '-nacl', 'bin'), |
| True), |
| GccCommand(host, target, |
| MAKE_DESTDIR_CMD + ['install-strip-gcc']), |
| REMOVE_INFO_DIR, |
| # Note we include COPYING.RUNTIME here and not with gcc_libs. |
| ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']), |
| }, |
| |
| # GDB can support all the targets in one host tool. |
| H('gdb'): { |
| 'type': 'build', |
| 'dependencies': ['gdb', H('expat')], |
| 'commands': [ |
| command.Command( |
| ConfigureCommand('gdb') + |
| ConfigureHostTool(host) + [ |
| '--target=x86_64-nacl', |
| '--enable-targets=arm-none-eabi-nacl', |
| '--with-expat', |
| # Windows (MinGW) is missing ncurses; we need to |
| # build one here and link it in statically for |
| # --enable-tui. See issue nativeclient:3911. |
| '--%s-tui' % WindowsAlternate('disable', 'enable'), |
| 'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include', |
| 'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib', |
| ] + |
| # TODO(mcgrathr): Should use --with-python to ensure |
| # we have it on Linux/Mac. |
| WindowsAlternate(['--without-python'], []) + |
| # TODO(mcgrathr): The default -Werror only breaks because |
| # the OSX default compiler is an old front-end that does |
| # not understand all the GCC options. Maybe switch to |
| # using clang (system or Chromium-supplied) on Mac. |
| (['--disable-werror'] if HostIsMac(host) else [])), |
| command.Command(MakeCommand(host) + ['all-gdb']), |
| command.Command(MAKE_DESTDIR_CMD + [ |
| '-C', 'gdb', 'install-strip', |
| ]), |
| REMOVE_INFO_DIR, |
| ] + [command.Command(['ln', '-f', |
| command.path.join('%(abs_output)s', |
| 'bin', |
| Exe('x86_64-nacl-gdb')), |
| command.path.join('%(abs_output)s', |
| 'bin', |
| Exe(arch + '-nacl-gdb'))]) |
| for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [ |
| 'COPYING3', |
| command.path.join('gdb', 'NEWS'), |
| ]), |
| }, |
| } |
| |
| # TODO(mcgrathr): The ARM cross environment does not supply a termcap |
| # library, so it cannot build GDB. |
| if host.startswith('arm') and CrossCompiling(host): |
| del tools[H('gdb')] |
| |
| return tools |
| |
| def TargetCommands(host, target, command_list): |
| # First we have to copy the host tools into a common directory. |
| # We can't just have both directories in our PATH, because the |
| # compiler looks for the assembler and linker relative to itself. |
| commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s', |
| '%(' + ForHost('gcc_' + target, host) + ')s']) |
| bindir = command.path.join('%(cwd)s', 'all_deps', 'bin') |
| commands += [command.Command(cmd, path_dirs=[bindir]) |
| for cmd in command_list] |
| return commands |
| |
| |
| def TargetLibs(host, target): |
| lib_deps = [ForHost(component + '_' + target, host) |
| for component in ['binutils', 'gcc']] |
| |
| def NewlibFile(subdir, name): |
| return command.path.join('%(output)s', target + '-nacl', subdir, name) |
| |
| newlib_sysroot = '%(abs_newlib_' + target + ')s' |
| newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target) |
| |
| # See the comment at ConfigureTargetPrep, above. |
| newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target', |
| '%(abs_newlib)s/install-sh', |
| '-c', '-s', '-m', '644']) |
| |
| iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4' |
| newlib_configure_args = [ |
| '--disable-libgloss', |
| '--enable-newlib-iconv', |
| '--enable-newlib-iconv-from-encodings=' + iconv_encodings, |
| '--enable-newlib-iconv-to-encodings=' + iconv_encodings, |
| '--enable-newlib-io-long-long', |
| '--enable-newlib-io-long-double', |
| '--enable-newlib-io-c99-formats', |
| '--enable-newlib-mb', |
| 'CFLAGS=-O2', |
| 'INSTALL_DATA=' + newlib_install_data, |
| ] |
| |
| newlib_post_install = [ |
| command.Rename(NewlibFile('lib', 'libc.a'), |
| NewlibFile('lib', 'libcrt_common.a')), |
| command.WriteData(NewlibLibcScript(target), |
| NewlibFile('lib', 'libc.a')), |
| ] + [ |
| command.Copy( |
| command.path.join('%(pthread_headers)s', header), |
| NewlibFile('include', header)) |
| for header in ('pthread.h', 'semaphore.h') |
| ] |
| |
| |
| libs = { |
| 'newlib_' + target: { |
| 'type': 'build', |
| 'dependencies': ['newlib'] + lib_deps, |
| 'inputs': { 'pthread_headers': |
| os.path.join(NACL_DIR, 'src', 'untrusted', |
| 'pthread') }, |
| 'commands': (ConfigureTargetPrep(target) + |
| TargetCommands(host, target, [ |
| ConfigureCommand('newlib') + |
| ConfigureHostTool(host) + |
| ConfigureTargetArgs(target) + |
| newlib_configure_args, |
| MakeCommand(host), |
| MAKE_DESTDIR_CMD + ['install-strip'], |
| ]) + |
| newlib_post_install + |
| InstallDocFiles('newlib', ['COPYING.NEWLIB'])), |
| }, |
| |
| 'gcc_libs_' + target: { |
| 'type': 'build', |
| 'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] + |
| HostGccLibsDeps(host)), |
| # This actually builds the compiler again and uses that compiler |
| # to build the target libraries. That's by far the easiest thing |
| # to get going given the interdependencies of the target |
| # libraries (especially libgcc) on the gcc subdirectory, and |
| # building the compiler doesn't really take all that long in the |
| # grand scheme of things. |
| # TODO(mcgrathr): If upstream ever cleans up all their |
| # interdependencies better, unpack the compiler, configure with |
| # --disable-gcc, and just build all-target. |
| 'commands': ConfigureTargetPrep(target) + [ |
| ConfigureGccCommand('gcc_libs', host, target, [ |
| '--with-build-sysroot=' + newlib_sysroot, |
| ]), |
| GccCommand(host, target, |
| MakeCommand(host) + [ |
| 'build_tooldir=' + newlib_tooldir, |
| 'all-target', |
| ]), |
| GccCommand(host, target, |
| MAKE_DESTDIR_CMD + ['install-strip-target']), |
| REMOVE_INFO_DIR, |
| ], |
| }, |
| } |
| return libs |
| |
| # Compute it once. |
| NATIVE_TUPLE = pynacl.platform.PlatformTriple() |
| |
| |
| # For our purposes, "cross-compiling" means not literally that we are |
| # targetting a host that does not match NATIVE_TUPLE, but that we are |
| # targetting a host whose binaries we cannot run locally. So x86-32 |
| # on x86-64 does not count as cross-compiling. See NATIVE_ENOUGH_MAP, above. |
| def CrossCompiling(host): |
| return (host != NATIVE_TUPLE and |
| host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {})) |
| |
| |
| def HostIsWindows(host): |
| return host == WINDOWS_HOST_TUPLE |
| |
| |
| def HostIsMac(host): |
| return host == MAC_HOST_TUPLE |
| |
| |
| # We build target libraries only on Linux for two reasons: |
| # 1. We only need to build them once. |
| # 2. Linux is the fastest to build. |
| # TODO(mcgrathr): In future set up some scheme whereby non-Linux |
| # bots can build target libraries but not archive them, only verifying |
| # that the results came out the same as the ones archived by the |
| # official builder bot. That will serve as a test of the host tools |
| # on the other host platforms. |
| def BuildTargetLibsOn(host): |
| return host == LINUX_X86_64_TUPLE |
| |
| |
| def GetPackageTargets(): |
| """Package Targets describes all the final package targets. |
| |
| This build can be built among many build bots, but eventually all things |
| will be combined together. This package target dictionary describes the final |
| output of the entire build. |
| """ |
| package_targets = {} |
| |
| # Add in standard upload targets. |
| for host_target in UPLOAD_HOST_TARGETS: |
| for target in host_target.targets: |
| target_arch = target.name |
| package_prefix = target.pkg_prefix |
| |
| # Each package target contains non-platform specific newlib and gcc libs. |
| # These packages are added inside of TargetLibs(host, target). |
| newlib_package = 'newlib_%s' % target_arch |
| gcc_lib_package = 'gcc_libs_%s' % target_arch |
| sdk_lib_packages = ['sdk_libs_%s' % target_arch] |
| shared_packages = [newlib_package, gcc_lib_package] |
| |
| # Each package target contains arm binutils and gcc. |
| # These packages are added inside of HostTools(host, target). |
| platform_triple = pynacl.platform.PlatformTriple(host_target.os, |
| host_target.arch) |
| binutils_package = ForHost('binutils_%s' % target_arch, platform_triple) |
| gcc_package = ForHost('gcc_%s' % target_arch, platform_triple) |
| gdb_package = ForHost('gdb', platform_triple) |
| |
| # Create a list of packages for a target. |
| platform_packages = [binutils_package, gcc_package, gdb_package] |
| raw_packages = shared_packages + platform_packages |
| all_packages = raw_packages + sdk_lib_packages |
| |
| os_name = pynacl.platform.GetOS(host_target.os) |
| if host_target.differ3264: |
| arch_name = pynacl.platform.GetArch3264(host_target.arch) |
| else: |
| arch_name = pynacl.platform.GetArch(host_target.arch) |
| package_target = '%s_%s' % (os_name, arch_name) |
| package_name = '%snacl_%s_newlib' % (package_prefix, |
| pynacl.platform.GetArch(target_arch)) |
| raw_package_name = package_name + '_raw' |
| |
| # Toolchains by default are "raw" unless they include the Core SDK |
| package_target_dict = package_targets.setdefault(package_target, {}) |
| package_target_dict.setdefault(raw_package_name, []).extend(raw_packages) |
| package_target_dict.setdefault(package_name, []).extend(all_packages) |
| |
| # GDB is a special and shared, we will inject it into various other packages. |
| for platform, arch in GDB_INJECT_HOSTS: |
| platform_triple = pynacl.platform.PlatformTriple(platform, arch) |
| os_name = pynacl.platform.GetOS(platform) |
| arch_name = pynacl.platform.GetArch(arch) |
| |
| gdb_packages = [ForHost('gdb', platform_triple)] |
| package_target = '%s_%s' % (os_name, arch_name) |
| |
| for package_name, package_archives in GDB_INJECT_PACKAGES: |
| combined_packages = package_archives + gdb_packages |
| package_target_dict = package_targets.setdefault(package_target, {}) |
| package_target_dict.setdefault(package_name, []).extend(combined_packages) |
| |
| return dict(package_targets) |
| |
| |
| def CollectPackagesForHost(host, targets): |
| packages = HostGccLibs(host).copy() |
| for target in targets: |
| packages.update(HostTools(host, target)) |
| if BuildTargetLibsOn(host): |
| packages.update(TargetLibs(host, target)) |
| packages.update(SDKLibs(host, target)) |
| return packages |
| |
| |
| def CollectPackages(targets): |
| packages = CollectSources() |
| |
| packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets)) |
| |
| for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []): |
| packages.update(CollectPackagesForHost(host, targets)) |
| |
| return packages |
| |
| |
| PACKAGES = CollectPackages(TARGET_LIST) |
| PACKAGE_TARGETS = GetPackageTargets() |
| |
| |
| if __name__ == '__main__': |
| tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:]) |
| # TODO(mcgrathr): The bot ought to run some native_client tests |
| # using the new toolchain, like the old x86 toolchain bots do. |
| tb.Main() |