blob: 0c463c5d9d3ab54525fb3a4b6edc2b6214b0c1df [file] [log] [blame]
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""NEXE building script
This module will take a set of source files, include paths, library paths, and
additional arguments, and use them to build.
import hashlib
import json
from optparse import OptionParser
import os
import re
import Queue
import shutil
import StringIO
import subprocess
import sys
import tempfile
import threading
import urllib2
from build_nexe_tools import (CommandRunner, Error, FixPath,
IsFile, MakeDir, RemoveFile)
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import pynacl.platform
# When a header file defining NACL_BUILD_SUBARCH is introduced,
# we can simply remove this map.
# cf)
'x86-32': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'],
'x86-32-nonsfi': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=32'],
'x86-64': ['NACL_BUILD_ARCH=x86', 'NACL_BUILD_SUBARCH=64'],
'arm-nonsfi': ['NACL_BUILD_ARCH=arm', 'NACL_BUILD_SUBARCH=32'],
'mips': ['NACL_BUILD_ARCH=mips', 'NACL_BUILD_SUBARCH=32'],
'pnacl': ['NACL_BUILD_ARCH=pnacl'],
def RemoveQuotes(opt):
if opt and opt[0] == '"':
assert opt[-1] == '"', opt
return opt[1:-1].replace('\\"', '"')
return opt
def ArgToList(opt):
outlist = []
if opt is None:
return outlist
optlist = opt.split(' ')
for optitem in optlist:
optitem = RemoveQuotes(optitem)
if optitem:
return outlist
def GetMTime(filepath):
"""GetMTime returns the last modification time of the file or None."""
return os.path.getmtime(FixPath(filepath))
except OSError:
return None
def IsStale(out_ts, src_ts, rebuilt=False):
# If either source or output timestamp was not available, assume stale.
if not out_ts or not src_ts:
return True
# If just rebuilt timestamps may be equal due to time granularity.
if rebuilt:
return out_ts < src_ts
# If about to build, be conservative and rebuilt just in case.
return out_ts <= src_ts
def IsEnvFlagTrue(flag_name, default=False):
"""Return true when the given flag is true.
Any values that do not match the true pattern are False.
flag_name: a string name of a flag.
default: default return value if the flag is not set.
True if the flag is in the true pattern. Otherwise False.
flag_value = os.environ.get(flag_name)
if flag_value is None:
return default
return bool('^([tTyY]|1:?)', flag_value))
def GetIntegerEnv(flag_name, default=0):
"""Parses and returns integer environment variable.
flag_name: a string name of a flag.
default: default return value if the flag is not set.
Integer value of the flag.
flag_value = os.environ.get(flag_name)
if flag_value is None:
return default
return int(flag_value)
except ValueError:
raise Error('Invalid ' + flag_name + ': ' + flag_value)
class Builder(CommandRunner):
"""Builder object maintains options and generates build command-lines.
The Builder object takes a set of script command-line options, and generates
a set of paths, and command-line options for the NaCl toolchain.
def __init__(self, options):
super(Builder, self).__init__(options)
arch = options.arch
self.arch = arch
build_type ='_')
toolname = build_type[0]
self.outtype = build_type[1]
self.osname = pynacl.platform.GetOS()
# pnacl toolchain can be selected in three different ways
# 1. by specifying --arch=pnacl directly to generate
# pexe targets.
# 2. by specifying --build=newlib_translate to generated
# nexe via translation
# 3. by specifying --build=newlib_{nexe,nlib}_pnacl use pnacl
# toolchain in native mode (e.g. the IRT shim)
self.is_pnacl_toolchain = False
if self.outtype == 'translate':
self.is_pnacl_toolchain = True
if len(build_type) > 2 and build_type[2] == 'pnacl':
self.is_pnacl_toolchain = True
self.is_nacl_clang = len(build_type) > 2 and build_type[2] == 'clang'
if arch.endswith('-nonsfi'):
arch = arch[:-len('-nonsfi')]
if arch in ['x86-32', 'x86-64']:
mainarch = 'x86'
self.tool_prefix = 'x86_64-nacl-'
elif arch == 'arm':
self.tool_prefix = 'arm-nacl-'
mainarch = 'arm'
elif arch == 'mips':
self.is_pnacl_toolchain = True
elif arch == 'pnacl':
self.is_pnacl_toolchain = True
raise Error('Toolchain architecture %s not supported.' % arch)
if toolname not in ['newlib', 'glibc']:
raise Error('Toolchain of type %s not supported.' % toolname)
if arch == 'arm' and toolname == 'glibc':
raise Error('arm glibc not yet supported.')
if arch == 'mips' and toolname == 'glibc':
raise Error('mips glibc not supported.')
if arch == 'pnacl' and toolname == 'glibc':
raise Error('pnacl glibc not yet supported.')
if self.is_pnacl_toolchain:
self.tool_prefix = 'pnacl-'
tool_subdir = 'pnacl_newlib'
elif self.is_nacl_clang:
tool_subdir = 'pnacl_newlib'
tool_subdir = 'nacl_%s_%s' % (mainarch, toolname)
# The pnacl-clang, etc. tools are scripts. Note that for the CommandRunner
# so that it can know if a shell is needed or not.
build_arch = pynacl.platform.GetArch()
tooldir = os.path.join('%s_%s' % (self.osname, build_arch), tool_subdir)
self.root_path = options.root
self.nacl_path = os.path.join(self.root_path, 'native_client')
project_path, project_name = os.path.split(
self.outdir = options.objdir
# Set the toolchain directories
self.toolchain = os.path.join(options.toolpath, tooldir)
self.toolbin = os.path.join(self.toolchain, 'bin')
self.toolstamp = os.path.join(self.toolchain, tool_subdir + '.json')
if not IsFile(self.toolstamp):
raise Error('Could not find toolchain prep stamp file: ' + self.toolstamp)
self.inc_paths = ArgToList(options.incdirs)
self.lib_paths = ArgToList(options.libdirs)
self.define_list = ArgToList(options.defines) =
self.cmd_file = options.cmd_file
options.compile_flags, self.define_list, options.arch)
self.strip = options.strip
self.empty = options.empty
self.strip_all = options.strip_all
self.strip_debug = options.strip_debug
self.tls_edit = options.tls_edit
self.finalize_pexe = options.finalize_pexe and arch == 'pnacl'
goma_config = self.GetGomaConfig(options.gomadir, arch, toolname)
self.gomacc = goma_config.get('gomacc', '')
self.goma_burst = goma_config.get('burst', False)
self.goma_threads = goma_config.get('threads', 1)
# Define NDEBUG for Release builds.
if options.build_config.startswith('Release'):
# Use unoptimized native objects for debug IRT builds for faster compiles.
if (self.is_pnacl_toolchain
and (self.outtype == 'nlib'
or self.outtype == 'nexe')
and self.arch != 'pnacl'):
if (options.build_config is not None
and options.build_config.startswith('Debug')):
'-arch', self.arch])
# Also use fast translation because we are still translating libc/libc++
self.irt_linker = options.irt_linker
self.Log('Compile options: %s' % self.compile_options)
self.Log('Linker options: %s' % self.link_options)
def GenNaClPath(self, path):
"""Helper which prepends path with the native client source directory."""
return os.path.join(self.root_path, 'native_client', path)
def GetBinName(self, name):
"""Helper which prepends executable with the toolchain bin directory."""
return os.path.join(self.toolbin, self.tool_prefix + name)
def GetCCompiler(self):
"""Helper which returns C compiler path."""
if self.is_pnacl_toolchain or self.is_nacl_clang:
return self.GetBinName('clang')
return self.GetBinName('gcc')
def GetCXXCompiler(self):
"""Helper which returns C++ compiler path."""
if self.is_pnacl_toolchain or self.is_nacl_clang:
return self.GetBinName('clang++')
return self.GetBinName('g++')
def GetAr(self):
"""Helper which returns ar path."""
return self.GetBinName('ar')
def GetStrip(self):
"""Helper which returns strip path."""
return self.GetBinName('strip')
def GetObjCopy(self):
"""Helper which returns objcopy path."""
return self.GetBinName('objcopy')
def GetReadElf(self):
"""Helper which returns readelf path."""
return self.GetBinName('readelf')
def GetPnaclFinalize(self):
"""Helper which returns pnacl-finalize path."""
assert self.is_pnacl_toolchain
return self.GetBinName('finalize')
def BuildAssembleOptions(self, options):
options = ArgToList(options)
self.assemble_options = options + ['-I' + name for name in self.inc_paths]
def DebugName(self):
return + '.debug'
def UntaggedName(self):
return + '.untagged'
def LinkOutputName(self):
if (self.is_pnacl_toolchain and self.finalize_pexe or
self.strip_all or self.strip_debug):
return self.DebugName()
def ArchiveOutputName(self):
if self.strip_debug:
return self.DebugName()
def StripOutputName(self):
def TranslateOutputName(self):
def Soname(self):
def BuildCompileOptions(self, options, define_list, arch):
"""Generates compile options, called once by __init__."""
options = ArgToList(options)
# We want to shared gyp 'defines' with other targets, but not
# ones that are host system dependent. Filtering them out.
# This really should be better.
# See:
define_list = [define for define in define_list
if not (define.startswith('NACL_WINDOWS=') or
define.startswith('NACL_OSX=') or
define.startswith('NACL_LINUX=') or
define.startswith('NACL_ANDROID=') or
define.startswith('NACL_BUILD_ARCH=') or
define.startswith('NACL_BUILD_SUBARCH=') or
define == 'COMPONENT_BUILD' or
'WIN32' in define or
'WINDOWS' in define or
'WINVER' in define)]
options += ['-D' + define for define in define_list]
self.compile_options = options + ['-I' + name for name in self.inc_paths]
def BuildLinkOptions(self, options):
"""Generates link options, called once by __init__."""
options = ArgToList(options)
if self.outtype == 'nso':
options += ['-Wl,-rpath-link,' + name for name in self.lib_paths]
options += ['-shared']
options += ['-Wl,-soname,' + os.path.basename(self.Soname())]
self.link_options = options + ['-L' + name for name in self.lib_paths]
def BuildArchiveOptions(self):
"""Generates link options, called once by __init__."""
self.archive_options = []
def GetObjectName(self, src):
if self.strip:
src = src.replace(self.strip,'')
# Hash the full path of the source file and add 32 bits of that hash onto
# the end of the object file name. This helps disambiguate files with the
# same name, because all of the object files are placed into the same
# directory. Technically, the correct solution would be to preserve the
# directory structure of the input source files inside the object file
# directory, but doing that runs the risk of running into filename length
# issues on Windows.
h = hashlib.sha1()
wart = h.hexdigest()[:8]
_, filename = os.path.split(src)
filename, _ = os.path.splitext(filename)
return os.path.join(self.outdir, filename + '_' + wart + '.o')
def FixWindowsPath(self, path):
# The windows version of the nacl toolchain returns badly
# formed system header paths. As we do want changes in the
# toolchain to trigger rebuilds, compensate by detecting
# malformed paths (starting with /libexec/) and assume these
# are actually toolchain relative.
# Additionally, in some cases the toolchains emit cygwin paths
# which don't work in a win32 python.
# Assume they are all /cygdrive/ relative and convert to a
# drive letter.
cygdrive = '/cygdrive/'
if path.startswith('/cygdrive/'):
path = os.path.normpath(
path[len(cygdrive)] + ':' + path[len(cygdrive)+1:])
elif path.startswith('/libexec/'):
path = os.path.normpath(os.path.join(self.toolchain, path[1:]))
return path
def GetGomaConfig(self, gomadir, arch, toolname):
"""Returns a goma config dictionary if goma is available or {}."""
# Start goma support from os/arch/toolname that have been tested.
# Set NO_NACL_GOMA=true to force to avoid using goma.
default_no_nacl_goma = True if pynacl.platform.IsWindows() else False
if (arch not in ['x86-32', 'x86-64', 'pnacl']
or toolname not in ['newlib', 'glibc']
or IsEnvFlagTrue('NO_NACL_GOMA', default=default_no_nacl_goma)
or IsEnvFlagTrue('GOMA_DISABLED')):
return {}
goma_config = {}
gomacc_base = 'gomacc.exe' if pynacl.platform.IsWindows() else 'gomacc'
# Search order of gomacc:
# --gomadir command argument -> GOMA_DIR env. -> PATH env.
search_path = []
# 1. --gomadir in the command argument.
if gomadir:
# 2. Use GOMA_DIR environment variable if exist.
goma_dir_env = os.environ.get('GOMA_DIR')
if goma_dir_env:
# 3. Append PATH env.
path_env = os.environ.get('PATH')
if path_env:
for directory in search_path:
gomacc = os.path.join(directory, gomacc_base)
if os.path.isfile(gomacc):
port = int(subprocess.Popen(
[gomacc, 'port'],
status = urllib2.urlopen(
'' % port).read().strip()
if status == 'ok':
goma_config['gomacc'] = gomacc
except (OSError, ValueError, urllib2.URLError) as e:
# Try another gomacc in the search path.
self.Log('Strange gomacc %s found, try another one: %s' % (gomacc, e))
if goma_config:
goma_config['burst'] = IsEnvFlagTrue('NACL_GOMA_BURST')
default_threads = 100 if pynacl.platform.IsLinux() else 10
goma_config['threads'] = GetIntegerEnv('NACL_GOMA_THREADS',
return goma_config
def NeedsRebuild(self, outd, out, src, rebuilt=False):
if not IsFile(self.toolstamp):
if rebuilt:
raise Error('Could not find toolchain stamp file %s.' % self.toolstamp)
return True
if not IsFile(self.cmd_file):
if rebuilt:
raise Error('Could not find cmd file %s.' % self.cmd_file)
return True
if not IsFile(outd):
if rebuilt:
raise Error('Could not find dependency file %s.' % outd)
return True
if not IsFile(out):
if rebuilt:
raise Error('Could not find output file %s.' % out)
return True
inputs = [__file__, self.toolstamp, src, self.cmd_file]
outputs = [out, outd]
# Find their timestamps if any.
input_times = [(GetMTime(f), f) for f in inputs]
output_times = [(GetMTime(f), f) for f in outputs]
# All inputs must exist.
missing_inputs = [p[1] for p in input_times if p[0] is None]
if missing_inputs:
raise Error('Missing inputs: %s' % str(missing_inputs))
# Rebuild if any outputs are missing.
missing_outputs = [p[1] for p in output_times if p[0] is None]
if missing_outputs:
if rebuilt:
raise Error('Outputs missing after rebuild: %s' % str(missing_outputs))
return True
newest_input = max(input_times)
oldest_output = min(output_times)
if IsStale(oldest_output[0], newest_input[0], rebuilt):
if rebuilt:
raise Error('Output %s is older than toolchain stamp %s' % (
oldest_output[1], newest_input[1]))
return True
# Decode emitted makefile.
with open(FixPath(outd), 'r') as fh:
deps =
# Remove line continuations
deps = deps.replace('\\\n', ' ')
deps = deps.replace('\n', '')
# The dependencies are whitespace delimited following the first ':'
# (that is not part of a windows drive letter)
deps = deps.split(':', 1)
if pynacl.platform.IsWindows() and len(deps[0]) == 1:
# The path has a drive letter, find the next ':'
deps = deps[1].split(':', 1)[1]
deps = deps[1]
deps = deps.split()
if pynacl.platform.IsWindows():
deps = [self.FixWindowsPath(d) for d in deps]
# Check if any input has changed.
for filename in deps:
file_tm = GetMTime(filename)
if IsStale(oldest_output[0], file_tm, rebuilt):
if rebuilt:
raise Error('Dependency %s is older than output %s.' % (
filename, oldest_output[1]))
return True
return False
def Compile(self, src):
"""Compile the source with pre-determined options."""
compile_options = self.compile_options[:]
_, ext = os.path.splitext(src)
if ext in ['.c', '.S']:
bin_name = self.GetCCompiler()
if self.is_pnacl_toolchain and ext == '.S':
elif ext in ['.cc', '.cpp']:
bin_name = self.GetCXXCompiler()
if ext != '.h':
self.Log('Skipping unknown type %s for %s.' % (ext, src))
return None
# This option is only applicable to C, and C++ compilers warn if
# it is present, so remove it for C++ to avoid the warning.
if ext != '.c' and '-Wstrict-prototypes' in compile_options:
self.Log('\nCompile %s' % src)
out = self.GetObjectName(src)
outd = out + '.d'
# Don't rebuild unneeded.
if not self.NeedsRebuild(outd, out, src):
return out
cmd_line = [bin_name, '-c', src, '-o', out,
'-MD', '-MF', outd] + compile_options
if self.gomacc:
cmd_line.insert(0, self.gomacc)
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
self.NeedsRebuild(outd, out, src, True)
except Error as e:
raise Error('Failed to compile %s to %s with deps %s and cmdline:\t%s'
'\nNeedsRebuild returned error: %s' % (
src, out, outd, ' '.join(cmd_line), e))
return out
def RunLink(self, cmd_line, link_out):
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
def Link(self, srcs):
"""Link these objects with predetermined options and output name."""
out = self.LinkOutputName()
self.Log('\nLink %s' % out)
bin_name = self.GetCXXCompiler()
srcs_flags = []
if not self.empty:
srcs_flags += srcs
srcs_flags += self.link_options
# Handle an IRT link specially, using a separate script.
if self.irt_linker:
if self.tls_edit is None:
raise Error('Linking the IRT requires tls_edit')
irt_link_cmd = [sys.executable, self.irt_linker,
'--output=' + out,
'--tls-edit=' + self.tls_edit,
'--link-cmd=' + bin_name,
'--readelf-cmd=' + self.GetReadElf()]
if self.commands_are_scripts:
irt_link_cmd += ['--commands-are-scripts']
irt_link_cmd += srcs_flags
err = self.Run(irt_link_cmd, normalize_slashes=False)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(irt_link_cmd)))
return out
cmd_line = [bin_name, '-o', out, '-Wl,--as-needed']
cmd_line += srcs_flags
self.RunLink(cmd_line, out)
return out
# For now, only support translating a pexe, and not .o file(s)
def Translate(self, src):
"""Translate a pexe to a nexe."""
out = self.TranslateOutputName()
self.Log('\nTranslate %s' % out)
bin_name = self.GetBinName('translate')
cmd_line = [bin_name, '-arch', self.arch, src, '-o', out]
cmd_line += self.link_options
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
return out
def Archive(self, srcs):
"""Archive these objects with predetermined options and output name."""
out = self.ArchiveOutputName()
self.Log('\nArchive %s' % out)
if '-r' in self.link_options:
bin_name = self.GetCXXCompiler()
cmd_line = [bin_name, '-o', out, '-Wl,--as-needed']
if not self.empty:
cmd_line += srcs
cmd_line += self.link_options
bin_name = self.GetAr()
cmd_line = [bin_name, '-rc', out]
if not self.empty:
cmd_line += srcs
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
return out
def Strip(self, src):
"""Strip the NEXE"""
self.Log('\nStrip %s' % src)
out = self.StripOutputName()
pre_debug_tagging = self.UntaggedName()
# Strip from foo.debug to foo.untagged.
strip_name = self.GetStrip()
strip_option = '--strip-all' if self.strip_all else '--strip-debug'
# pnacl does not have an objcopy so there are no way to embed a link
if self.is_pnacl_toolchain:
cmd_line = [strip_name, strip_option, src, '-o', out]
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
cmd_line = [strip_name, strip_option, src, '-o', pre_debug_tagging]
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
# Tag with a debug link to foo.debug copying from foo.untagged to foo.
objcopy_name = self.GetObjCopy()
cmd_line = [objcopy_name, '--add-gnu-debuglink', src,
pre_debug_tagging, out]
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
# Drop the untagged intermediate.
return out
def Finalize(self, src):
"""Finalize the PEXE"""
self.Log('\nFinalize %s' % src)
out = self.StripOutputName()
bin_name = self.GetPnaclFinalize()
cmd_line = [bin_name, src, '-o', out]
err = self.Run(cmd_line)
if err:
raise Error('FAILED with %d: %s' % (err, ' '.join(cmd_line)))
return out
def Generate(self, srcs):
"""Generate final output file.
Link or Archive the final output file, from the compiled sources.
if self.outtype in ['nexe', 'pexe', 'nso']:
out = self.Link(srcs)
if self.is_pnacl_toolchain and self.finalize_pexe:
# Note: pnacl-finalize also does stripping.
elif self.strip_all or self.strip_debug:
elif self.outtype in ['nlib', 'plib']:
out = self.Archive(srcs)
if self.strip_debug:
elif self.strip_all:
raise Error('FAILED: --strip-all on libs will result in unusable libs.')
raise Error('FAILED: Unknown outtype: %s' % (self.outtype))
def UpdateBuildArgs(args, filename):
new_cmd = json.dumps(args)
with open(filename, 'r') as fileobj:
old_cmd =
old_cmd = None
if old_cmd == new_cmd:
return False
with open(filename, 'w') as fileobj:
return True
def Main(argv):
parser = OptionParser()
parser.add_option('--empty', dest='empty', default=False,
help='Do not pass sources to library.', action='store_true')
parser.add_option('--no-suffix', dest='suffix', default=True,
help='Do not append arch suffix.', action='store_false')
parser.add_option('--strip-debug', dest='strip_debug', default=False,
help='Strip the NEXE for debugging', action='store_true')
parser.add_option('--strip-all', dest='strip_all', default=False,
help='Strip the NEXE for production', action='store_true')
parser.add_option('--strip', dest='strip', default='',
help='Strip the filename')
parser.add_option('--nonstable-pnacl', dest='finalize_pexe', default=True,
help='Do not finalize pnacl bitcode for ABI stability',
parser.add_option('--source-list', dest='source_list',
help='Filename to load a source list from')
parser.add_option('--tls-edit', dest='tls_edit', default=None,
help='tls_edit location if TLS should be modified for IRT')
parser.add_option('--irt-linker', dest='irt_linker', default=None,
help='linker tool to use if linking the IRT')
parser.add_option('-a', '--arch', dest='arch',
help='Set target architecture')
parser.add_option('-c', '--compile', dest='compile_only', default=False,
help='Compile only.', action='store_true')
parser.add_option('-i', '--include-dirs', dest='incdirs',
help='Set include directories.')
parser.add_option('-l', '--lib-dirs', dest='libdirs',
help='Set library directories.')
parser.add_option('-n', '--name', dest='name',
help='Base path and name of the nexe.')
parser.add_option('-o', '--objdir', dest='objdir',
help='Base path of the object output dir.')
parser.add_option('-r', '--root', dest='root',
help='Set the root directory of the sources')
parser.add_option('--product-directory', dest='product_directory',
help='Set the root directory of the build')
parser.add_option('-b', '--build', dest='build',
help='Set build type (<toolchain>_<outtype>, ' +
'where toolchain is newlib or glibc and outtype is ' +
'one of nexe, nlib, nso, pexe, or translate)')
parser.add_option('--compile_flags', dest='compile_flags',
help='Set compile flags.')
parser.add_option('--defines', dest='defines',
help='Set defines')
parser.add_option('--link_flags', dest='link_flags',
help='Set link flags.')
parser.add_option('-v', '--verbose', dest='verbose', default=False,
help='Enable verbosity', action='store_true')
parser.add_option('-t', '--toolpath', dest='toolpath',
help='Set the path for of the toolchains.')
parser.add_option('--config-name', dest='build_config',
help='GYP build configuration name (Release/Debug)')
parser.add_option('--gomadir', dest='gomadir',
help='Path of the goma directory.')
options, files = parser.parse_args(argv[1:])
if is None:
parser.error('--name is required!')
if options.build_config is None:
parser.error('--config-name is required!')
if options.root is None:
parser.error('--root is required!')
if options.arch is None:
parser.error('--arch is required!')
if is None:
parser.error('--build is required!')
if not argv:
return 1
# Compare command-line options to last run, and force a rebuild if they
# have changed.
options.cmd_file = + '.cmd'
UpdateBuildArgs(argv, options.cmd_file)
if options.product_directory is None:
parser.error('--product-dir is required')
product_dir = options.product_directory
# Normalize to forward slashes because re.sub interprets backslashes
# as escape characters. This also simplifies the subsequent regexes.
product_dir = product_dir.replace('\\', '/')
# Remove fake child that may be apended to the path.
# See untrusted.gypi.
product_dir = re.sub(r'/+xyz$', '', product_dir)
build = None
if options.source_list:
source_list_handle = open(options.source_list, 'r')
source_list =
for file_name in source_list:
file_name = RemoveQuotes(file_name)
if "$" in file_name:
# The "make" backend can have an "obj" interpolation variable.
file_name = re.sub(r'\$!?[({]?obj[)}]?', product_dir + '/obj',
# Expected patterns:
# $!PRODUCT_DIR in ninja.
# $(builddir) in make.
# $(OutDir) in MSVC.
# $(BUILT_PRODUCTS_DIR) in xcode.
# Also strip off and re-add the trailing directory seperator because
# different platforms are inconsistent on if it's there or not.
# HACK assume only the product directory is the only var left.
file_name = re.sub(r'\$!?[({]?\w+[)}]?/?', product_dir + '/',
assert "$" not in file_name, file_name
# Use set instead of list not to compile the same file twice.
# To keep in mind that the order of files may differ from the .gypcmd file,
# the set is not converted to a list.
# Having duplicated files can cause race condition of compiling during
# parallel build using goma.
# TODO(sbc): remove the duplication and turn it into an error.
files = set(files)
# Fix slash style to insulate invoked toolchains.
options.toolpath = os.path.normpath(options.toolpath)
build = Builder(options)
objs = []
if build.outtype == 'translate':
# Just translate a pexe to a nexe
if len(files) != 1:
parser.error('Pexe translation requires exactly one input file.')
return 0
if build.gomacc and (build.goma_burst or build.goma_threads > 1):
returns = Queue.Queue()
# Push all files into the inputs queue
inputs = Queue.Queue()
for filename in files:
def CompileThread(input_queue, output_queue):
while True:
filename = input_queue.get_nowait()
except Queue.Empty:
output_queue.put((filename, build.Compile(filename)))
except Exception:
# Put current exception info to the queue.
# Don't limit number of threads in the burst mode.
if build.goma_burst:
num_threads = len(files)
num_threads = min(build.goma_threads, len(files))
# Start parallel build.
build_threads = []
for _ in xrange(num_threads):
thr = threading.Thread(target=CompileThread, args=(inputs, returns))
# Wait for results.
src_to_obj = {}
for _ in files:
out = returns.get()
# An exception raised in the thread may come through the queue.
# Raise it again here.
if (isinstance(out, tuple) and len(out) == 3 and
isinstance(out[1], Exception)):
raise out[0], out[1], out[2]
elif out and len(out) == 2:
src_to_obj[out[0]] = out[1]
raise Error('Unexpected element in CompileThread output_queue %s' %
# Keep the input files ordering consistent for link phase to ensure
# determinism.
for filename in files:
# If input file to build.Compile is something it cannot handle, it
# returns None.
if src_to_obj[filename]:
assert inputs.empty()
# Wait until all threads have stopped and verify that there are no more
# results.
for thr in build_threads:
assert returns.empty()
else: # slow path.
for filename in files:
out = build.Compile(filename)
if out:
# Do not link if building an object. However we still want the output file
# to be what was specified in
if options.compile_only:
if len(objs) > 1:
raise Error('--compile mode cannot be used with multiple sources')
return 0
except Error as e:
sys.stderr.write('%s\n' % e)
if build is not None:
return 1
if build is not None:
if __name__ == '__main__':