blob: 69cb28dd4a957187924d167baab043a9e2d271de [file] [log] [blame]
#! -*- python -*-
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import atexit
import json
import os
import platform
import re
import subprocess
import sys
import zlib
from SCons.Errors import UserError
from SCons.Script import GetBuildFailures
import SCons.Warnings
import SCons.Util
import command_tester
import test_lib
import pynacl.platform
# turning garbage collection off reduces startup time by 10%
import gc
def PrintFinalReport():
"""This function is run just before scons exits and dumps various reports.
# Note, these global declarations are not strictly necessary
global pre_base_env
if pre_base_env.Bit('target_stats'):
print '*' * 70
print '*' * 70
for k in sorted(CMD_COUNTER.keys()):
print "%4d %s" % (CMD_COUNTER[k], k)
print '*' * 70
print '*' * 70
for k in sorted(ENV_COUNTER.keys()):
print "%4d %s" % (ENV_COUNTER[k], k)
failures = []
for failure in GetBuildFailures():
for node in Flatten(failure.node):
# If this wasn't a test, "GetTestName" will return raw_name.
'test_name': GetTestName(node),
'raw_name': str(node.path),
'errstr': failure.errstr
json_path = ARGUMENTS.get('json_build_results_output_file')
if json_path:
with open(json_path, 'w') as f:
json.dump(failures, f, sort_keys=True, indent=2)
if not failures:
print '*' * 70
print 'ERROR REPORT: %d failures' % len(failures)
print '*' * 70
for failure in failures:
test_name = failure['test_name']
if test_name != failure['raw_name']:
test_name = '%s (%s)' % (test_name, failure['raw_name'])
print "%s failed: %s\n" % (test_name, failure['errstr'])
def VerboseConfigInfo(env):
"Should we print verbose config information useful for bug reports"
if '--help' in sys.argv: return False
if env.Bit('prebuilt') or env.Bit('built_elsewhere'): return False
return env.Bit('sysinfo')
# NOTE BitFromArgument(...) implicitly defines additional ACCEPTABLE_ARGUMENTS.
# TODO: add comments what these mean
# TODO: check which ones are obsolete
#### ASCII SORTED ####
# Use a destination directory other than the default "scons-out".
# Limit bandwidth of browser tester
# Location to download Chromium binaries to and/or read them from.
# used for chrome_browser_tests: path to the browser
# A comma-separated list of test names to disable by excluding the
# tests from a test suite. For example, 'small_tests
# disable_tests=run_hello_world_test' will run small_tests without
# including hello_world_test. Note that if a test listed here
# does not exist you will not get an error or a warning.
# used for chrome_browser_tests: path to a pre-built browser plugin.
# force emulator use by tests
# force sel_ldr use by tests
# force irt image used by tests
# generate_ninja=FILE enables a Ninja backend for SCons. This writes a
# .ninja build file to FILE describing all of SCons' build targets.
# Path to a JSON file for machine-readable output.
# Replacement memcheck command for overriding the DEPS-in memcheck
# script. May have commas to separate separate shell args. There
# is no quoting, so this implies that this mechanism will fail if
# the args actually need to have commas. See
# for
# the discussion of why this argument is needed.
# If the replacement memcheck command only works for trusted code,
# set memcheck_trusted_only to non-zero.
# When building with MSan, this can be set to values 0 (fastest, least
# useful reports) through 2 (slowest, most useful reports). Default is 1.
# colon-separated list of linker flags, e.g. "-lfoo:-Wl,-u,bar".
# prefix to add in-front of perf tracking trace labels.
# colon-separated list of pnacl bcld flags, e.g. "-lfoo:-Wl,-u,bar".
# Not using nacl_linkflags since that gets clobbered in some tests.
# Run tests under this tool (e.g. valgrind, tsan, strace, etc).
# If the tool has options, pass them after comma: 'tool,--opt1,--opt2'.
# NB: no way to use tools the names or the args of
# which contains a comma.
# More args for the tool.
# Multiply timeout values by this number.
# test_wrapper specifies a wrapper program such as
# tools/, which runs tests on a remote host
# using rsync and SSH. Example usage:
# ./scons run_hello_world_test platform=arm force_emulator= \
# test_wrapper="./tools/ --host=armbox --subdir=tmp"
# Replacement tsan command for overriding the DEPS-in tsan
# script. May have commas to separate separate shell args. There
# is no quoting, so this implies that this mechanism will fail if
# the args actually need to have commas. See
# for
# the discussion of why this argument is needed.
# Run browser tests under this tool. See
# tools/browser_tester/browsertester/ for tool names.
# activates buildbot-specific presets
# Where to install header files for public consumption.
# Where to install libraries for public consumption.
# Where to install trusted-code binaries for public (SDK) consumption.
# Where a Breakpad build output directory is for optional Breakpad testing.
# Allows overriding of the nacl newlib toolchain directory.
# Allows override of the nacl glibc toolchain directory.
# Allows override of the pnacl newlib toolchain directory.
# Allows overriding the version number in the toolchain's
# FEATURE_VERSION file. This is used for PNaCl ABI compatibility
# testing.
# Overly general to provide compatibility with existing build bots, etc.
# In the future it might be worth restricting the values that are accepted.
_TRUE_STRINGS = set(['1', 'true', 'yes'])
_FALSE_STRINGS = set(['0', 'false', 'no'])
# Converts a string representing a Boolean value, of some sort, into an actual
# Boolean value. Python's built in type coercion does not work because
# bool('False') == True
def StringValueToBoolean(value):
# ExpandArguments may stick non-string values in ARGUMENTS. Be accommodating.
if isinstance(value, bool):
return value
if not isinstance(value, basestring):
raise Exception("Expecting a string but got a %s" % repr(type(value)))
if value.lower() in _TRUE_STRINGS:
return True
elif value.lower() in _FALSE_STRINGS:
return False
raise Exception("Cannot convert '%s' to a Boolean value" % value)
def GetBinaryArgumentValue(arg_name, default):
if not isinstance(default, bool):
raise Exception("Default value for '%s' must be a Boolean" % arg_name)
if arg_name not in ARGUMENTS:
return default
return StringValueToBoolean(ARGUMENTS[arg_name])
# name is the name of the bit
# arg_name is the name of the command-line argument, if it differs from the bit
def BitFromArgument(env, name, default, desc, arg_name=None):
# In most cases the bit name matches the argument name
if arg_name is None:
arg_name = name
DeclareBit(name, desc)
assert arg_name not in ACCEPTABLE_ARGUMENTS, repr(arg_name)
if GetBinaryArgumentValue(arg_name, default):
# SetUpArgumentBits declares binary command-line arguments and converts them to
# bits. For example, one of the existing declarations would result in the
# argument "bitcode=1" causing env.Bit('bitcode') to evaluate to true.
# NOTE Command-line arguments are a SCons-ism that is separate from
# command-line options. Options are prefixed by "-" or "--" whereas arguments
# are not. The function SetBitFromOption can be used for options.
# NOTE This function must be called before the bits are used
# NOTE This function must be called after all modifications of ARGUMENTS have
# been performed. See: ExpandArguments
def SetUpArgumentBits(env):
BitFromArgument(env, 'bitcode', default=False,
desc='We are building bitcode')
BitFromArgument(env, 'nacl_clang', default=False,
desc='Use the native nacl-clang newlib compiler instead of nacl-gcc')
BitFromArgument(env, 'translate_fast', default=False,
desc='When using pnacl TC (bitcode=1) use accelerated translation step')
BitFromArgument(env, 'built_elsewhere', default=False,
desc='The programs have already been built by another system')
BitFromArgument(env, 'skip_trusted_tests', default=False,
desc='Only run untrusted tests - useful for translator testing'
' (also skips tests of the IRT itself')
BitFromArgument(env, 'nacl_pic', default=False,
desc='generate position indepent code for (P)NaCl modules')
BitFromArgument(env, 'nacl_static_link', default=not env.Bit('nacl_glibc'),
desc='Whether to use static linking instead of dynamic linking '
'for building NaCl executables during tests. '
'For nacl-newlib, the default is 1 (static linking). '
'For nacl-glibc, the default is 0 (dynamic linking).')
BitFromArgument(env, 'nacl_disable_shared', default=not env.Bit('nacl_glibc'),
desc='Do not build shared versions of libraries. '
'For nacl-newlib, the default is 1 (static libraries only). '
'For nacl-glibc, the default is 0 (both static and shared libraries).')
# Defaults on when --verbose is specified.
# --verbose sets 'brief_comstr' to False, so this looks a little strange
BitFromArgument(env, 'target_stats', default=not GetOption('brief_comstr'),
desc='Collect and display information about which commands are executed '
'during the build process')
BitFromArgument(env, 'werror', default=True,
desc='Treat warnings as errors (-Werror)')
BitFromArgument(env, 'disable_nosys_linker_warnings', default=False,
desc='Disable warning mechanism in src/untrusted/nosys/warning.h')
BitFromArgument(env, 'naclsdk_validate', default=True,
desc='Verify the presence of the SDK')
BitFromArgument(env, 'running_on_valgrind', default=False,
desc='Compile and test using valgrind')
BitFromArgument(env, 'enable_tmpfs_redirect_var', default=False,
desc='Allow redirecting tmpfs location for shared memory '
'(by default, /dev/shm is used)')
BitFromArgument(env, 'pp', default=False,
desc='Enable pretty printing')
# Defaults on when --verbose is specified
# --verbose sets 'brief_comstr' to False, so this looks a little strange
BitFromArgument(env, 'sysinfo', default=not GetOption('brief_comstr'),
desc='Print verbose system information')
BitFromArgument(env, 'disable_flaky_tests', default=False,
desc='Do not run potentially flaky tests - used on Chrome bots')
BitFromArgument(env, 'use_sandboxed_translator', default=False,
desc='use pnacl sandboxed translator for linking (not available for arm)')
BitFromArgument(env, 'pnacl_generate_pexe', default=env.Bit('bitcode'),
desc='use pnacl to generate pexes and translate in a separate step')
BitFromArgument(env, 'translate_in_build_step', default=True,
desc='Run translation during build phase (e.g. if do_not_run_tests=1)')
BitFromArgument(env, 'pnacl_unsandboxed', default=False,
desc='Translate pexe to an unsandboxed, host executable')
BitFromArgument(env, 'minsfi', default=False,
desc='Build MinSFI libraries and run tests')
BitFromArgument(env, 'nonsfi_nacl', default=False,
desc='Use Non-SFI Mode instead of the original SFI Mode. This uses '
'nonsfi_loader instead of sel_ldr, and it tells the PNaCl toolchain '
'to translate pexes to Non-SFI nexes.')
BitFromArgument(env, 'use_newlib_nonsfi_loader', default=True,
desc='Test nonsfi_loader linked against NaCl newlib instead of the one '
'linked against host libc. This flag makes sense only with '
BitFromArgument(env, 'browser_headless', default=False,
desc='Where possible, set up a dummy display to run the browser on '
'when running browser tests. On Linux, this runs the browser through '
'xvfb-run. This Scons does not need to be run with an X11 display '
'and we do not open a browser window on the user\'s desktop. '
'Unfortunately there is no equivalent on Mac OS X.')
BitFromArgument(env, 'disable_crash_dialog', default=True,
desc='Disable Windows\' crash dialog box, which Windows pops up when a '
'process exits with an unhandled fault. Windows enables this by '
'default for processes launched from the command line or from the '
'GUI. Our default is to disable it, because the dialog turns crashes '
'into hangs on Buildbot, and our test suite includes various crash '
BitFromArgument(env, 'do_not_run_tests', default=False,
desc='Prevents tests from running. This lets SCons build the files needed '
'to run the specified test(s) without actually running them. This '
'argument is a counterpart to built_elsewhere.')
BitFromArgument(env, 'no_gdb_tests', default=False,
desc='Prevents GDB tests from running. If GDB is not available, you can '
'test everything else by specifying this flag.')
# TODO(shcherbina): add support for other golden-based tests, not only
# run_x86_*_validator_testdata_tests.
BitFromArgument(env, 'regenerate_golden', default=False,
desc='When running golden-based tests, instead of comparing results '
'save actual output as golden data.')
BitFromArgument(env, 'x86_64_zero_based_sandbox', default=False,
desc='Use the zero-address-based x86-64 sandbox model instead of '
'the r15-based model.')
BitFromArgument(env, 'android', default=False,
desc='Build for Android target')
BitFromArgument(env, 'skip_nonstable_bitcode', default=False,
desc='Skip tests involving non-stable bitcode')
# This is for generating a testing library for use within private test
# enuminsts, where we want to compare and test different validators.
BitFromArgument(env, 'ncval_testing', default=False,
desc='EXPERIMENTAL: Compile validator code for testing within enuminsts')
# PNaCl sanity checks
if not env.Bit('bitcode'):
pnacl_only_flags = ('minsfi',
for flag_name in pnacl_only_flags:
if env.Bit(flag_name):
raise UserError('The option %r only makes sense when using the '
'PNaCl toolchain (i.e. with bitcode=1)'
% flag_name)
pnacl_incompatible_flags = ('nacl_clang',
for flag_name in pnacl_incompatible_flags:
if env.Bit(flag_name):
raise UserError('The option %r cannot be used when building with '
'PNaCl (i.e. with bitcode=1)' % flag_name)
def CheckArguments():
for key in ARGUMENTS:
raise UserError('bad argument: %s' % key)
# Sets a command line argument. Dies if an argument with this name is already
# defined.
def SetArgument(key, value):
print ' %s=%s' % (key, str(value))
if key in ARGUMENTS:
raise UserError('ERROR: %s redefined' % (key, ))
ARGUMENTS[key] = value
# Expands "macro" command line arguments.
def ExpandArguments():
if ARGUMENTS.get('buildbot') == 'memcheck':
print 'buildbot=memcheck expands to the following arguments:'
'src/third_party/valgrind/') +
SetArgument('scale_timeout', 20)
SetArgument('running_on_valgrind', True)
elif ARGUMENTS.get('buildbot') == 'tsan':
print 'buildbot=tsan expands to the following arguments:'
'src/third_party/valgrind/') +
',--nacl-untrusted,--error-exitcode=1,' +
SetArgument('scale_timeout', 20)
SetArgument('running_on_valgrind', True)
elif ARGUMENTS.get('buildbot') == 'tsan-trusted':
print 'buildbot=tsan-trusted expands to the following arguments:'
'src/third_party/valgrind/') +
',--error-exitcode=1,' +
SetArgument('scale_timeout', 20)
SetArgument('running_on_valgrind', True)
elif ARGUMENTS.get('buildbot') == 'memcheck-browser-tests':
print 'buildbot=memcheck-browser-tests expands to the following arguments:'
SetArgument('browser_test_tool', 'memcheck')
SetArgument('scale_timeout', 20)
SetArgument('running_on_valgrind', True)
elif ARGUMENTS.get('buildbot') == 'tsan-browser-tests':
print 'buildbot=tsan-browser-tests expands to the following arguments:'
SetArgument('browser_test_tool', 'tsan')
SetArgument('scale_timeout', 20)
SetArgument('running_on_valgrind', True)
elif ARGUMENTS.get('buildbot'):
raise UserError('ERROR: unexpected argument buildbot="%s"' % (
ARGUMENTS.get('buildbot'), ))
def GetTargetPlatform():
return pynacl.platform.GetArch3264(ARGUMENTS.get('platform', 'x86-32'))
def GetBuildPlatform():
return pynacl.platform.GetArch3264()
environment_list = []
# Base environment for both nacl and non-nacl variants.
kwargs = {}
pre_base_env = Environment(
# Use the environment that scons was run in to run scons invoked commands.
# This allows in things like externally provided PATH, PYTHONPATH.
ENV = os.environ.copy(),
tools = ['component_setup'],
# SOURCE_ROOT is one leave above the native_client directory.
SOURCE_ROOT = Dir('#/..').abspath,
# Publish dlls as final products (to staging).
# Use workaround in special scons version.
# Select where to find coverage tools.
COVERAGE_MCOV = '../third_party/lcov/bin/mcov',
COVERAGE_GENHTML = '../third_party/lcov/bin/genhtml',
if 'generate_ninja' in ARGUMENTS:
import pynacl.scons_to_ninja
pre_base_env, dest_file=ARGUMENTS['generate_ninja'])
breakpad_tools_dir = ARGUMENTS.get('breakpad_tools_dir')
if breakpad_tools_dir is not None:
pre_base_env['BREAKPAD_TOOLS_DIR'] = pre_base_env.Dir(
DeclareBit('clang', 'Use clang to build trusted code')
pre_base_env.SetBitFromOption('clang', False)
'Use AddressSanitizer to build trusted code (implies --clang)')
pre_base_env.SetBitFromOption('asan', False)
if pre_base_env.Bit('asan'):
'Use MemorySanitizer to build trusted code (implies --clang)')
pre_base_env.SetBitFromOption('msan', False)
if pre_base_env.Bit('msan'):
DeclareBit('coverage_enabled', 'The build should be instrumented to generate'
'coverage information')
# If the environment variable BUILDBOT_BUILDERNAME is set, we can determine
# if we are running in a VM by the lack of a '-bare-' (aka bare metal) in the
# bot name. Otherwise if the builder name is not set, then assume real HW.
DeclareBit('running_on_vm', 'Returns true when environment is running in a VM')
builder = os.environ.get('BUILDBOT_BUILDERNAME')
if builder and builder.find('-bare-') == -1:
DeclareBit('nacl_glibc', 'Use nacl-glibc for building untrusted code')
pre_base_env.SetBitFromOption('nacl_glibc', False)
# This function should be called ASAP after the environment is created, but
# after ExpandArguments.
# Register PrintFinalReport only after SetUpArgumentBits since it references
# bits that get declared in SetUpArgumentBits
def DisableCrashDialog():
if sys.platform == 'win32':
import win32api
import win32con
# The double call is to preserve existing flags, as discussed at
new_flags = win32con.SEM_NOGPFAULTERRORBOX
existing_flags = win32api.SetErrorMode(new_flags)
win32api.SetErrorMode(existing_flags | new_flags)
if pre_base_env.Bit('disable_crash_dialog'):
# We want to pull CYGWIN setup in our environment or at least set flag
# nodosfilewarning. It does not do anything when CYGWIN is not involved
# so let's do it in all cases.
pre_base_env['ENV']['CYGWIN'] = os.environ.get('CYGWIN', 'nodosfilewarning')
# Note: QEMU_PREFIX_HOOK may influence test runs and sb translator invocations
pre_base_env['ENV']['QEMU_PREFIX_HOOK'] = os.environ.get('QEMU_PREFIX_HOOK', '')
# Allow the zero-based sandbox model to run insecurely.
# TODO(arbenson): remove this once binutils bug is fixed (see
# src/trusted/service_runtime/arch/x86_64/sel_addrspace_posix_x86_64.c)
if pre_base_env.Bit('x86_64_zero_based_sandbox'):
if pre_base_env.Bit('werror'):
werror_flags = ['-Werror']
werror_flags = []
# Allow variadic macros
werror_flags = werror_flags + ['-Wno-variadic-macros']
if pre_base_env.Bit('clang'):
# Allow 'default' label in switch even when all enumeration cases
# have been covered.
werror_flags += ['-Wno-covered-switch-default']
# Allow C++11 extensions (for "override")
werror_flags += ['-Wno-c++11-extensions']
# Method to make sure -pedantic, etc, are not stripped from the
# default env, since occasionally an engineer will be tempted down the
# dark -- but wide and well-trodden -- path of expediency and stray
# from the path of correctness.
def EnsureRequiredBuildWarnings(env):
if (env.Bit('linux') or env.Bit('mac')) and not env.Bit('android'):
required_env_flags = set(['-pedantic', '-Wall'] + werror_flags)
ccflags = set(env.get('CCFLAGS'))
if not required_env_flags.issubset(ccflags):
raise UserError('required build flags missing: '
+ ' '.join(required_env_flags.difference(ccflags)))
# windows get a pass for now
# Expose MakeTempDir and MakeTempFile to scons scripts
def MakeEmptyFile(env, **kwargs):
fd, path = test_lib.MakeTempFile(env, **kwargs)
return path
# Method to add target suffix to name.
def NaClTargetArchSuffix(env, name):
return name + '_' + env['TARGET_FULLARCH'].replace('-', '_')
# Generic Test Wrapper
# Add list of Flaky or Bad tests to skip per platform. A
# platform is defined as build type
bad_build_lists = {
'arm': [],
# This is a list of tests that do not yet pass when using nacl-glibc.
# TODO(mseaborn): Enable more of these tests!
nacl_glibc_skiplist = set([
# Struct layouts differ.
# Syscall wrappers not implemented yet.
# Fails because clock() is not hooked up.
# Needs further investigation.
# This test fails with nacl-glibc: glibc reports an internal
# sanity check failure in free().
# TODO(robertm): This needs further investigation.
# we would need to list all of the glibc components as
# web accessible resources in the extensions's manifest.json,
# not just the nexe and nmf file.
nacl_glibc_skiplist.update(['%s_irt' % test for test in nacl_glibc_skiplist])
# Whitelist of tests to run for Non-SFI Mode. Note that typos here will
# not be caught automatically!
# TODO(mseaborn): Eventually we should run all of small_tests instead of
# this whitelist.
nonsfi_test_whitelist = set([
# If a test is not in one of these suites, it will probally not be run on a
# regular basis. These are the suites that will be run by the try bot or that
# a large number of users may run by hand.
# Tests using the pepper plugin, only run with chrome
# TODO(ncbray): migrate pepper_browser_tests to chrome_browser_tests
# Lightweight browser tests
# Special testing environment for testing comparing x86 validators.
# Environment for validator difference testing
# Subset of tests enabled for Non-SFI Mode.
# These are the test suites we know exist, but aren't run on a regular basis.
# These test suites are essentially shortcuts that run a specific subset of the
# test cases.
'eh_tests', # Tests for C++ exception handling
'exception_tests', # Tests for hardware exception handling
# Special testing of the decoder for the ARM validator.
# Under --mode=nacl_irt_test we build variants of numerous tests normally
# built under --mode=nacl. The test names and suite names for these
# variants are set (in IrtTestAddNodeToTestSuite, below) by appending _irt
# to the names used for the --mode=nacl version of the same tests.
MAJOR_TEST_SUITES |= set([name + '_irt'
for name in MAJOR_TEST_SUITES])
ACCEPTABLE_TEST_SUITES |= set([name + '_irt'
# The major test suites are also acceptable names. Suite names are checked
# against this set in order to catch typos.
def ValidateTestSuiteNames(suite_name, node_name):
if node_name is None:
node_name = '<unknown>'
# Prevent a silent failiure - strings are iterable!
if not isinstance(suite_name, (list, tuple)):
raise Exception("Test suites for %s should be specified as a list, "
"not as a %s: %s" % (node_name, type(suite_name).__name__,
if not suite_name:
raise Exception("No test suites are specified for %s. Set the 'broken' "
"parameter on AddNodeToTestSuite in the cases where there's a known "
"issue and you don't want the test to run" % (node_name,))
# Make sure each test is in at least one test suite we know will run
major_suites = set(suite_name).intersection(MAJOR_TEST_SUITES)
if not major_suites:
raise Exception("None of the test suites %s for %s are run on a "
"regular basis" % (repr(suite_name), node_name))
# Make sure a wierd test suite hasn't been inadvertantly specified
for s in suite_name:
raise Exception("\"%s\" is not a known test suite. Either this is "
"a typo for %s, or it should be added to ACCEPTABLE_TEST_SUITES in "
"SConstruct" % (s, node_name))
def GetPlatformString(env):
build = env['BUILD_TYPE']
# If we are testing 'NACL' we really need the trusted info
if build=='nacl' and 'TRUSTED_ENV' in env:
trusted_env = env['TRUSTED_ENV']
build = trusted_env['BUILD_TYPE']
subarch = trusted_env['BUILD_SUBARCH']
subarch = env['BUILD_SUBARCH']
# Build the test platform string
return build + '-' + subarch
tests_to_disable_qemu = set([
# These tests do not work under QEMU but do work on ARM hardware.
# You should use the is_broken argument in preference to adding
# tests to this list.
# TODO(dschuff) some of these tests appear to work with the new QEMU.
# find out which
# Note, for now these tests disable both the irt and non-irt variants
'run_atomic_ops_test', # still broken with qemu 2012/06/12
'run_egyptian_cotton_test', # still broken with qemu 2012/06/12
'run_mmap_atomicity_test', # still broken with qemu 2012/06/12
# subprocess needs to also have qemu prefix, which isn't supported
# The next 2 tests seem flaky on QEMU
# qemu has bugs that make TestCatchingFault flaky (see
#, and
# we don't particularly need to measure performance under qemu.
tests_to_disable = set()
if ARGUMENTS.get('disable_tests', '') != '':
def ShouldSkipTest(env, node_name):
if (env.Bit('skip_trusted_tests')
return True
if env.Bit('do_not_run_tests'):
# This hack is used for pnacl testing where we might build tests
# without running them on one bot and then transfer and run them on another.
# The skip logic only takes the first bot into account e.g. qemu
# restrictions, while it really should be skipping based on the second
# bot. By simply disabling the skipping completely we work around this.
return False
# There are no known-to-fail tests any more, but this code is left
# in so that if/when we port to a new architecture or add a test
# that is known to fail on some platform(s), we can continue to have
# a central location to disable tests from running. NB: tests that
# don't *build* on some platforms need to be omitted in another way.
if node_name in tests_to_disable:
return True
if env.UsingEmulator():
if node_name in tests_to_disable_qemu:
return True
# For now also disable the irt variant
if node_name.endswith('_irt') and node_name[:-4] in tests_to_disable_qemu:
return True
# Retrieve list of tests to skip on this platform
skiplist = bad_build_lists.get(env.GetPlatformString(), [])
if node_name in skiplist:
return True
if env.Bit('nacl_glibc') and node_name in nacl_glibc_skiplist:
return True
return False
def AddImplicitTestSuites(suite_list, node_name):
if node_name in nonsfi_test_whitelist:
suite_list = suite_list + ['nonsfi_tests']
return suite_list
def AddNodeToTestSuite(env, node, suite_name, node_name, is_broken=False,
# CommandTest can return an empty list when it silently discards a test
if not node:
assert node_name is not None
test_name_regex = r'run_.*_(unit)?test.*$'
assert re.match(test_name_regex, node_name), (
'test %r does not match "run_..._test" naming convention '
'(precise regex is %s)' % (node_name, test_name_regex))
suite_name = AddImplicitTestSuites(suite_name, node_name)
ValidateTestSuiteNames(suite_name, node_name)
if is_broken or is_flaky and env.Bit('disable_flaky_tests'):
# Only print if --verbose is specified
if not GetOption('brief_comstr'):
print '*** BROKEN ', node_name
env.Alias('broken_tests', node)
elif env.ShouldSkipTest(node_name):
print '*** SKIPPING ', env.GetPlatformString(), ':', node_name
env.Alias('broken_tests', node)
env.Alias('all_tests', node)
for s in suite_name:
env.Alias(s, node)
if node_name:
env.ComponentTestOutput(node_name, node)
test_name = node_name
# This is rather shady, but the tests need a name without dots so they match
# what gtest does.
# TODO(ncbray) node_name should not be optional.
test_name = os.path.basename(str(node[0].path))
if test_name.endswith('.out'):
test_name = test_name[:-4]
test_name = test_name.replace('.', '_')
SetTestName(node, test_name)
def TestBindsFixedTcpPort(env, node):
# This tells Scons that tests that bind a fixed TCP port should not
# run concurrently, because they would interfere with each other.
# These tests are typically tests for NaCl's GDB debug stub. The
# dummy filename used below is an arbitrary token that just has to
# match across the tests.
SideEffect(env.File('${SCONSTRUCT_DIR}/test_binds_fixed_tcp_port'), node)
# Convenient testing aliases
# NOTE: work around for scons non-determinism in the following two lines
Alias('sel_ldr_sled_tests', [])
Alias('small_tests', [])
Alias('medium_tests', [])
Alias('large_tests', [])
Alias('small_tests_irt', [])
Alias('medium_tests_irt', [])
Alias('large_tests_irt', [])
Alias('pepper_browser_tests', [])
Alias('chrome_browser_tests', [])
Alias('unit_tests', 'small_tests')
Alias('smoke_tests', ['small_tests', 'medium_tests'])
if pre_base_env.Bit('nacl_glibc'):
Alias('memcheck_bot_tests', ['small_tests'])
Alias('tsan_bot_tests', ['small_tests'])
Alias('memcheck_bot_tests', ['small_tests', 'medium_tests', 'large_tests'])
Alias('tsan_bot_tests', [])
def Banner(text):
print '=' * 70
print text
print '=' * 70
# Define the platforms, and use them to define the path for the
# scons-out directory (aka TARGET_ROOT)
# Various variables in the scons environment are related to this, e.g.
# BUILD_ARCH: (arm, mips, x86)
# BUILD_SUBARCH: (32, 64)
DeclareBit('build_x86_32', 'Building binaries for the x86-32 architecture',
DeclareBit('build_x86_64', 'Building binaries for the x86-64 architecture',
DeclareBit('build_mips32', 'Building binaries for the MIPS architecture',
DeclareBit('build_arm_arm', 'Building binaries for the ARM architecture',
# Shorthand for either the 32 or 64 bit version of x86.
DeclareBit('build_x86', 'Building binaries for the x86 architecture')
DeclareBit('build_arm', 'Building binaries for the arm architecture')
def MakeArchSpecificEnv(platform=None):
env = pre_base_env.Clone()
if platform is None:
platform = GetTargetPlatform()
arch = pynacl.platform.GetArch(platform)
if pynacl.platform.IsArch64Bit(platform):
subarch = '64'
subarch = '32'
env.SetBits('build_%s' % platform.replace('-', '_'))
if env.Bit('build_x86_32') or env.Bit('build_x86_64'):
if env.Bit('build_arm_arm'):
if env.Bit('build_mips32'):
# This is a silent default on MIPS.
# Determine where the object files go
# This may be changed later; see target_variant_map, below.
return env
# Valgrind
pre_base_env.AddMethod(lambda self: ARGUMENTS.get('running_on_valgrind'),
DeclareBit('with_leakcheck', 'Running under Valgrind leak checker')
def RunningUnderLeakCheck():
run_under = ARGUMENTS.get('run_under')
if run_under:
extra_args = ARGUMENTS.get('run_under_extra_args')
if extra_args:
run_under += extra_args
if run_under.find('leak-check=full') > 0:
return True
return False
if RunningUnderLeakCheck():
def HasSuffix(item, suffix):
if isinstance(item, str):
return item.endswith(suffix)
elif hasattr(item, '__getitem__'):
return HasSuffix(item[0], suffix)
return item.path.endswith(suffix)
def StripSuffix(string, suffix):
assert string.endswith(suffix)
return string[:-len(suffix)]
def DualLibrary(env, lib_name, *args, **kwargs):
"""Builder to build both .a and _shared.a library in one step.
env: Environment in which we were called.
lib_name: Library name.
args: Positional arguments.
kwargs: Keyword arguments.
static_objs = [i for i in Flatten(args[0]) if not HasSuffix(i, '.os')]
shared_objs = [i for i in Flatten(args[0]) if not HasSuffix(i, '.o')]
# Built static library as ususal.
env.ComponentLibrary(lib_name, static_objs, **kwargs)
# For coverage bots, we only want one object file since two versions would
# write conflicting information to the same .gdca/.gdna files.
if env.Bit('coverage_enabled'): return
# Build a static library using -fPIC for the .o's.
if env.Bit('linux'):
env_shared = env.Clone(OBJSUFFIX='.os')
# -fPIE overrides -fPIC, and shared libraries should not be linked
# as executables.
env_shared.ComponentLibrary(lib_name + '_shared', shared_objs, **kwargs)
# for arm trusted we usually build -static
def DualObject(env, *args, **kwargs):
"""Builder to build both .o and .os in one step.
env: Environment in which we were called.
args: Positional arguments.
kwargs: Keyword arguments.
# Built static library as ususal.
ret = env.ComponentObject(*args, **kwargs)
# For coverage bots, we only want one object file since two versions would
# write conflicting information to the same .gdca/.gdna files.
if env.Bit('coverage_enabled'): return ret
# Build a static library using -fPIC for the .o's.
if env.Bit('linux'):
env_shared = env.Clone(OBJSUFFIX='.os')
ret += env_shared.ComponentObject(*args, **kwargs)
return ret
def AddDualLibrary(env):
# For coverage bots we only build one set of objects and we always set
# '-fPIC' so we do not need a "special" library.
if env.Bit('coverage_enabled'):
env['SHARED_LIBS_SPECIAL'] = env.Bit('linux')
# In prebuild mode we ignore the dependencies so that stuff does
# NOT get build again
# Optionally ignore the build process.
DeclareBit('prebuilt', 'Disable all build steps, only support install steps')
pre_base_env.SetBitFromOption('prebuilt', False)
def GetEmulator(env):
emulator = ARGUMENTS.get('force_emulator')
if emulator is None and 'TRUSTED_ENV' in env:
emulator = env['TRUSTED_ENV'].get('EMULATOR')
return emulator
def UsingEmulator(env):
return bool(env.GetEmulator())
def GetValidator(env, validator):
# NOTE: that the variable TRUSTED_ENV is set by ExportSpecialFamilyVars()
if 'TRUSTED_ENV' not in env:
return None
# TODO(shyamsundarr): rename ncval_new to ncval.
if validator is None:
if env.Bit('build_arm'):
validator = 'arm-ncval-core'
elif env.Bit('build_mips32'):
validator = 'mips-ncval-core'
validator = 'ncval_new'
trusted_env = env['TRUSTED_ENV']
return trusted_env.File('${STAGING_DIR}/${PROGPREFIX}%s${PROGSUFFIX}' %
# Perform os.path.abspath rooted at the directory SConstruct resides in.
def SConstructAbsPath(env, path):
return os.path.normpath(os.path.join(env['MAIN_DIR'], path))
def GetPlatformBuildTargetDir(env):
# Currently we do not support any cross OS compiles, eventually the OS name
# will probably be passed in through arguments.
os_name = pynacl.platform.GetOS()
# Currently 32/64 share the same tool build target directory. When we have
# separate toolchains for each the architectures will probably have to use
# the Arch3264() variant.
build_arch = pynacl.platform.GetArch(GetBuildPlatform())
return '%s_%s' % (os_name, build_arch)
def GetToolchainDir(env, platform_build_dir=None, toolchain_name=None,
target_arch=None, is_pnacl=None, lib_name=None):
if platform_build_dir is None:
platform_build_dir = env.GetPlatformBuildTargetDir()
if toolchain_name is None:
# Fill in default arguments based on environment.
if is_pnacl is None:
# For the purposes of finding the toolchain dir, nacl_clang is PNaCl.
is_pnacl = env.Bit('bitcode') or env.Bit('nacl_clang')
if lib_name is None:
if is_pnacl or not env.Bit('nacl_glibc'):
lib_name = 'newlib'
lib_name = 'glibc'
if target_arch is None:
target_arch = pynacl.platform.GetArch(GetTargetPlatform())
if is_pnacl:
target_env = 'pnacl'
target_env = 'nacl_%s' % target_arch
# See if we have a custom toolchain directory set.
if is_pnacl:
toolchain_arg = 'pnacl_%s_dir' % lib_name
toolchain_arg = 'nacl_%s_dir' % lib_name
custom_toolchain_dir = ARGUMENTS.get(toolchain_arg, None)
if custom_toolchain_dir:
return env.SConstructAbsPath(custom_toolchain_dir)
# Get the standard toolchain name since no directory custom was found.
if is_pnacl:
target_env = 'pnacl'
target_env = 'nacl_%s' % target_arch
toolchain_name = '%s_%s_raw' % (target_env, lib_name)
# Get the absolute path for the platform build directory and toolchain.
toolchain_sub_dir = os.path.join('toolchain',
return env.SConstructAbsPath(toolchain_sub_dir)
def GetSelLdr(env):
sel_ldr = ARGUMENTS.get('force_sel_ldr')
if sel_ldr:
return env.File(env.SConstructAbsPath(sel_ldr))
# NOTE: that the variable TRUSTED_ENV is set by ExportSpecialFamilyVars()
if 'TRUSTED_ENV' not in env:
return None
trusted_env = env['TRUSTED_ENV']
return trusted_env.File('${STAGING_DIR}/${PROGPREFIX}sel_ldr${PROGSUFFIX}')
def GetSelLdrSeccomp(env):
# NOTE: that the variable TRUSTED_ENV is set by ExportSpecialFamilyVars()
if 'TRUSTED_ENV' not in env:
return None
if not (env.Bit('linux') and env.Bit('build_x86_64')):
return None
trusted_env = env['TRUSTED_ENV']
return trusted_env.File('${STAGING_DIR}/${PROGPREFIX}'
def SupportsSeccompBpfSandbox(env):
if not (env.Bit('linux') and env.Bit('build_x86_64')):
return False
# The gcov runtime does some extra calls (such as 'access') that
# are not permitted by the policy.
if env.Bit('coverage_enabled'):
return False
# This is a lame detection if seccomp bpf filters are supported by the kernel.
# We suppose that any Linux kernel v3.2+ supports it, but it is only true
# for Ubuntu kernels. Seccomp BPF filters reached the mainline at 3.5,
# so this check will be wrong on some relatively old non-Ubuntu Linux distros.
kernel_version = map(int, platform.release().split('.', 2)[:2])
return kernel_version >= [3, 2]
def GetBootstrap(env):
if env.Bit('msan'):
# Bootstrap doens't currently work with MSan. However, MSan is only
# available on x86_64 where we don't need bootstrap anyway.
return None, None
if 'TRUSTED_ENV' in env:
trusted_env = env['TRUSTED_ENV']
if trusted_env.Bit('linux'):
template_digits = 'X' * 16
return (trusted_env.File('${STAGING_DIR}/nacl_helper_bootstrap'),
['--r_debug=0x' + template_digits,
'--reserved_at_zero=0x' + template_digits])
return None, None
def AddBootstrap(env, executable, args):
bootstrap, bootstrap_args = env.GetBootstrap()
if bootstrap is None:
return [executable] + args
return [bootstrap, executable] + bootstrap_args + args
def GetNonSfiLoader(env):
if env.Bit('use_newlib_nonsfi_loader'):
return nacl_env.GetTranslatedNexe(nacl_env.File(
if 'TRUSTED_ENV' not in env:
return None
return env['TRUSTED_ENV'].File(
def GetIrtNexe(env, chrome_irt=False):
image = ARGUMENTS.get('force_irt')
if image:
return env.SConstructAbsPath(image)
if chrome_irt:
return nacl_irt_env.File('${STAGING_DIR}/irt.nexe')
return nacl_irt_env.File('${STAGING_DIR}/irt_core.nexe')
def ApplyTLSEdit(env, nexe_name, raw_nexe):
# If the environment was built elsewhere, we do not need to apply tls_edit
# since it only needs to be done during building.
if env.Bit('built_elsewhere'):
return env.File(nexe_name)
tls_edit_exe = env['BUILD_ENV'].File('${STAGING_DIR}/tls_edit${PROGSUFFIX}')
return env.Command(
[tls_edit_exe, raw_nexe],
'${SOURCES[0]} --verbose ${SOURCES[1:]} ${TARGET}')
def CommandValidatorTestNacl(env, name, image,
validator = env.GetValidator(validator)
if validator is None:
print 'WARNING: no validator found. Skipping test %s' % name
return []
if validator_flags is None:
validator_flags = []
if env.Bit('pnacl_generate_pexe'):
return []
command = [validator] + validator_flags + [image]
return env.CommandTest(name, command, size, **extra)
def ExtractPublishedFiles(env, target_name):
run_files = ['$STAGING_DIR/' + os.path.basename(published_file.path)
for published_file in env.GetPublished(target_name, 'run')]
nexe = '$STAGING_DIR/%s${PROGSUFFIX}' % target_name
return [env.File(file) for file in run_files + [nexe]]
# Only include the chrome side of the build if present.
if os.path.exists(pre_base_env.File(
enable_chrome = True
def AddChromeFilesFromGroup(env, file_group):
enable_chrome = False
'Is the chrome side present.')
pre_base_env.SetBitFromOption('enable_chrome_side', enable_chrome)
def ProgramNameForNmf(env, basename):
""" Create an architecture-specific filename that can be used in an NMF URL.
if env.Bit('pnacl_generate_pexe'):
return basename
return '%s_%s' % (basename, env.get('TARGET_FULLARCH'))
def SelUniversalTest(env,
# The dynamic linker's ability to receive arguments over IPC at
# startup currently requires it to reject the plugin's first
# connection, but this interferes with the sel_universal-based
# testing because sel_universal does not retry the connection.
# TODO(mseaborn): Fix by retrying the connection or by adding an
# option to to disable its argv-over-IPC feature.
if env.Bit('nacl_glibc') and not env.Bit('nacl_static_link'):
return []
# TODO(petarj): Sel_universal hangs on qemu-mips. Enable when fixed.
if env.Bit('build_mips32') and env.UsingEmulator():
return []
if sel_universal_flags is None:
sel_universal_flags = []
# When run under qemu, sel_universal must sneak in qemu to the execv
# call that spawns sel_ldr.
if env.UsingEmulator():
if 'TRUSTED_ENV' not in env:
return []
sel_universal = env['TRUSTED_ENV'].File(
# Point to sel_ldr using an environment variable.
sel_ldr = env.GetSelLdr()
if sel_ldr is None:
print 'WARNING: no sel_ldr found. Skipping test %s' % name
return []
kwargs.setdefault('osenv', []).append('NACL_SEL_LDR=' + sel_ldr.abspath)
bootstrap, _ = env.GetBootstrap()
if bootstrap is not None:
kwargs['osenv'].append('NACL_SEL_LDR_BOOTSTRAP=%s' % bootstrap.abspath)
node = CommandSelLdrTestNacl(env,
if not env.Bit('built_elsewhere'):
env.Depends(node, sel_ldr)
if bootstrap is not None:
env.Depends(node, bootstrap)
return node
def MakeNaClLogOption(env, target):
""" Make up a filename related to the [target], for use with NACLLOG.
The file should end up in the build directory (scons-out/...).
# NOTE: to log to the source directory use file.srcnode().abspath instead.
# See
return env.File(target + '.nacllog').abspath
def MakeVerboseExtraOptions(env, target, log_verbosity, extra):
""" Generates **extra options that will give access to service runtime logs,
at a given log_verbosity. Slips the options into the given extra dict. """
log_file = env.MakeNaClLogOption(target)
extra['log_file'] = log_file
extra_env = ['NACLLOG=%s' % log_file,
'NACLVERBOSITY=%d' % log_verbosity]
extra['osenv'] = extra.get('osenv', []) + extra_env
def ShouldUseVerboseOptions(env, extra):
""" Heuristic for setting up Verbose NACLLOG options. """
return ('process_output_single' in extra or
'log_golden' in extra)
DeclareBit('tests_use_irt', 'Non-browser tests also load the IRT image', False)
# Bit to be set by individual test/nacl.scons files that need to opt out.
DeclareBit('nonstable_bitcode', 'Tests use non-stable bitcode features', False)
def GetFinalizedPexe(env, pexe):
""" Prep and finalize the ABI for a given pexe if needed.
if not env.Bit('pnacl_generate_pexe') or env.Bit('nonstable_bitcode'):
return pexe
# We can remove this once we move all CommandSelLdrTestNacl to a nacl.scons
# file instead. There are currently some canned nexe tests in build.scons.
return pexe
# Otherwise, finalize during the build step, since there is no finalize tool
# that can run on triggered bots such as the ARM HW bots.
pexe_name = pexe.abspath
final_name = StripSuffix(pexe_name, '.nonfinal.pexe') + '.final.pexe'
# Make sure the pexe doesn't get removed by the fake builders when
# built_elsewhere=1
node = env.Command(target=final_name, source=[pexe_name],
env.Alias('all_programs', node)
assert len(node) == 1, node
return node[0]
# Translate the given pexe.
def GetTranslatedNexe(env, pexe):
# First finalize the pexe.
pexe = GetFinalizedPexe(env, pexe)
# Then check if we need to translate.
# Check if we started with a pexe, so there is actually a translation step.
if not env.Bit('pnacl_generate_pexe'):
return pexe
# We can remove this once we move all CommandSelLdrTestNacl to a nacl.scons
# file instead. There are currently some canned nexe tests in build.scons.
return pexe
# Often there is a build step (do_not_run_tests=1) and a test step
# (which is run with -j1). Normally we want to translate in the build step
# so we can translate in parallel. However when we do sandboxed translation
# on arm hw, we do the build step on x86 and translation on arm, so we have
# to force the translation to be done in the test step. Hence,
# we check the bit 'translate_in_build_step' / check if we are
# in the test step.
if not env.Bit('translate_in_build_step') and env.Bit('do_not_run_tests'):
return pexe
pexe_name = pexe.abspath
# Tidy up the suffix (remove the .final.pexe or .nonfinal.pexe),
# depending on whether or not the pexe was finalized.
suffix_to_strip = '.final.pexe'
if not pexe_name.endswith(suffix_to_strip):
suffix_to_strip = '.nonfinal.pexe'
nexe_name = StripSuffix(pexe_name, suffix_to_strip) + '.nexe'
# Make sure the pexe doesn't get removed by the fake builders when
# built_elsewhere=1
command = '${TRANSLATECOM}'
if env.Bit('nonstable_bitcode'):
command += ' --allow-llvm-bitcode-input'
node = env.Command(target=nexe_name, source=[pexe_name],
action=[Action(command, '${TRANSLATECOMSTR}')])
env.Alias('all_programs', node)
assert len(node) == 1, node
return node[0]
def CommandTestFileDumpCheck(env,
"""Create a test that disassembles a binary (|target|) and checks for
patterns in the |check_file|. Disassembly is done using |objdump_flags|.
# Do not try to run OBJDUMP if 'built_elsewhere', since that *might* mean
# that a toolchain is not even present. E.g., the arm hw buildbots do
# not have the pnacl toolchain. We should be able to look for the host
# ARM objdump though... a TODO(jvoung) for when there is time.
if env.Bit('built_elsewhere'):
return []
target = env.GetTranslatedNexe(target)
return env.CommandTestFileCheck(name,
['${OBJDUMP}', objdump_flags, target],
def CommandTestFileCheck(env, name, cmd, check_file):
"""Create a test that runs a |cmd| (array of strings),
which is expected to print to stdout. The results
of stdout will then be piped to the tool which
will search for the regexes specified in |check_file|. """
return env.CommandTest(name,
check_file] + cmd,
# don't run ${PYTHON} under the emulator.
def CommandSelLdrTestNacl(env, name, nexe,
args = None,
# True for *.nexe statically linked with glibc
# e.g., [ 'python', '', '--' ]
# Disable all sel_ldr tests for windows under coverage.
# Currently several .S files block sel_ldr from being instrumented.
# See
if ('TRUSTED_ENV' in env and
env['TRUSTED_ENV'].Bit('coverage_enabled') and
return []
# The nexe might be a pexe that needs finalization, and translation.
nexe = env.GetTranslatedNexe(nexe)
command = [nexe]
if args is not None:
command += args
if env.Bit('pnacl_unsandboxed') or (env.Bit('nonsfi_nacl') and
not env.Bit('tests_use_irt')):
# Run unsandboxed executable directly, without sel_ldr.
return env.CommandTest(name, command, size, **extra)
if loader is None:
if env.Bit('nonsfi_nacl'):
loader = env.GetNonSfiLoader()
loader = env.GetSelLdr()
if loader is None:
print 'WARNING: no sel_ldr found. Skipping test %s' % name
return []
# Avoid problems with [] as default arguments
if sel_ldr_flags is None:
sel_ldr_flags = []
# Avoid modifying original list
sel_ldr_flags = list(sel_ldr_flags)
# Disable the validator if running a GLibC test under Valgrind.
if env.IsRunningUnderValgrind() and env.Bit('nacl_glibc'):
sel_ldr_flags += ['-cc']
# We don't currently have for LD_PRELOAD to use. That .so
# is not used for newlib.
# TODO(sehr): add built for NaCl.
return []
# Skip platform qualification checks on configurations with known issues.
if env.GetEmulator() or env.IsRunningUnderValgrind():
sel_ldr_flags += ['-Q']
# Skip validation if we are using the x86-64 zero-based sandbox.
# TODO(arbenson): remove this once the validator supports the x86-64
# zero-based sandbox model.
if env.Bit('x86_64_zero_based_sandbox'):
sel_ldr_flags += ['-c']
# The glibc modifications only make sense for nacl_env tests.
# But this function gets used by some base_env (i.e. src/trusted/...)
# tests too. Don't add the --nacl_glibc changes to the command
# line for those cases.
if env.Bit('nacl_glibc') and env['NACL_BUILD_FAMILY'] != 'TRUSTED':
if not glibc_static and not env.Bit('nacl_static_link'):
command = ['${NACL_SDK_LIB}/',
# Locally-built shared libraries come from ${LIB_DIR}
# while toolchain-provided ones come from ${NACL_SDK_LIB}.
'--library-path', '${LIB_DIR}:${NACL_SDK_LIB}'] + command
# Enable file access.
sel_ldr_flags += ['-a']
# Turn off sandbox for mac so coverage files can be written out.
if ('TRUSTED_ENV' in env and
env['TRUSTED_ENV'].Bit('coverage_enabled') and
env.Bit('host_mac') and
'-a' not in sel_ldr_flags):
sel_ldr_flags += ['-a']
if env.Bit('tests_use_irt'):
sel_ldr_flags += ['-B', nacl_env.GetIrtNexe()]
if skip_bootstrap:
loader_cmd = [loader]
loader_cmd = env.AddBootstrap(loader, [])
if env.Bit('nonsfi_nacl'):
# nonsfi_loader does not accept the same flags as sel_ldr yet, so
# we ignore sel_ldr_flags here.
command = [loader] + command
command = loader_cmd + sel_ldr_flags + ['--'] + command
if env.Bit('host_linux'):
extra['using_nacl_signal_handler'] = True
if env.ShouldUseVerboseOptions(extra):
env.MakeVerboseExtraOptions(name, log_verbosity, extra)
node = env.CommandTest(name, command, size, posix_path=True,
wrapper_program_prefix=wrapper_program_prefix, **extra)
if env.Bit('tests_use_irt'):
env.Alias('irt_tests', node)
return node
TEST_EXTRA_ARGS = ['stdin', 'log_file',
'stdout_golden', 'stderr_golden', 'log_golden',
'filter_regex', 'filter_inverse', 'filter_group_only',
'osenv', 'arch', 'subarch', 'exit_status',
'num_runs', 'process_output_single',
'process_output_combined', 'using_nacl_signal_handler',
'declares_exit_status', 'time_warning', 'time_error']
'small': 2,
'medium': 10,
'large': 60,
'huge': 1800,
# Valgrind handles SIGSEGV in a way our testing tools do not expect.
'untrusted_sigill' ,
def GetPerfEnvDescription(env):
"""Return a string describing architecture, library, etc. options.
This function attempts to gather a string that might inform why a performance
change has occurred.
# Trusted tests do not depend on the untrusted toolchain, untrusted libc,
# whether or not the IRT is used, etc.
description_list = ['trusted',
return ARGUMENTS.get('perf_prefix', '') + '_'.join(description_list)
description_list = [env['TARGET_FULLARCH']]
# Using a list to keep the order consistent.
bit_to_description = [ ('tests_use_irt', ('with_irt', '')),
('bitcode', ('pnacl', 'nnacl')),
('translate_fast', ('fast', '')),
('nacl_glibc', ('glibc', 'newlib')),
('nacl_static_link', ('static', 'dynamic')),
for (bit, (descr_yes, descr_no)) in bit_to_description:
if env.Bit(bit):
additional = descr_yes
additional = descr_no
if additional:
return ARGUMENTS.get('perf_prefix', '') + '_'.join(description_list)
def GetTestName(target):
key = str(target.path)
return TEST_NAME_MAP.get(key, key)
pre_base_env['GetTestName'] = GetTestName
def SetTestName(node, name):
for target in Flatten(node):
TEST_NAME_MAP[str(target.path)] = name
def ApplyTestWrapperCommand(command_args, extra_deps):
new_args = ARGUMENTS['test_wrapper'].split()
for input_file in extra_deps:
new_args.extend(['-F', input_file])
for arg in command_args:
if isinstance(arg, str):
new_args.extend(['-a', arg])
new_args.extend(['-f', arg])
return new_args
def CommandTest(env, name, command, size='small', direct_emulation=True,
extra_deps=[], posix_path=False, capture_output=True,
capture_stderr=True, wrapper_program_prefix=None,
scale_timeout=None, **extra):
if not name.endswith('.out') or name.startswith('$'):
raise Exception('ERROR: bad test filename for test output %r' % name)
if env.IsRunningUnderValgrind():
skip = 'Valgrind'
elif env.Bit('asan'):
skip = 'AddressSanitizer'
skip = None
# Valgrind tends to break crash tests by changing the exit status.
# So far, tests using declares_exit_status are crash tests. If this
# changes, we will have to find a way to make declares_exit_status
# work with Valgrind.
if (skip is not None and
(extra.get('exit_status') in UNSUPPORTED_VALGRIND_EXIT_STATUS or
bool(int(extra.get('declares_exit_status', 0))))):
print 'Skipping death test "%s" under %s' % (name, skip)
return []
if env.Bit('asan'):
extra.setdefault('osenv', [])
# Ensure that 'osenv' is a list.
if isinstance(extra['osenv'], str):
extra['osenv'] = [extra['osenv']]
# ASan normally intercepts SIGSEGV and disables our SIGSEGV signal
# handler, which interferes with various NaCl tests, including the
# platform qualification test built into sel_ldr. We fix this by
# telling ASan not to mess with SIGSEGV.
asan_options = ['handle_segv=0']
if env.Bit('host_mac') and int(platform.mac_ver()[0].split('.')[1]) < 7:
# MacOS 10.6 has a bug in the libsandbox system library where it
# makes a memcmp call that reads off the end of a malloc'd block.
# The bug appears to be harmless, but trips an ASan report. So
# tell ASan to suppress memcmp checks.
# TODO(mcgrathr): Remove this when we clean up all the crufty old
# code to be leak-free.
# Note that the ASan runtime doesn't use : specifically as a separator.
# It actually just looks for "foo=" anywhere in the string with strstr,
# so any separator will do. The most obvious choices, ' ', ',', and ';'
# all cause to split things up and get confused.
extra['osenv'].append('ASAN_OPTIONS=' + ':'.join(asan_options))
name = '${TARGET_ROOT}/test_results/' + name
# NOTE: using the long version of 'name' helps distinguish opt vs dbg
max_time = TEST_TIME_THRESHOLD[size]
if 'scale_timeout' in ARGUMENTS:
max_time = max_time * int(ARGUMENTS['scale_timeout'])
if scale_timeout:
max_time = max_time * scale_timeout
if env.Bit('nacl_glibc'):
suite = 'nacl_glibc'
suite = 'nacl_newlib'
if env.Bit('bitcode'):
suite = 'p' + suite
script_flags = ['--name', '%s.${GetTestName(TARGET)}' % suite,
'--time_warning', str(max_time),
'--time_error', str(10 * max_time),
run_under = ARGUMENTS.get('run_under')
if run_under:
run_under_extra_args = ARGUMENTS.get('run_under_extra_args')
if run_under_extra_args:
run_under = run_under + ',' + run_under_extra_args
emulator = env.GetEmulator()
if emulator and direct_emulation:
command = [emulator] + command
# test wrapper should go outside of emulators like qemu, since the
# test wrapper code is not emulated.
if wrapper_program_prefix is not None:
command = wrapper_program_prefix + command
# Add architecture info.
extra['arch'] = env['BUILD_ARCHITECTURE']
extra['subarch'] = env['BUILD_SUBARCH']
for flag_name, flag_value in extra.iteritems():
assert flag_name in TEST_EXTRA_ARGS, repr(flag_name)
if isinstance(flag_value, list):
# Options to which are actually lists must not be
# separated by whitespace. This stringifies the lists with a separator
# char to satisfy command_tester.
flag_value = command_tester.StringifyList(flag_value)
# do not add --flag + |flag_name| |flag_value| if
# |flag_value| is false (empty).
if flag_value:
script_flags.append('--' + flag_name)
# Make sure flag values are strings (or SCons objects) when building
# up the command. Right now, this only means convert ints to strings.
if isinstance(flag_value, int):
flag_value = str(flag_value)
# Other extra flags
if not capture_output:
script_flags.extend(['--capture_output', '0'])
if not capture_stderr:
script_flags.extend(['--capture_stderr', '0'])
# Set's output filename. We skip this when using
# test_wrapper because the wrapper does not have
# the ability to copy result files back from the remote host.
if 'test_wrapper' not in ARGUMENTS:
script_flags.extend(['--output_stamp', name])
test_script = env.File('${SCONSTRUCT_DIR}/tools/')
extra_deps = extra_deps + [env.File('${SCONSTRUCT_DIR}/tools/')]
command = ['${PYTHON}', test_script] + script_flags + command
if 'test_wrapper' in ARGUMENTS:
command = ApplyTestWrapperCommand(command, extra_deps)
return env.AutoDepsCommand(name, command,
extra_deps=extra_deps, posix_path=posix_path,
def FileSizeTest(env, name, envFile, max_size=None):
"""FileSizeTest() returns a scons node like the other XYZTest generators.
It logs the file size of envFile in a perf-buildbot-recognizable format.
Optionally, it can cause a test failure if the file is larger than max_size.
def doSizeCheck(target, source, env):
filepath = source[0].abspath
actual_size = os.stat(filepath).st_size
'%.3f' % (actual_size / 1024.0),
# Also get zipped size.
nexe_file = open(filepath, 'rb')
zipped_size = len(zlib.compress(
'ZIPPED_' + env.GetPerfEnvDescription(),
'%.3f' % (zipped_size / 1024.0),
# Finally, do the size check.
if max_size is not None and actual_size > max_size:
# NOTE: this exception only triggers a failure for this particular test,
# just like any other test failure.
raise Exception("File %s larger than expected: expected up to %i, got %i"
% (filepath, max_size, actual_size))
# If 'built_elsewhere', the file should should have already been built.
# Do not try to built it and/or its pieces.
if env.Bit('built_elsewhere'):
env.Ignore(name, envFile)
return env.Command(name, envFile, doSizeCheck)
def StripExecutable(env, name, exe):
"""StripExecutable returns a node representing the stripped version of |exe|.
The stripped version will be given the basename |name|.
NOTE: for now this only works with the untrusted toolchain.
STRIP does not appear to be a first-class citizen in SCons and
STRIP has only been set to point at the untrusted toolchain.
return env.Command(
action=[Action('${STRIPCOM} ${SOURCES} -o ${TARGET}', '${STRIPCOMSTR}')])
# TODO(ncbray): pretty up the log output when running this builder.
def DisabledCommand(target, source, env):
pre_base_env['BUILDERS']['DisabledCommand'] = Builder(action=DisabledCommand)
def AutoDepsCommand(env, name, command, extra_deps=[], posix_path=False,
"""AutoDepsCommand() takes a command as an array of arguments. Each
argument may either be:
* a string, or
* a Scons file object, e.g. one created with env.File() or as the
result of another build target.
In the second case, the file is automatically declared as a
dependency of this command.
command = list(command)
deps = []
for index, arg in enumerate(command):
if not isinstance(arg, str):
if len(Flatten(arg)) != 1:
# Do not allow this, because it would cause "deps" to get out
# of sync with the indexes in "command".
# See
raise AssertionError('Argument to AutoDepsCommand() actually contains '
'multiple (or zero) arguments: %r' % arg)
if posix_path:
command[index] = '${SOURCES[%d].posix}' % len(deps)
command[index] = '${SOURCES[%d].abspath}' % len(deps)
# If built_elsewhere, build commands are replaced by no-ops, so make sure
# the targets don't get removed first
if env.Bit('built_elsewhere'):
env.Depends(name, extra_deps)
if disabled:
return env.DisabledCommand(name, deps)
return env.Command(name, deps, ' '.join(command))
def GetPrintableCommandName(cmd):
"""Look at the first few elements of cmd to derive a suitable command name."""
cmd_tokens = cmd.split()
if "python" in cmd_tokens[0] and len(cmd_tokens) >= 2:
cmd_name = cmd_tokens[1]
cmd_name = cmd_tokens[0].split('(')[0]
# undo some pretty printing damage done by hammer
cmd_name = cmd_name.replace('________','')
# use file name part of a path
return cmd_name.split('/')[-1]
def GetPrintableEnvironmentName(env):
# use file name part of a obj root path as env name
return env.subst('${TARGET_ROOT}').split('/')[-1]
def CustomCommandPrinter(cmd, targets, source, env):
# Abuse the print hook to count the commands that are executed
if env.Bit('target_stats'):
cmd_name = GetPrintableCommandName(cmd)
env_name = env.GetPrintableEnvironmentName()
CMD_COUNTER[cmd_name] = CMD_COUNTER.get(cmd_name, 0) + 1
ENV_COUNTER[env_name] = ENV_COUNTER.get(env_name, 0) + 1
if env.Bit('pp'):
# Our pretty printer
if targets:
cmd_name = GetPrintableCommandName(cmd)
env_name = env.GetPrintableEnvironmentName()
sys.stdout.write('[%s] [%s] %s\n' % (cmd_name, env_name,
# The SCons default (copied from print_cmd_line in
sys.stdout.write(cmd + u'\n')
if 'generate_ninja' not in ARGUMENTS:
def GetAbsDirArg(env, argument, target):
"""Fetch the named command-line argument and turn it into an absolute
directory name. If the argument is missing, raise a UserError saying
that the given target requires that argument be given."""
dir = ARGUMENTS.get(argument)
if not dir:
raise UserError('%s must be set when invoking %s' % (argument, target))
return os.path.join(env.Dir('$MAIN_DIR').abspath, dir)
def MakeGTestEnv(env):
# Create an environment to run unit tests using Gtest.
gtest_env = env.Clone()
# This became necessary for the arm cross TC v4.6
# but probable applies to all new gcc TCs
# Define compile-time flag that communicates that we are compiling in the test
# environment (rather than for the TCB).
if gtest_env['NACL_BUILD_FAMILY'] == 'TRUSTED':
# This is necessary for unittest_main.c which includes gtest/gtest.h
# The problem is that gtest.h includes other files expecting the
# include path to be set.
# gtest does not compile with our stringent settings.
if gtest_env.Bit('linux') or gtest_env.Bit('mac'):
# "-pedantic" is because of: gtest-typed-test.h:236:46: error:
# anonymous variadic macros were introduced in C99
# Also, gtest does not compile successfully with "-Wundef".
gtest_env.FilterOut(CCFLAGS=['-pedantic', '-Wundef'])
gtest_env.FilterOut(CXXFLAGS=['-fno-rtti', '-Weffc++'])
# gtest is incompatible with static linking due to obscure libstdc++
# linking interactions.
# See
return gtest_env
def MakeUntrustedNativeEnv(env):
native_env = nacl_env.Clone()
if native_env.Bit('bitcode') and not native_env.Bit('build_mips32'):
native_env = native_env.PNaClGetNNaClEnv()
return native_env
def MakeBaseTrustedEnv(platform=None):
base_env = MakeArchSpecificEnv(platform)
if base_env.Bit('ncval_testing'):
base_env.Append(CPPDEFINES = ['NCVAL_TESTING'])
base_env.Append(BUILD_SCONSCRIPTS = [
# The ARM and MIPS validators can be built for any target that doesn't use
if not base_env.Bit('build_x86_64'):
# Add optional scons files if present in the directory tree.
if os.path.exists(pre_base_env.subst('${MAIN_DIR}/supplement/build.scons')):
return base_env
# Select tests to run under coverage build.
pre_base_env['COVERAGE_TARGETS'] = [
'small_tests', 'medium_tests', 'large_tests',
Help for NaCl
Common tasks:
* cleaning: scons -c
* building: scons
* build mandel: scons --mode=nacl mandel.nexe
* smoke test: scons --mode=nacl,opt-linux -k pp=1 smoke_tests
* sel_ldr: scons --mode=opt-linux sel_ldr
* build the plugin: scons --mode=opt-linux ppNaClPlugin
* or: scons --mode=opt-linux src/trusted/plugin
Targets to build trusted code destined for the SDK:
* build trusted-code tools: scons build_bin
* install trusted-code tools: scons install_bin bindir=...
* These default to opt build, or add --mode=dbg-host for debug build.
Targets to build untrusted code destined for the SDK:
* build just libraries: scons build_lib
* install just headers: scons install_headers includedir=...
* install just libraries: scons install_lib libdir=...
* install headers and libraries:scons install includedir=... libdir=...
* dump system info: scons --mode=nacl,opt-linux dummy
--prebuilt Do not build things, just do install steps
--verbose Full command line logging before command execution
pp=1 Use command line pretty printing (more concise output)
sysinfo=1 Verbose system info printing
naclsdk_validate=0 Suppress presence check of sdk
Automagically generated help:
def SetUpClang(env):
env['CLANG_DIR'] = '${SOURCE_ROOT}/third_party/llvm-build/Release+Asserts/bin'
env['CLANG_OPTS'] = []
if env.Bit('asan'):
if not (env.Bit('host_linux') or env.Bit('host_mac')):
raise UserError("ERROR: ASan is only available for Linux and Mac")
if env.Bit('host_mac'):
# The built executables will try to find this library at runtime
# in the directory containing the executable itself. In the
# Chromium build, the library just gets copied into that
# directory. Here, there isn't a single directory from which
# all the test binaries are run (sel_ldr is run from staging/
# but other trusted test binaries are run from their respective
# obj/.../ directories). So instead just point the dynamic linker
# at the right directory using an environment variable.
clang_lib_dir = '${CLANG_DIR}/../lib/clang/*/lib/darwin'
env['ENV']['DYLD_LIBRARY_PATH'] = ':'.join([dir.abspath for dir in
if 'PROPAGATE_ENV' not in env:
env['PROPAGATE_ENV'] = []
if env.Bit('msan'):
if not env.Bit('host_linux') or not env.Bit('build_x86_64'):
raise UserError('ERROR: MSan is only available for x86-64 Linux')
track_origins = ARGUMENTS.get('msan_track_origins', '1')
'-fsanitize-memory-track-origins=%s' % track_origins,
env['CC'] = '${CLANG_DIR}/clang ${CLANG_OPTS}'
env['CXX'] = '${CLANG_DIR}/clang++ ${CLANG_OPTS}'
# Make sure we find Clang-supplied libraries like -lprofile_rt
# in the Clang build we use, rather than from the system.
# The system-installed versions go with the system-installed Clang
# and might not be compatible with the Clang we're running.
def GenerateOptimizationLevels(env):
if env.Bit('clang'):
# Generate debug variant.
debug_env = env.Clone(tools = ['target_debug'])
debug_env['OPTIMIZATION_LEVEL'] = 'dbg'
debug_env['BUILD_TYPE'] = debug_env.subst('$BUILD_TYPE')
debug_env['BUILD_DESCRIPTION'] = debug_env.subst('$BUILD_DESCRIPTION')
# Add to the list of fully described environments.
# Generate opt variant.
opt_env = env.Clone(tools = ['target_optimized'])
opt_env['OPTIMIZATION_LEVEL'] = 'opt'
opt_env['BUILD_TYPE'] = opt_env.subst('$BUILD_TYPE')
opt_env['BUILD_DESCRIPTION'] = opt_env.subst('$BUILD_DESCRIPTION')
# Add to the list of fully described environments.
return (debug_env, opt_env)
def SDKInstallBin(env, name, node, target=None):
"""Add the given node to the build_bin and install_bin targets.
It will be installed under the given name with the build target appended.
The optional target argument overrides the setting of what that target is."""
env.Alias('build_bin', node)
if 'install_bin' in COMMAND_LINE_TARGETS:
dir = env.GetAbsDirArg('bindir', 'install_bin')
if target is None:
target = env['TARGET_FULLARCH'].replace('-', '_')
file_name, file_ext = os.path.splitext(name)
output_name = file_name + '_' + target + file_ext
install_node = env.InstallAs(os.path.join(dir, output_name), node)
env.Alias('install_bin', install_node)
def MakeWindowsEnv(platform=None):
base_env = MakeBaseTrustedEnv(platform)
windows_env = base_env.Clone(
tools = ['target_platform_windows'],
# Windows /SAFESEH linking requires either an .sxdata section be
# present or that @feat.00 be defined as a local, absolute symbol
# with an odd value.
'$WINASM -defsym @feat.00=1 -o $TARGET'),
PDB = '${TARGET.base}.pdb',
# Strict doesn't currently work for Windows since some of the system
# libraries like wsock32 are magical.
TARGET_ARCH='x86_64' if base_env.Bit('build_x86_64') else 'x86',
['_WIN32_WINNT', '0x0501'],
['NOMINMAX', '1'],
# WIN32 is used by ppapi
['WIN32', '1'],
# WIN32_LEAN_AND_MEAN tells windows.h to omit obsolete and rarely
# used #include files. This allows use of Winsock 2.0 which otherwise
# would conflict with Winsock 1.x included by windows.h.
['WIN32_LEAN_AND_MEAN', ''],
LIBS = ['ws2_32', 'advapi32'],
# TODO(bsy) remove 4355 once cross-repo
CCFLAGS = ['/EHsc', '/WX', '/wd4355', '/wd4800',
# build_config.h is injected as a header in all sources to
# provide macro definitions for the operating system and
# architecture. This is injected so it's never accidentally
# omitted in source files.
'/FI', '$SOURCE_ROOT/native_client/src/include/build_config.h']
# This linker option allows us to ensure our builds are compatible with
# Chromium, which uses it.
if windows_env.Bit('build_x86_32'):
windows_env.Append(LINKFLAGS = "/safeseh")
# We use the GNU assembler (gas) on Windows so that we can use the
# same .S assembly files on all platforms. Microsoft's assembler uses
# a completely different syntax for x86 code.
if windows_env.Bit('build_x86_64'):
# This assembler only works for x86-64 code.
windows_env['WINASM'] = \
# This assembler only works for x86-32 code.
windows_env['WINASM'] = \
return windows_env
windows_optimized_env) = GenerateOptimizationLevels(MakeWindowsEnv())
def MakeUnixLikeEnv(platform=None):
unix_like_env = MakeBaseTrustedEnv(platform)
# -Wdeclaration-after-statement is desirable because MS studio does
# not allow declarations after statements in a block, and since much
# of our code is portable and primarily initially tested on Linux,
# it'd be nice to get the build error earlier rather than later
# (building and testing on Linux is faster).
# TODO(nfullagar): should we consider switching to -std=c99 ?
# Require defining functions as "foo(void)" rather than
# "foo()" because, in C (but not C++), the latter defines a
# function with unspecified arguments rather than no
# arguments.
# '-malign-double',
# build_config.h is injected as a header in all sources to provide
# macro definitions for the operating system and architecture. This is
# injected so it's never accidentally omitted in source files.
'-include', '$SOURCE_ROOT/native_client/src/include/build_config.h',
] + werror_flags,
# NOTE: pthread is only neeeded for and on arm
LIBS = ['pthread'],
# Android's stlport uses __STRICT_ANSI__ to exclude "long long".
# This breaks basically all C++ code that uses stlport.
if not unix_like_env.Bit('android'):
if not unix_like_env.Bit('clang'):
unix_like_env.Append(CCFLAGS=['--param', 'ssp-buffer-size=4'])
if unix_like_env.Bit('enable_tmpfs_redirect_var'):
unix_like_env.Append(CPPDEFINES=[['NACL_ENABLE_TMPFS_REDIRECT_VAR', '1']])
unix_like_env.Append(CPPDEFINES=[['NACL_ENABLE_TMPFS_REDIRECT_VAR', '0']])
return unix_like_env
def MakeMacEnv(platform=None):
mac_env = MakeUnixLikeEnv(platform).Clone(
tools = ['target_platform_mac'],
# TODO(bradnelson): this should really be able to live in unix_like_env
# but can't due to what the target_platform_x module is
# doing.
LINK = '$CXX',
PLUGIN_SUFFIX = '.bundle',
# On Mac, only the newer clang toolchains can parse some of the trusted
# code's assembly syntax, so turn clang on by default.
# This should be kept in synch with mac_deployment_target
# in build/common.gypi, which in turn should be kept in synch
# with chromium/src/build/common.gypi.
mac_deployment_target = '10.6'
CCFLAGS=['-mmacosx-version-min=' + mac_deployment_target],
LINKFLAGS=['-mmacosx-version-min=' + mac_deployment_target])
subarch_flag = '-m%s' % mac_env['BUILD_SUBARCH']
CCFLAGS=[subarch_flag, '-fPIC'],
LINKFLAGS=[subarch_flag, '-fPIC'],
CPPDEFINES = [# defining _DARWIN_C_SOURCE breaks 10.4
#['_DARWIN_C_SOURCE', '1'],
return mac_env
(mac_debug_env, mac_optimized_env) = GenerateOptimizationLevels(MakeMacEnv())
def which(cmd, paths=os.environ.get('PATH', '').split(os.pathsep)):
for p in paths:
if os.access(os.path.join(p, cmd), os.X_OK):
return True
return False
def SetUpLinuxEnvArm(env):
jail = env.GetToolchainDir(toolchain_name='arm_trusted')
if not platform.machine().startswith('arm'):
# Allow emulation on non-ARM hosts.
env.Replace(EMULATOR=jail + '/run_under_qemu_arm')
if env.Bit('built_elsewhere'):
def FakeInstall(dest, source, env):
print 'Not installing', dest
# Replace build commands with no-ops
env.Replace(CC='true', CXX='true', LD='true',
AR='true', RANLIB='true', INSTALL=FakeInstall)
# The -rpath-link argument is needed on Ubuntu/Precise to
# avoid linker warnings about missing
# TODO(sbc): remove this once we stop supporting Precise
# as a build environment.
LINKFLAGS=['-Wl,-rpath-link=' + jail +
# Note we let the compiler choose whether it's -marm or -mthumb by
# default. The hope is this will have the best chance of testing
# the prevailing compilation mode used for Chromium et al.
# has a dependency on dladdr
def SetUpAndroidEnv(env):
env.FilterOut(CPPDEFINES=[['_LARGEFILE64_SOURCE', '1']])
android_ndk_root = os.path.join('${SOURCE_ROOT}', 'third_party',
'android_tools', 'ndk')
android_ndk_experimental_root = os.path.join('${SOURCE_ROOT}',
'third_party', 'android_tools',
android_sdk_root = os.path.join('${SOURCE_ROOT}', 'third_party',
'android_tools', 'sdk')
android_sdk_version = 21
android_stlport_root = os.path.join(android_ndk_root, 'sources', 'cxx-stl',
ndk_host_os_map = {
pynacl.platform.OS_WIN : 'win',
pynacl.platform.OS_MAC: 'darwin',
pynacl.platform.OS_LINUX : 'linux'
host_os = ndk_host_os_map[pynacl.platform.GetOS()]
android_sdk = os.path.join(android_sdk_root, 'platforms',
'android-%s' % android_sdk_version)
arch_cflags = []
if env.Bit('build_arm'):
android_ndk_target_prefix = 'arm-linux-androideabi'
android_ndk_version = '4.8'
android_app_abi = 'armeabi-v7a'
android_ndk_sysroot = os.path.join(android_ndk_root, 'platforms',
'android-14', 'arch-arm')
android_ndk_lib_dir = os.path.join('usr', 'lib')
android_toolchain = os.path.join(android_ndk_root, 'toolchains',
'arm-linux-androideabi-4.8', 'prebuilt',
'%s-x86_64' % host_os, 'bin')
arch_cflags += ['-march=armv7-a', '-mfloat-abi=softfp']
elif env.Bit('build_x86_32'):
android_ndk_target_prefix = 'i686-linux-android'
android_ndk_version = '4.8'
android_app_abi = 'x86'
android_ndk_sysroot = os.path.join(android_ndk_root, 'platforms',
'android-14', 'arch-x86')
android_ndk_lib_dir = os.path.join('usr', 'lib')
android_toolchain = os.path.join(android_ndk_root, 'toolchains',
'x86-4.8', 'prebuilt',
'%s-x86_64' % host_os, 'bin')
arch_cflags += ['-m32', '-msse2']
# TODO(sehr): add other android architecture platform settings here.
android_ndk_include = os.path.join(android_ndk_sysroot, 'usr', 'include')
android_ndk_lib = os.path.join(android_ndk_sysroot, android_ndk_lib_dir)
android_sdk_jar = os.path.join(android_sdk, 'android.jar')
android_stlport_include = os.path.join(android_stlport_root, 'stlport')
android_stlport_libs_dir = os.path.join(android_stlport_root, 'libs',
android_ndk_libgcc_path = os.path.join(android_toolchain, '..', 'lib', 'gcc',
'%s-gcc' % android_ndk_target_prefix),
'%s-g++' % android_ndk_target_prefix),
'%s-g++' % android_ndk_target_prefix),
'%s-ar' % android_ndk_target_prefix),
'%s-ranlib' % android_ndk_target_prefix),
'%s-readelf' % android_ndk_target_prefix),
'%s-strip' % android_ndk_target_prefix),
EMULATOR=os.path.join(android_sdk_root, 'tools', 'emulator'),
os.path.join(android_stlport_root, 'libs',
# SHLINKFLAGS should not inherit options from LINKFLAGS.
env.Append(CCFLAGS=['--sysroot=' + android_ndk_sysroot,
'-isystem=' + os.path.join(android_ndk_sysroot, 'usr',
'-I%s' % android_stlport_include,
# Due to bogus warnings on uintptr_t formats.
] + arch_cflags,
CXXFLAGS=['-I%s' % android_stlport_include,
'-I%s' % android_ndk_include,
LINKFLAGS=['--sysroot=' + android_ndk_sysroot,
# Don't export symbols from statically linked libraries.
# crtbegin_dynamic.o should be the last item in ldflags.
os.path.join(android_ndk_lib, 'crtbegin_dynamic.o'),
SHLINKFLAGS=['--sysroot=' + android_ndk_sysroot,
# crtbegin_so.o should be the last item in ldflags.
os.path.join(android_ndk_lib, 'crtbegin_so.o'),
return env
def SetUpLinuxEnvMips(env):
jail = env.GetToolchainDir(toolchain_name='mips_trusted')
if not platform.machine().startswith('mips'):
# Allow emulation on non-MIPS hosts.
env.Replace(EMULATOR=jail + '/run_under_qemu_mips32')
if env.Bit('built_elsewhere'):
def FakeInstall(dest, source, env):
print 'Not installing', dest
# Replace build commands with no-ops
env.Replace(CC='true', CXX='true', LD='true',
AR='true', RANLIB='true', INSTALL=FakeInstall)
tc_dir = os.path.join(jail, 'bin')
if not which(os.path.join(tc_dir, 'mipsel-linux-gnu-gcc')):
raise UserError("\nERRROR: "
"MIPS trusted TC is not installed - try running:\n"
" nacl_sdk")
env.Replace(CC=os.path.join(tc_dir, 'mipsel-linux-gnu-gcc'),
CXX=os.path.join(tc_dir, 'mipsel-linux-gnu-g++'),
LD=os.path.join(tc_dir, 'mipsel-linux-gnu-ld'),
jail + '/sysroot/usr/lib'],
os.path.join(jail, 'ld_script_mips_trusted')]
env.Append(LIBS=['rt', 'dl', 'pthread'],
# Makes a generic Linux development environment.
# Linux development environments are used in two different ways.
# 1) To produce trusted tools (e.g., sel_ldr), called TRUSTED_ENV
# 2) To produce build tools (e.g., tls_edit), called BUILD_ENV
def MakeGenericLinuxEnv(platform=None):
linux_env = MakeUnixLikeEnv(platform).Clone(
tools = ['target_platform_linux'],
# TODO(bradnelson): this should really be able to live in unix_like_env
# but can't due to what the target_platform_x module is
# doing.
LINK = '$CXX',
# Prepend so we can disable warnings via Append
['_BSD_SOURCE', '1'],
['_POSIX_C_SOURCE', '199506'],
['_XOPEN_SOURCE', '600'],
['_GNU_SOURCE', '1'],
LIBS = ['rt'],
if linux_env.Bit('build_x86_32'):
CCFLAGS = ['-m32'],
LINKFLAGS = ['-m32'],
elif linux_env.Bit('build_x86_64'):
CCFLAGS = ['-m64'],
LINKFLAGS = ['-m64'],
elif linux_env.Bit('build_arm'):
elif linux_env.Bit('build_mips32'):
Banner('Strange platform: %s' % GetTargetPlatform())
# These are desireable options for every Linux platform:
# _FORTIFY_SOURCE: general paranoia "hardening" option for library functions
# -fPIE/-pie: create a position-independent executable
# relro/now: "hardening" options for linking
# noexecstack: ensure that the executable does not get a PT_GNU_STACK
# header that causes the kernel to set the READ_IMPLIES_EXEC
# personality flag, which disables NX page protection.
LINKFLAGS=['-pie', '-Wl,-z,relro', '-Wl,-z,now', '-Wl,-z,noexecstack'],
# The ARM toolchain has a linker that doesn't handle the code its
# compiler generates under -fPIE.
if linux_env.Bit('build_arm') or linux_env.Bit('build_mips32'):
# TODO(mcgrathr): Temporarily punt _FORTIFY_SOURCE for ARM because
# it causes a libc dependency newer than the old bots have installed.
linux_env.FilterOut(CPPDEFINES=[['-D_FORTIFY_SOURCE', '2']])
# We always want to use the same flags for .S as for .c because
# code-generation flags affect the predefines we might test there.
return linux_env
# Specializes a generic Linux development environment to be a trusted
# environment.
def MakeTrustedLinuxEnv(platform=None):
linux_env = MakeGenericLinuxEnv(platform)
if linux_env.Bit('android'):
return linux_env
(linux_debug_env, linux_optimized_env) = \
def BiasedBitcodeFlags(env):
""" Return clang flags to use biased bitcode and generate native-ABI-compliant
code. Does not imply pre-translation.
if env.Bit('build_x86_32'):
return ['--target=i686-unknown-nacl']
if env.Bit('build_x86_64'):
return ['--target=x86_64-unknown-nacl']
if env.Bit('build_arm'):
return ['--target=armv7-unknown-nacl-gnueabihf', '-mfloat-abi=hard']
if env.Bit('build_mips32'):
return []
raise UserError('No known target bits set')
# Do this before the site_scons/site_tools/ stuff to pass it along.
PNACL_BCLDFLAGS = ARGUMENTS.get('pnacl_bcldflags', '').split(':'))
# The nacl_env is used to build native_client modules
# using a special tool chain which produces platform
# independent binaries
# NOTE: this loads stuff from: site_scons/site_tools/
nacl_env = MakeArchSpecificEnv()
# See comment below about libc++ and libpthread in NONIRT_LIBS.
using_nacl_libcxx = nacl_env.Bit('bitcode') or nacl_env.Bit('nacl_clang')
nacl_env = nacl_env.Clone(
tools = ['naclsdk'],
BUILD_TYPE = 'nacl',
BUILD_TYPE_DESCRIPTION = 'NaCl module build',
ARFLAGS = 'rc',
# ${SOURCE_ROOT} for #include <ppapi/...>
EXTRA_LINKFLAGS = ARGUMENTS.get('nacl_linkflags', '').split(':'),
# always optimize binaries
CCFLAGS = ['-O2',
# This makes sure unwind/backtrace info is available for
# all code locations. Note build/untrusted.gypi uses it too.
] +
CFLAGS = ['-std=gnu99',
CXXFLAGS = ['-std=gnu++98',
# This magic is copied from scons-2.0.1/engine/SCons/
# where this pattern is used for _LIBDIRFLAGS, which produces -L
# switches. Here we are producing a -Wl,-rpath-link,DIR for each
# element of LIBPATH, i.e. for each -LDIR produced.
'__env__, RDirs, TARGET, SOURCE)} $)',
RPATHLINKPREFIX = '-Wl,-rpath-link,',
LIBS = [],
# These are settings for in-tree, non-browser tests to use.
# They use libraries that circumvent the IRT-based implementations
# in the public libraries.
# Note that pthread_private is part of NONIRT_LIBS for clang because
# libc++ depends on libpthread. However we can't just add
# libpthread_private to the link line because those libs get added before
# the standard libs, so the references that come from libc++ itself will
# still get satisfied from libpthread instead of libpthread_private (and
# that code will crash because it requires the IRT). So put libc++ on the
# user link line before libpthread_private to ensure that its references
# to libpthread also get satisfied by libpthread_private.
# TODO(dschuff): Also remove the hack in pnacl-ld and use this for pnacl.
NONIRT_LIBS = (['nacl_sys_private'] +
(['c++','pthread_private'] if using_nacl_libcxx else [])),
PTHREAD_LIBS = ['pthread_private'],
DYNCODE_LIBS = ['nacl_dyncode_private'],
EXCEPTION_LIBS = ['nacl_exception_private'],
LIST_MAPPINGS_LIBS = ['nacl_list_mappings_private'],
RANDOM_LIBS = ['nacl_random_private'],
def UsesAbiNote(env):
"""Return True if using a new-style GCC with .note.NaCl.ABI.* notes.
This means there will always be an RODATA segment, even if just for the note."""
return env.Bit('build_arm') and not env.Bit('bitcode')
def UnderWindowsCoverage(env):
"""Return True if using running on coverage under windows."""
if 'TRUSTED_ENV' not in env:
return False
return env['TRUSTED_ENV'].Bit('coverage_enabled') and env.Bit('host_windows')
def AllowNonStableBitcode(env, allow_sb_translator=False):
""" This modifies the environment to allow features that aren't part
of PNaCl's stable ABI. If tests using these features should be
skipped entirely, this returns False. Otherwise, on success, it
returns True.
if env.Bit('bitcode') and env.Bit('skip_nonstable_bitcode'):
return False
# The PNaCl sandboxed translator (for the most part) only accepts stable
# bitcode, so in most cases we skip building non-stable tests.
# However, there are some limited cases like debug information which
# we support but do not guarantee stability. Tests targeting such cases
# can opt-in to testing w/ allow_sb_translator=True.
if env.Bit('use_sandboxed_translator') and not allow_sb_translator:
return False
# Change environment to skip finalization step.
return True
def AllowInlineAssembly(env):
""" This modifies the environment to allow inline assembly in
untrusted code. If the environment cannot be modified to allow
inline assembly, it returns False. Otherwise, on success, it
returns True.
if env.Bit('bitcode'):
# For each architecture, we only attempt to make our inline
# assembly code work with one untrusted-code toolchain. For x86,
# we target GCC, but not PNaCl/Clang, because the latter's
# assembly support has various quirks that we don't want to have
# to debug. For ARM, we target PNaCl/Clang, because that is the
# only current ARM toolchain. One day, we will have an ARM GCC
# toolchain, and we will no longer need to use inline assembly
# with PNaCl/Clang at all.
# For Non-SFI NaCl we use inline assembly in PNaCl/Clang.
if not (env.Bit('build_arm') or env.Bit('build_mips32')
or env.Bit('nonsfi_nacl')):
return False
# Inline assembly does not work in pexes.
if env.Bit('pnacl_generate_pexe'):
return False
if env.Bit('build_x86_32'):
elif env.Bit('build_x86_64'):
elif env.Bit('build_arm'):
return True
# TODO(mseaborn): Enable this unconditionally once the C code on the
# Chromium side compiles successfully with this warning.
if not enable_chrome:
# This is the address at which a user executable is expected to place its
# data segment in order to be compatible with the integrated runtime (IRT)
# library. This address should not be changed lightly.
irt_compatible_rodata_addr = 0x10000000
# This is the address at which the IRT's own code will be located.
# It must be below irt_compatible_rodata and leave enough space for
# the code segment of the IRT. It should be as close as possible to
# irt_compatible_rodata so as to leave the maximum contiguous area
# available for the dynamic code loading area that falls below it.
# This can be adjusted as necessary for the actual size of the IRT code.
irt_code_addr = irt_compatible_rodata_addr - (6 << 20) # max 6M IRT code
# This is the address at which the IRT's own data will be located. The
# 32-bit sandboxes limit the address space to 1GB; the initial thread's
# stack sits at the top of the address space and extends down for
# NACL_DEFAULT_STACK_MAX (src/trusted/service_runtime/sel_ldr.h) below.
# So this must be below there, and leave enough space for the IRT's own
# data segment. It should be as high as possible so as to leave the
# maximum contiguous area available for the user's data and break below.
# This can be adjusted as necessary for the actual size of the IRT data
# (that is RODATA, rounded up to 64k, plus writable data).
# 1G (address space) - 16M (NACL_DEFAULT_STACK_MAX) - 1MB (IRT rodata+data)