blob: d3c39907c7bea8f9dfa28a98a124031abbb68430 [file] [log] [blame]
#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
#@ PNaCl Test Helper
#@-------------------------------------------------------------------
#@
#@ SCons Test Usage:
#@
#@ test.sh test-<arch>[-sbtc] [extra_arguments_to_scons]
#@
#@ Runs the SCons tests with selected arch (optionally with sandboxed tc)
#@ Valid arches:
#@ x86-32
#@ x86-64
#@ arm
#@
#@ Examples:
#@ test.sh test-x86-32
#@ test.sh test-arm-sbtc
#@
#@ The env variables: PNACL_CONCURRENCY, PNACL_BUILDBOT, PNACL_DEBUG
#@ control behavior of this script
#@
#@
######################################################################
# Config
######################################################################
set -o nounset
set -o errexit
# The script is located in "pnacl/".
# Set pwd to native_client/
cd "$(dirname "$0")"/..
if [[ $(basename "$(pwd)") != "native_client" ]] ; then
echo "ERROR: cannot find native_client/ directory"
exit -1
fi
readonly NACL_ROOT="$(pwd)"
readonly DRYRUN=${DRYRUN:-false}
source pnacl/scripts/common-tools.sh
SetScriptPath "${NACL_ROOT}/pnacl/test.sh"
SetLogDirectory "${NACL_ROOT}/toolchain/test-log"
# For different levels of make parallelism change this in your env
readonly PNACL_CONCURRENCY=${PNACL_CONCURRENCY:-8}
# This needs to be kept in sync with the var of the same name in build.sh
readonly TC_BUILD_LLVM="$(pwd)/pnacl/build/llvm_${HOST_ARCH}"
######################################################################
######################################################################
#
# < TESTING >
#
######################################################################
######################################################################
# TODO(robertm): figure out what to do about concurrency in debug mode.
# Perhaps it is fine just tweaking that via PNACL_CONCURRENCY.
if ${PNACL_DEBUG} || ${PNACL_BUILDBOT}; then
readonly SCONS_ARGS=(MODE=nacl,opt-host
bitcode=1
skip_trusted_tests=1
--verbose
sysinfo=0
-j${PNACL_CONCURRENCY})
else
readonly SCONS_ARGS=(MODE=nacl,opt-host
bitcode=1
skip_trusted_tests=1
naclsdk_validate=0
sysinfo=0
-j${PNACL_CONCURRENCY})
fi
#@ show-tests - see what tests can be run
show-tests() {
StepBanner "SHOWING TESTS"
cat $(find tests -name nacl.scons) | grep -o 'run_[A-Za-z_-]*' | sort | uniq
}
Run() {
echo "$@"
if ! ${DRYRUN}; then
"$@"
fi
}
RunScons() {
local arch="$1"
shift 1
Run ./scons "${SCONS_ARGS[@]}" platform=${arch} "$@"
}
# Returns true if the arguments specify a test
# or target name to SCons.
has-target-name() {
while [ $# -gt 0 ]; do
# Skip arguments of the form -foo, --foo, or foo=bar.
if [[ "$1" =~ ^-.* ]] || [[ "$1" =~ = ]]; then
shift 1
continue
fi
return 0
done
return 1
}
scons-clean () {
local arch=$1
local mode=$2
local frontend=clang
# Clear both the pexe and the nonpexe scons-out directory, since we run
# a mix of both tests.
if [ "${mode}" == "native" ] ; then
Run rm -rf scons-out/nacl-${arch}-pnacl-${frontend}
Run rm -rf scons-out/nacl-${arch}-${frontend}
Run rm -rf scons-out/nacl-${arch}-pnacl-pexe-${frontend}
else
Run rm -rf scons-out/nacl-${arch}-pnacl-${mode}-${frontend}
Run rm -rf scons-out/nacl-${arch}-pnacl-${mode}-pexe-${frontend}
fi
}
build-sbtc-prerequisites() {
local arch=$1
# Sandboxed translators currently only require irt_core since they do not
# use PPAPI.
RunScons ${arch} sel_ldr irt_core elf_loader
}
#+ Run scons test under a certain configuration
#+ scons-tests <arch> <mode={sbtc|}> [optional list of test names]
#+ If no optional tests are listed, we will build all the tests then
#+ run the "smoke_tests" test suite.
scons-tests () {
local arch="$1"
local mode="$2"
shift 2
scons-clean ${arch} ${mode}
local modeflags=""
if [ ${mode} == "sbtc" ]; then
build-sbtc-prerequisites "${arch}"
modeflags="use_sandboxed_translator=1"
fi
if has-target-name "$@" ; then
# By default this uses pexe mode (except where not supported) but this
# can be overridden
RunScons ${arch} ${modeflags} "$@"
else
# For now, mostly just duplicate mode-buildbot-x86 in buildbot_pnacl.sh
# (but don't bother separating build/run for now) until we
# converge on exactly what we want
RunScons ${arch} ${modeflags} "$@" smoke_tests
# None of the other tests make sense with the sandboxed translator.
if [ ${mode} == "sbtc" ]; then
return 0
fi
# nacl-clang tests
RunScons ${arch} ${modeflags} bitcode=0 nacl_clang=1 "$@" smoke_tests
# nonpexe tests
RunScons ${arch} ${modeflags} pnacl_generate_pexe=0 "$@" nonpexe_tests
if [ ${arch} != "x86-64" ]; then
RunScons ${arch} ${modeflags} nonsfi_nacl=1 "$@" nonsfi_tests
RunScons ${arch} ${modeflags} nonsfi_nacl=1 pnacl_generate_pexe=0 \
"$@" nonsfi_tests
fi
fi
}
test-driver() {
${NACL_ROOT}/pnacl/driver/tests/driver_tests.py --platform="$1"
}
test-arm() { test-driver arm && scons-tests arm native "$@" ; }
test-x86-32() { test-driver x86-32 && scons-tests x86-32 native "$@" ; }
test-x86-64() { test-driver x86-64 && scons-tests x86-64 native "$@" ; }
test-arm-sbtc() { scons-tests arm sbtc "$@" ; }
test-x86-32-sbtc() { scons-tests x86-32 sbtc "$@" ; }
test-x86-64-sbtc() { scons-tests x86-64 sbtc "$@" ; }
#@
#@ test-spec <official-spec-dir> <setup> [ref|train] [<benchmarks>]*
#@ - run spec tests
test-spec() {
if [[ $# -lt 2 ]]; then
echo "not enough arguments for test-spec"
exit 1
fi;
official=$(GetAbsolutePath $1)
setup=$2
shift 2
spushd tests/spec2k
./run_all.sh BuildPrerequisitesSetupBased ${setup}
./run_all.sh CleanBenchmarks "$@"
./run_all.sh PopulateFromSpecHarness ${official} "$@"
./run_all.sh BuildAndRunBenchmarks ${setup} "$@"
spopd
}
#+ CollectTimingInfo <directory> <timing_result_file> <tagtype...>
#+ CD's into the directory in a subshell and collects all the
#+ relevant timed run info
#+ tagtype just gets printed out.
CollectTimingInfo() {
wd=$1
result_file=$2
setup=$3
(cd ${wd};
mkdir -p $(dirname ${result_file})
echo "##################################################" >>${result_file}
date +"# Completed at %F %H:%M:%S %A ${result_file}" >> ${result_file}
echo "# " ${wd}
echo "#" $(uname -a) >> ${result_file}
echo "# SETUP: ${setup}" >>${result_file}
echo "##################################################" >>${result_file}
echo "# COMPILE " >> ${result_file}
for ff in $(find . -name "*.compile_time"); do
cat ${ff} >> ${result_file}
done
echo "# RUN " >> ${result_file}
for ff in $(find . -name "*.run_time"); do
cat ${ff} >> ${result_file}
done
cat ${result_file}
)
}
#@
#@ timed-test-spec <result-file> <official-spec-dir> <setup> ... - run spec and
#@ measure time / size data. Data is emitted to stdout, but also collected
#@ in <result-file>. <result-file> is not cleared across runs (but temp files
#@ are cleared on each run).
#@ Note that the VERIFY variable effects the timing!
timed-test-spec() {
if ${BUILD_PLATFORM_MAC} ; then
echo "Timed-test-spec is not currently supported on MacOS"
echo "Namely, /usr/bin/time -f is not supported."
exit -1
fi
if [ "$#" -lt "3" ]; then
echo "timed-test-spec {result-file} {spec2krefdir} {setupfunc}" \
"[ref|train] [benchmark]*"
exit 1
fi
result_file=$1
official=$(GetAbsolutePath $2)
setup=$3
shift 3
spushd tests/spec2k
./run_all.sh BuildPrerequisitesSetupBased ${setup}
./run_all.sh CleanBenchmarks "$@"
./run_all.sh PopulateFromSpecHarness ${official} "$@"
./run_all.sh TimedBuildAndRunBenchmarks ${setup} "$@"
CollectTimingInfo $(pwd) ${result_file} ${setup}
spopd
}
#@ help - Usage information.
help() {
Usage
}
#@ help-full - Usage information including internal functions.
help-full() {
Usage2
}
######################################################################
######################################################################
#
# < MAIN >
#
######################################################################
######################################################################
[ $# = 0 ] && set -- help # Avoid reference to undefined $1.
if [ "$(type -t $1)" != "function" ]; then
#Usage
echo "ERROR: unknown function '$1'." >&2
echo "For help, try:"
echo " $0 help"
exit 1
fi
"$@"