blob: 656db8cc6342944aaecb35073b933fccc58698f2 [file] [log] [blame]
#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Run toolchain torture tests and llvm testsuite tests.
# For now, run on linux64, build and run unsandboxed newlib tests
# for all 3 architectures.
# Note: This script builds the toolchain from scratch but does
# not build the translators and hence the translators
# are from an older revision, see comment below.
set -o xtrace
set -o nounset
set -o errexit
readonly PNACL_BUILD="pnacl/build.sh"
readonly TOOLCHAIN_BUILD="toolchain_build/toolchain_build_pnacl.py"
readonly PACKAGES_SCRIPT="buildbot/packages.py"
readonly TEMP_PACKAGES="toolchain_build/out/packages.txt"
readonly UP_DOWN_LOAD="buildbot/file_up_down_load.sh"
readonly TORTURE_TEST="tools/toolchain_tester/torture_test.py"
readonly LLVM_TEST="pnacl/scripts/llvm-test.py"
readonly CLANG_UPDATE="../tools/clang/scripts/update.py"
readonly INSTALL_SUBDIR="toolchain/linux_x86/pnacl_newlib"
readonly INSTALL_ABSPATH=$(pwd)/${INSTALL_SUBDIR}
readonly LIBRARY_ABSPATH=${INSTALL_ABSPATH}/lib
# build.sh, llvm test suite and torture tests all use this value
export PNACL_CONCURRENCY=${PNACL_CONCURRENCY:-4}
# Change the toolchain build script (PNACL_BUILD) behavior slightly
# wrt to error logging and mecurial retry delays.
# TODO(robertm): if this special casing is still needed,
# make this into separate vars
export PNACL_BUILDBOT=true
# Make the toolchain build script (PNACL_BUILD) more verbose.
# This will also prevent bot timeouts which otherwise gets triggered
# by long periods without console output.
export PNACL_VERBOSE=true
EXIT_STATUS=0
clobber() {
echo @@@BUILD_STEP clobber@@@
rm -rf scons-out
# Don't clobber pnacl_translator; these bots currently don't build
# it, but they use the DEPSed-in version.
rm -rf toolchain/linux_x86/pnacl_newlib* \
toolchain/mac_x86/pnacl_newlib* \
toolchain/win_x86/pnacl_newlib*
}
handle-error() {
echo "@@@STEP_FAILURE@@@"
EXIT_STATUS=1
}
ignore-error() {
echo "@== IGNORING AN ERROR ==@"
}
#### Support for running arm sbtc tests on this bot, since we have
# less coverage on the main waterfall now:
# http://code.google.com/p/nativeclient/issues/detail?id=2581
readonly SCONS_COMMON="./scons --verbose bitcode=1 -j${PNACL_CONCURRENCY}"
readonly SCONS_COMMON_SLOW="./scons --verbose bitcode=1 -j2"
build-run-prerequisites() {
local platform=$1
${SCONS_COMMON} platform=${platform} sel_ldr irt_core elf_loader
}
scons-tests-translator() {
local platform=$1
local flags="--mode=opt-host,nacl use_sandboxed_translator=1 \
platform=${platform} -k skip_trusted_tests=1"
local targets="small_tests medium_tests large_tests"
# ROUND 1: regular builds
# generate pexes with full parallelism
echo "@@@BUILD_STEP scons-sb-trans-pexe [${platform}] [${targets}]@@@"
${SCONS_COMMON} ${flags} ${targets} \
translate_in_build_step=0 do_not_run_tests=1 || handle-error
# translate pexes
echo "@@@BUILD_STEP scons-sb-trans-trans [${platform}] [${targets}]@@@"
if [[ ${platform} = arm ]] ; then
# For ARM we use less parallelism to avoid mysterious QEMU crashes.
# We also force a timeout for translation only.
export QEMU_PREFIX_HOOK="timeout 120"
# Run sb translation twice in case we failed to translate some of the
# pexes. If there was an error in the first run this shouldn't
# trigger a buildbot error. Only the second run can make the bot red.
${SCONS_COMMON_SLOW} ${flags} ${targets} \
do_not_run_tests=1 || ignore-error
${SCONS_COMMON_SLOW} ${flags} ${targets} \
do_not_run_tests=1 || handle-error
# Do not use the prefix hook for running actual tests as
# it will break some of them due to exit code sign inversion.
unset QEMU_PREFIX_HOOK
else
${SCONS_COMMON} ${flags} ${targets} \
do_not_run_tests=1 || handle-error
fi
# finally run the tests
echo "@@@BUILD_STEP scons-sb-trans-run [${platform}] [${targets}]@@@"
${SCONS_COMMON_SLOW} ${flags} ${targets} || handle-error
# ROUND 2: builds with "fast translation"
flags="${flags} translate_fast=1"
echo "@@@BUILD_STEP scons-sb-trans-pexe [fast] [${platform}] [${targets}]@@@"
${SCONS_COMMON} ${flags} ${targets} \
translate_in_build_step=0 do_not_run_tests=1 || handle-error
echo "@@@BUILD_STEP scons-sb-trans-trans [fast] [${platform}] [${targets}]@@@"
if [[ ${platform} = arm ]] ; then
# For ARM we use less parallelism to avoid mysterious QEMU crashes.
# We also force a timeout for translation only.
export QEMU_PREFIX_HOOK="timeout 120"
# Run sb translation twice in case we failed to translate some of the
# pexes. If there was an error in the first run this shouldn't
# trigger a buildbot error. Only the second run can make the bot red.
${SCONS_COMMON_SLOW} ${flags} ${targets} \
do_not_run_tests=1 || ignore-error
${SCONS_COMMON_SLOW} ${flags} ${targets} \
do_not_run_tests=1 || handle-error
# Do not use the prefix hook for running actual tests as
# it will break some of them due to exit code sign inversion.
unset QEMU_PREFIX_HOOK
else
${SCONS_COMMON} ${flags} ${targets} \
do_not_run_tests=1 || handle-error
fi
echo "@@@BUILD_STEP scons-sb-trans-run [fast] [${platform}] [${targets}]@@@"
${SCONS_COMMON_SLOW} ${flags} ${targets} || handle-error
}
scons-tests-x86-64-zero-based-sandbox() {
echo "@@@BUILD_STEP hello_world (x86-64 zero-based sandbox)@@@"
local flags="--mode=opt-host,nacl platform=x86-64 \
x86_64_zero_based_sandbox=1"
${SCONS_COMMON} ${flags} "run_hello_world_test"
}
tc-test-bot() {
local archset="$1"
clobber
# Update Clang
python ${CLANG_UPDATE}
# Only build MIPS stuff on mips bots
if [[ ${archset} == "mips" ]]; then
export PNACL_BUILD_MIPS=true
# Don't run any of the tests yet
echo "MIPS bot: Only running build, and not tests"
archset=
fi
# Build the un-sandboxed toolchain. The build script outputs its own buildbot
# annotations.
${TOOLCHAIN_BUILD} --verbose --sync --clobber --testsuite-sync \
--packages-file ${TEMP_PACKAGES}
# Extract the built packages using the packages script.
python ${PACKAGES_SCRIPT} extract --overlay-packages --skip-missing \
--packages ${TEMP_PACKAGES}
# Linking the tests require additional sdk libraries like libnacl.
# Do this once and for all early instead of attempting to do it within
# each test step and having some late test steps rely on early test
# steps building the prerequisites -- sometimes the early test steps
# get skipped.
echo "@@@BUILD_STEP install sdk libraries @@@"
${PNACL_BUILD} sdk
for arch in ${archset}; do
# Similarly, build the run prerequisites (sel_ldr and the irt) early.
echo "@@@BUILD_STEP build run prerequisites [${arch}]@@@"
build-run-prerequisites ${arch}
done
# Run the torture tests and compiler_rt tests.
for arch in ${archset}; do
echo "@@@BUILD_STEP pnacl bitcode compiler_rt tests@@@"
export PNACL_RUN_ARCH=${arch}
make -C toolchain_build/src/compiler-rt \
-f lib/builtins/Makefile-pnacl-bitcode \
TCROOT=${INSTALL_ABSPATH} nacltest-pnacl || handle-error
# The CC arg is just a dummy to keep the make scripts from complaining
# if clang is not found in PATH
echo "@@@BUILD_STEP clang compiler_rt tests $arch @@@"
make -C toolchain_build/src/compiler-rt TCROOT=${INSTALL_ABSPATH} \
CC=gcc nacltest-${arch} || handle-error
echo "@@@BUILD_STEP torture_tests_clang $arch @@@"
${TORTURE_TEST} clang ${arch} --verbose \
--concurrency=${PNACL_CONCURRENCY} || handle-error
if [[ "${arch}" == "x86-32" ]]; then
# Torture tests on x86-32 are covered by tc-tests-all in
# buildbot_pnacl.sh.
continue
fi
echo "@@@BUILD_STEP torture_tests_pnacl $arch @@@"
${TORTURE_TEST} pnacl ${arch} --verbose \
--concurrency=${PNACL_CONCURRENCY} || handle-error
done
local optset
optset[1]="--opt O3f --opt O2b"
for arch in ${archset}; do
# Run all appropriate frontend/backend optimization combinations. For now,
# this means running 2 combinations for x86 (plus one more for Subzero on
# x86-32) since each takes about 20 minutes on the bots, and making a single
# run elsewhere since e.g. arm takes about 75 minutes. In a perfect world,
# all 4 combinations would be run, plus more for Subzero.
if [[ ${archset} =~ x86 ]]; then
optset[2]="--opt O3f --opt O0b"
if [[ ${archset} == x86-32 || ${archset} == x86-64 ]]; then
# Run a Subzero -O2 test set on x86-32.
optset[3]="--opt O3f --opt O2b_sz"
fi
fi
for opt in "${optset[@]}"; do
echo "@@@BUILD_STEP llvm-test-suite ${arch} ${opt} @@@"
python ${LLVM_TEST} --testsuite-clean
LD_LIBRARY_PATH=${LIBRARY_ABSPATH} python ${LLVM_TEST} \
--testsuite-configure --testsuite-run --testsuite-report \
--arch ${arch} ${opt} -v -c || handle-error
done
echo "@@@BUILD_STEP libcxx-test ${arch} @@@"
LD_LIBRARY_PATH=${LIBRARY_ABSPATH} python ${LLVM_TEST} \
--libcxx-test --arch ${arch} -c || handle-error
# Note: we do not build the sandboxed translator on this bot
# because this would add another 20min to the build time.
# The upshot of this is that we are using the sandboxed
# toolchain which is currently deps'ed in.
# There is a small upside here: we will notice that bitcode has
# changed in a way that is incompatible with older translators.
# TODO(pnacl-team): rethink this.
# Note: the tests which use sandboxed translation are at the end,
# because they can sometimes hang on arm, causing buildbot to kill the
# script without running any more tests.
scons-tests-translator ${arch}
if [[ ${arch} = x86-64 ]] ; then
scons-tests-x86-64-zero-based-sandbox
fi
done
exit $EXIT_STATUS
}
if [ $# = 0 ]; then
# NOTE: this is used for manual testing only
tc-test-bot "x86-64 x86-32 arm"
else
"$@"
fi