::------------------------------------------------------------------------------------------------------- | |
:: Copyright (C) Microsoft. All rights reserved. | |
:: Copyright (c) 2021 ChakraCore Project Contributors. All rights reserved. | |
:: Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. | |
::------------------------------------------------------------------------------------------------------- | |
:: ============================================================================ | |
:: | |
:: testone.cmd | |
:: | |
:: Runs tests for continuous integration. This script is called from | |
:: the CI build and it runs all tests for x86 and x64, debug and test | |
:: build configs. | |
:: | |
:: Do not use this script to run all tests on your dev box. | |
:: - It will delete all your existing test logs | |
:: - It does not run the various flavors of the tests in parallel (though | |
:: this is not currently possible anyway because rl stages logs in a | |
:: common directory) | |
:: - It does nothing to provide useful output when there are failures, e.g. | |
:: they can be buried under thousands of lines of output from further | |
:: tests run. | |
:: - It cannot be cancelled without risk of polluting your command prompt | |
:: environment with environment variables that will make further calls to | |
:: runtests.cmd behave unexpectedly. | |
:: | |
:: ============================================================================ | |
@echo off | |
setlocal | |
REM check that we have enough parameters | |
if "%1"=="" ( | |
goto :usage | |
) | |
if "%2"=="" ( | |
goto :usage | |
) | |
if "%_ENTRY_SCRIPT_NAME%"=="" ( | |
set _ENTRY_SCRIPT_NAME=%0 | |
) | |
set _RootDir=%~dp0.. | |
set _HadFailures=0 | |
set _error=0 | |
:: ============================================================================ | |
:: Main script | |
:: ============================================================================ | |
:main | |
pushd %_RootDir%\test | |
set _TestDir=%CD% | |
call ci.parsetestargs.cmd %* | |
set _LogDir=%_TestDir%\logs\%_TestArch%_%_TestType% | |
set _TestArgs=%_TestArch%%_TestType% | |
set _BinDir=%_RootDir%\Build\VcBuild%_SpecialBuild%\bin | |
call :doSilent rd /s/q %_LogDir% | |
call :verifyBytcode | |
if "%_HadFailures%" == "0" ( | |
call :runTests %_TestArgs% | |
if "%_ReducedTestRun%" == "1" ( | |
echo -- ci.testone.cmd ^>^> Reduced test run: skipping native tests. | |
) else ( | |
call :runNativeTests %_TestArgs% | |
) | |
) | |
echo. | |
echo -- ci.testone.cmd ^>^> Failure code: %_HadFailures% | |
if "%_HadFailures%" NEQ "0" ( | |
if "%_HadFailures%" == "2" ( | |
echo -- ci.testone.cmd ^>^> Bytecode test failed, bytecode needs to be updated! 1>&2 | |
) else if "%_HadFailures%" == "3" ( | |
echo -- ci.testone.cmd ^>^> Unit tests failed! 1>&2 | |
) else if "%_HadFailures%" == "4" ( | |
echo -- ci.testone.cmd ^>^> Native tests failed! 1>&2 | |
call :summarizeLogs | |
) else ( | |
echo -- ci.testone.cmd ^>^> Unknown failure! 1>&2 | |
) | |
) else ( | |
echo -- ci.testone.cmd ^>^> Tests passed! | |
) | |
popd | |
exit /b %_HadFailures% | |
goto :eof | |
:: ============================================================================ | |
:: Verify that generated bytecode if up to date for one config report if not | |
:: ============================================================================ | |
:verifyBytcode | |
set bytecode_type=--jit | |
if "%_SpecialBuild%" == ".NoJIT" ( | |
set bytecode_type=--noJit | |
) | |
call :do python %_RootDir%\tools\regenByteCode.py --verify %bytecode_type% --%_TestArch% --binary=%_BinDir%\%_TestArch%_%_TestType%\ch.exe | |
if "%_error%" NEQ "0" ( | |
echo -- ci.testone.cmd ^>^> verifying bytecode failed | |
set _HadFailures=2 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Run one test suite against one build config and record if there were errors | |
:: ============================================================================ | |
:runTests | |
if "%_SpecialBuild%" == ".NoJIT" ( | |
set OverideVariant=--variants=disable_jit | |
) | |
call :do python %_TestDir%\runtests.py --%_TestArch% --%_TestType% %OverideVariant% %_ExtraTestArgs% --binary=%_BinDir%\%_TestArch%_%_TestType%\ch.exe | |
if "%_error%" NEQ "0" ( | |
echo -- ci.testone.cmd ^>^> runtests.py failed | |
set _HadFailures=3 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Run jsrt test suite against one build config and record if there were errors | |
:: ============================================================================ | |
:runNativeTests | |
echo -- ci.testone.cmd ^>^> Running native tests... this can take some time | |
if not exist %_LogDir%\ mkdir %_LogDir% | |
set _LogFile=%_LogDir%\nativetests.log | |
call :do %_TestDir%\runnativetests.cmd -%1 -binDir %_BinDir% -d yes > %_LogFile% 2>&1 | |
echo -- ci.testone.cmd ^>^> Running native tests... DONE! | |
if "%_error%" NEQ "0" ( | |
echo -- ci.testone.cmd ^>^> runnativetests.cmd failed; printing %_LogFile% | |
powershell "if (Test-Path %_LogFile%) { Get-Content %_LogFile% }" | |
set _HadFailures=4 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Summarize the logs into a listing of only the failures | |
:: ============================================================================ | |
:summarizeLogs | |
pushd %_LogDir% | |
findstr /sip failed nativetests.log >> summary.log | |
rem Echo to stderr so that VSO includes the output in the build summary | |
type summary.log 1>&2 | |
popd | |
:: ============================================================================ | |
:: Echo a command line before executing it | |
:: ============================================================================ | |
:do | |
echo -- ci.testone.cmd ^>^> :do %* | |
cmd /s /c "%*" | |
set _error=%ERRORLEVEL% | |
goto :eof | |
:: ============================================================================ | |
:: Echo a command line before executing it and redirect the command's output | |
:: to nul | |
:: ============================================================================ | |
:doSilent | |
echo -- ci.testone.cmd ^>^> :doSilent %* ^> nul 2^>^&1 | |
cmd /s /c "%* > nul 2>&1" | |
set _error=%ERRORLEVEL% | |
goto :eof | |
:: ============================================================================ | |
:: Not enough params | |
:: ============================================================================ | |
:usage | |
echo Not enough parameters. Please specify architecture and type. | |
echo Examples: | |
echo. | |
echo %_ENTRY_SCRIPT_NAME% x86 debug | |
echo %_ENTRY_SCRIPT_NAME% x86 test | |
echo. | |
echo %_ENTRY_SCRIPT_NAME% x64 debug | |
echo %_ENTRY_SCRIPT_NAME% x64 test | |
goto :end |