::------------------------------------------------------------------------------------------------------- | |
:: Copyright (C) Microsoft. All rights reserved. | |
:: Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. | |
::------------------------------------------------------------------------------------------------------- | |
:: ============================================================================ | |
:: | |
:: runcitests.cmd | |
:: | |
:: Runs tests for continuous integration. This script is called from the VSO | |
:: build and it runs all tests for x86 and x64, debug and test build configs. | |
:: Logs are copied to build drop. | |
:: | |
:: Do not use this script to run all tests on your dev box. | |
:: - It will delete all your existing test logs | |
:: - It does not run the various flavors of the tests in parallel (though | |
:: this is not currently possible anyway because rl stages logs in a | |
:: common directory) | |
:: - It does nothing to provide useful output when there are failures, e.g. | |
:: they can be buried under thousands of lines of output from further | |
:: tests run. | |
:: - It cannot be cancelled without risk of polluting your command prompt | |
:: environment with environment variables that will make further calls to | |
:: runtests.cmd behave unexpectedly. | |
:: - It will copy the logs to a folder named \testlogs in the directory you | |
:: started this script from. | |
:: | |
:: ============================================================================ | |
@echo off | |
setlocal | |
if "%TF_BUILD_BINARIESDIRECTORY%" == "" ( | |
echo TF_BUILD_BINARIESDIRECTORY is required for this script to work correctly. | |
exit /b 1 | |
) | |
set _RootDir=%~dp0.. | |
set _StagingDir=%TF_BUILD_BINARIESDIRECTORY% | |
REM %TF_BUILD_DROPLOCATION% is not required -- used only for an informational message | |
set _DropRootDir=%TF_BUILD_DROPLOCATION% | |
set _HadFailures=0 | |
:: ============================================================================ | |
:: Main script | |
:: ============================================================================ | |
:main | |
call :parseArgs %* | |
if not "%fShowUsage%" == "" ( | |
call :printUsage | |
goto :eof | |
) | |
call :validateArgs | |
if not "%fShowGetHelp%" == "" ( | |
call :printGetHelp | |
goto :eof | |
) | |
if not "%TF_BUILD%" == "True" ( | |
echo Error: TF_BUILD environment variable is not set to "True". | |
echo This script must be run under a TF Build Agent environment. | |
exit /b 2 | |
) | |
:: Cannot run tests for arm on build machine and release builds | |
:: do not work with ch.exe so no-op those configurations. | |
:: Include _RunAll in the check because if it is specified it | |
:: should trump this early out. | |
if "%_RunAll%%_BuildArch%" == "arm" goto :noTests | |
if "%_RunAll%%_BuildArch%" == "arm64" goto :noTests | |
if "%_RunAll%%_BuildType%" == "release" goto :noTests | |
pushd %_RootDir%\test | |
set _TestDir=%CD% | |
call :doSilent rd /s/q %_TestDir%\logs | |
if not "%_RunAll%" == "" ( | |
call :runTests x86 debug | |
call :runTests x86 test | |
call :runTests x64 debug | |
call :runTests x64 test | |
call :runNativeTests x86 debug | |
call :runNativeTests x86 test | |
call :runNativeTests x64 debug | |
call :runNativeTests x64 test | |
call :summarizeLogs summary.log | |
) else ( | |
call :runTests %_BuildArch% %_BuildType% %_ExtraArgs% | |
call :runNativeTests %_BuildArch% %_BuildType% | |
call :summarizeLogs summary.%_BuildArch%%_BuildType%.log | |
) | |
call :copyLogsToDrop | |
echo. | |
echo -- runcitests.cmd ^>^> Failure code: %_HadFailures% | |
if "%_HadFailures%" NEQ "0" ( | |
if "%_HadFailures%" == "3" ( | |
echo -- runcitests.cmd ^>^> Unit tests failed! 1>&2 | |
) else if "%_HadFailures%" == "4" ( | |
echo -- runcitests.cmd ^>^> Native tests failed! 1>&2 | |
) else ( | |
echo -- runcitests.cmd ^>^> Unknown failure! 1>&2 | |
) | |
) else ( | |
echo -- runcitests.cmd ^>^> Tests passed! | |
) | |
echo -- runcitests.cmd ^>^> Logs at %_DropRootDir%\testlogs | |
popd | |
exit /b %_HadFailures% | |
:noTests | |
echo -- runcitests.cmd ^>^> The tests are not supported on this build configuration. | |
echo -- runcitests.cmd ^>^> No tests were run. This is expected. | |
echo -- runcitests.cmd ^>^> Configuration: %_BuildArch% %_BuildType% | |
exit /b 0 | |
:: ============================================================================ | |
:: Run one test suite against one build config and record if there were errors | |
:: ============================================================================ | |
:runTests | |
:: save the architecture and build values, then obtain the rest of the arguments | |
set arch=%1 | |
set build=%2 | |
shift | |
shift | |
set rest= | |
:rest_loop | |
if "%1"=="" goto after_rest_loop | |
set rest=%rest% %1 | |
shift | |
goto rest_loop | |
:after_rest_loop | |
call :do %_TestDir%\runtests.cmd -%arch%%build% %rest% -quiet -cleanupall -binDir %_StagingDir%\bin | |
if "%_error%" NEQ "0" ( | |
echo -- runcitests.cmd ^>^> runtests.cmd failed | |
set _HadFailures=3 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Run jsrt test suite against one build config and record if there were errors | |
:: ============================================================================ | |
:runNativeTests | |
echo -- runcitests.cmd ^>^> Running native tests... this can take some time | |
if not exist %_LogDir%\ mkdir %_LogDir% | |
set _LogFile=%_TestDir%\logs\%1_%2\nativetests.log | |
call :do %_TestDir%\runnativetests.cmd -%1%2 -d yes > %_LogFile% 2>&1 | |
echo -- runcitests.cmd ^>^> Running native tests... DONE! | |
if "%_error%" NEQ "0" ( | |
echo -- runcitests.cmd ^>^> runnativetests.cmd failed; printing %_LogFile% | |
powershell "if (Test-Path %_LogFile%) { Get-Content %_LogFile% }" | |
set _HadFailures=4 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Copy all result logs to the drop share | |
:: ============================================================================ | |
:copyLogsToDrop | |
:: /S Copy all non-empty dirs | |
:: /Y Do not prompt for overwriting destination files | |
:: /C Continue copying if there are errors | |
:: /I Assume destination is a directory if it does not exist | |
call :do xcopy %_TestDir%\logs %_StagingDir%\testlogs /S /Y /C /I | |
goto :eof | |
:: ============================================================================ | |
:: Summarize the logs into a listing of only the failures | |
:: ============================================================================ | |
:summarizeLogs | |
pushd %_TestDir%\logs | |
findstr /sp failed rl.results.log > %1 | |
findstr /sip failed nativetests.log >> %1 | |
rem Echo to stderr so that VSO includes the output in the build summary | |
type %1 1>&2 | |
popd | |
goto :eof | |
:: ============================================================================ | |
:: Print usage | |
:: ============================================================================ | |
:printUsage | |
echo runcitests.cmd -x86^|-x64^|-arm -debug^|-test^|-release | |
echo. | |
echo Runs tests post-build for automated VSO TFS Builds. | |
echo Depends on TFS Build environment. | |
echo. | |
echo Required switches: | |
echo. | |
echo Specify architecture of build to test: | |
echo. | |
echo -x86 Build arch of binaries is x86 | |
echo -x64 Build arch of binaries is x64 | |
echo -arm Build arch of binaries is ARM | |
echo. | |
echo Specify type of of build to test: | |
echo. | |
echo -debug Build type of binaries is debug | |
echo -test Build type of binaries is test | |
echo -release Build type of binaries is release | |
echo. | |
echo Shorthand combinations can be used, e.g. -x64debug | |
echo. | |
echo Note: No tests are run for ARM or release as they are | |
echo not supported. The switches are provided for tooling | |
echo convenience. | |
goto :eof | |
:: ============================================================================ | |
:: Print how to get help | |
:: ============================================================================ | |
:printGetHelp | |
echo For help use runcitests.cmd -? | |
goto :eof | |
:: ============================================================================ | |
:: Parse the user arguments into environment variables | |
:: ============================================================================ | |
:parseArgs | |
:NextArgument | |
if "%1" == "-?" set fShowUsage=1& goto :ArgOk | |
if "%1" == "/?" set fShowUsage=1& goto :ArgOk | |
if /i "%1" == "-x86" set _BuildArch=x86& goto :ArgOk | |
if /i "%1" == "-x64" set _BuildArch=x64& goto :ArgOk | |
if /i "%1" == "-arm" set _BuildArch=arm& goto :ArgOk | |
if /i "%1" == "-debug" set _BuildType=debug& goto :ArgOk | |
if /i "%1" == "-test" set _BuildType=test& goto :ArgOk | |
if /i "%1" == "-release" set _BuildType=release& goto :ArgOk | |
if /i "%1" == "-x86debug" set _BuildArch=x86&set _BuildType=debug& goto :ArgOk | |
if /i "%1" == "-x64debug" set _BuildArch=x64&set _BuildType=debug& goto :ArgOk | |
if /i "%1" == "-armdebug" set _BuildArch=arm&set _BuildType=debug& goto :ArgOk | |
if /i "%1" == "-x86test" set _BuildArch=x86&set _BuildType=test& goto :ArgOk | |
if /i "%1" == "-x64test" set _BuildArch=x64&set _BuildType=test& goto :ArgOk | |
if /i "%1" == "-armtest" set _BuildArch=arm&set _BuildType=test& goto :ArgOk | |
if /i "%1" == "-x86release" set _BuildArch=x86&set _BuildType=release& goto :ArgOk | |
if /i "%1" == "-x64release" set _BuildArch=x64&set _BuildType=release& goto :ArgOk | |
if /i "%1" == "-armrelease" set _BuildArch=arm&set _BuildType=release& goto :ArgOk | |
if /i "%1" == "-all" set _RunAll=1& goto :ArgOk | |
if not "%1" == "" set _ExtraArgs=%_ExtraArgs% %1& goto :ArgOk | |
goto :eof | |
:ArgOk | |
shift | |
goto :NextArgument | |
:: ============================================================================ | |
:: Validate arguments; if non specified default to -all | |
:: ============================================================================ | |
:validateArgs | |
if "%_BuildArch%" == "" ( | |
set _RunAll=1 | |
) | |
if "%_BuildType%" == "" ( | |
set _RunAll=1 | |
) | |
goto :eof | |
:: ============================================================================ | |
:: Echo a command line before executing it | |
:: ============================================================================ | |
:do | |
echo -- runcitests.cmd ^>^> %* | |
cmd /s /c "%*" | |
set _error=%ERRORLEVEL% | |
goto :eof | |
:: ============================================================================ | |
:: Echo a command line before executing it and redirect the command's output | |
:: to nul | |
:: ============================================================================ | |
:doSilent | |
echo -- runcitests.cmd ^>^> %* ^> nul 2^>^&1 | |
cmd /s /c "%* > nul 2>&1" | |
set _error=%ERRORLEVEL% | |
goto :eof |