| #!/usr/bin/env python3 |
| # Copyright 2011 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Implements a simple "negative compile" test for C++ on linux. |
| |
| Sometimes a C++ API needs to ensure that various usages cannot compile. To |
| enable unittesting of these assertions, we use this python script to |
| invoke the compiler on a source file and assert that compilation fails. |
| |
| For more info, see: |
| http://dev.chromium.org/developers/testing/no-compile-tests |
| """ |
| |
| import argparse |
| import ast |
| import concurrent.futures |
| import functools |
| import io |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| |
| from typing import Any |
| from typing import IO |
| from typing import Optional |
| from typing import Sequence |
| from typing import Set |
| from typing import Tuple |
| from typing import TypedDict |
| |
| sys.path.append( |
| os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'build')) |
| import action_helpers |
| |
| # Matches lines that start with #if and have the substring TEST in the |
| # conditional. Also extracts the comment. This allows us to search for |
| # lines like the following: |
| # |
| # #ifdef NCTEST_NAME_OF_TEST // [r'expected output'] |
| # #if defined(NCTEST_NAME_OF_TEST) // [r'expected output'] |
| # #if NCTEST_NAME_OF_TEST // [r'expected output'] |
| # #elif NCTEST_NAME_OF_TEST // [r'expected output'] |
| # #elif DISABLED_NCTEST_NAME_OF_TEST // [r'expected output'] |
| # |
| # inside the unittest file. |
| NCTEST_CONFIG_RE = re.compile(r'^#(?:el)?if.*\s+(\S*NCTEST\S*)\s*(//.*)?') |
| |
| # Matches and removes the defined() preprocesor predicate. This is useful |
| # for test cases that use the preprocessor if-statement form: |
| # |
| # #if defined(NCTEST_NAME_OF_TEST) |
| # |
| # Should be used to post-process the results found by NCTEST_CONFIG_RE. |
| STRIP_DEFINED_RE = re.compile(r'defined\((.*)\)') |
| |
| # Used to grab the expectation from comment at the end of an #ifdef. See |
| # NCTEST_CONFIG_RE's comment for examples of what the format should look like. |
| # |
| # The extracted substring should be a python array of regular expressions. |
| EXTRACT_EXPECTATION_RE = re.compile(r'//\s*(\[.*\])') |
| |
| # The header for the result file so that it can be compiled. |
| RESULT_FILE_HEADER = """ |
| // This file is generated by the no compile test from: |
| // %s |
| |
| #include "base/logging.h" |
| #include "testing/gtest/include/gtest/gtest.h" |
| |
| """ |
| |
| # The log message on a test completion. |
| LOG_TEMPLATE = """ |
| TEST(%s, %s) took %f secs. Started at %f, ended at %f. |
| """ |
| |
| # The GUnit test function to output for a successful or disabled test. |
| GUNIT_TEMPLATE = """ |
| TEST(%s, %s) { } |
| """ |
| |
| # How long a nocompile test should be able to run for before timing out. |
| NCTEST_TERMINATE_TIMEOUT_SEC = 120 |
| |
| |
| class TestResult(TypedDict): |
| """Represents the result of one nocompile test. |
| |
| Attributes: |
| cmdline: The executed command line. |
| stdout: A temporary file object containing stdout. |
| name: The name of the test. |
| suite_name: The suite name to use when generating the gunit test result. |
| started_at: A timestamp in seconds since the epoch for when this test was |
| started. |
| aborted_at: A timestamp in seconds since the epoch for when this test was |
| aborted. If the test completed successfully, this value is 0. |
| finished_at: A timestamp in seconds since the epoch for when this test was |
| successfully complete. If the test is aborted, this value is 0. |
| expectations: A dictionary with the test expectations. See |
| ParseExpectation() for the structure. |
| return_code: The return code of the test process, if not aborted. |
| """ |
| cmdline: str |
| stdout: IO[str] |
| stderr: IO[str] |
| name: str |
| suite_name: str |
| started_at: float |
| aborted_at: float |
| finished_at: float |
| expectations: Sequence[re.Pattern] |
| returncode: int |
| |
| |
| def ValidateInput(compiler, parallelism, sourcefile_path, cflags, |
| resultfile_path): |
| """Make sure the arguments being passed in are sane.""" |
| assert os.path.isfile(compiler) |
| assert parallelism >= 1 |
| assert type(sourcefile_path) is str |
| assert type(cflags) is list |
| for flag in cflags: |
| assert type(flag) is str |
| assert type(resultfile_path) is str |
| |
| |
| def ParseExpectation(expectation_string) -> Sequence[re.Pattern]: |
| """Extracts expectation definition from the trailing comment on the ifdef. |
| |
| See the comment on NCTEST_CONFIG_RE for examples of the format we are parsing. |
| |
| Args: |
| expectation_string: A string like "// [r'some_regex']" |
| |
| Returns: |
| A list of compiled regular expressions indicating all possible valid |
| compiler outputs. If the list is empty, all outputs are considered valid. |
| """ |
| assert expectation_string is not None |
| |
| match = EXTRACT_EXPECTATION_RE.match(expectation_string) |
| assert match |
| |
| raw_expectation = ast.literal_eval(match.group(1)) |
| assert type(raw_expectation) is list |
| |
| expectation = [] |
| for regex_str in raw_expectation: |
| assert type(regex_str) is str |
| expectation.append(re.compile(regex_str)) |
| return expectation |
| |
| |
| def ExtractTestConfigs(sourcefile_path, suite_name, resultfile, resultlog): |
| """Parses the source file for test configurations. |
| |
| Each no-compile test in the file is separated by an ifdef macro. We scan |
| the source file with the NCTEST_CONFIG_RE to find all ifdefs that look like |
| they demark one no-compile test and try to extract the test configuration |
| from that. |
| |
| Args: |
| sourcefile_path: The path to the source file. |
| suite_name: The name of the test suite. |
| resultfile: File object for .cc file that results are written to. |
| resultlog: File object for the log file. |
| |
| Returns: |
| A list of test configurations, excluding tests prefixed with DISABLED_. Each |
| test configuration is a dictionary of the form: |
| |
| { name: 'NCTEST_NAME' |
| suite_name: 'SOURCE_FILE_NAME' |
| expectations: [re.Pattern, re.Pattern] } |
| |
| The |suite_name| is used to generate a pretty gtest output on successful |
| completion of the no compile test. |
| |
| The compiled regexps in |expectations| define the valid outputs of the |
| compiler. If any one of the listed patterns matches either the stderr or |
| stdout from the compilation, and the compilation failed, then the test is |
| considered to have succeeded. If the list is empty, than we ignore the |
| compiler output and just check for failed compilation. If |expectations| |
| is actually None, then this specifies a compiler sanity check test, which |
| should expect a SUCCESSFUL compilation. |
| """ |
| with open(sourcefile_path, 'r', encoding='utf-8') as sourcefile: |
| # Start with a compiler smoke test. This is important to show that compiler |
| # flags and configuration are not just wrong. Otherwise, having a |
| # misconfigured compiler, or an error in the shared portions of the .nc file |
| # would cause all tests to erroneously pass. |
| test_configs = [{ |
| 'name': 'NCTEST_SMOKE', |
| 'suite_name': suite_name, |
| 'expectations': None, |
| }] |
| |
| for line in sourcefile: |
| match_result = NCTEST_CONFIG_RE.match(line) |
| if not match_result: |
| continue |
| |
| groups = match_result.groups() |
| |
| # Grab the name and remove the defined() predicate if there is one. |
| name = groups[0] |
| strip_result = STRIP_DEFINED_RE.match(name) |
| if strip_result: |
| name = strip_result.group(1) |
| |
| config = { |
| 'name': name, |
| 'suite_name': suite_name, |
| 'expectations': ParseExpectation(groups[1]) |
| } |
| |
| if config['name'].startswith('DISABLED_'): |
| PassTest(resultfile, resultlog, config) |
| continue |
| |
| test_configs.append(config) |
| return test_configs |
| |
| |
| def RunTest(compiler, tempfile_dir, cflags, config) -> TestResult: |
| """Runs one negative compile test. |
| |
| Args: |
| compiler: The path to the compiler. |
| tempfile_dir: A directory to store temporary data from tests. |
| cflags: An array of strings with all the CFLAGS to give to gcc. |
| config: A dictionary describing the test. See ExtractTestConfigs |
| for a description of the config format. |
| |
| Returns: |
| A TestResult containing all the information about the started test. |
| """ |
| cmdline = [compiler] |
| cmdline.extend(cflags) |
| name = config['name'] |
| expectations = config['expectations'] |
| if expectations is not None: |
| cmdline.append('-D%s' % name) |
| test_stdout = tempfile.TemporaryFile(dir=tempfile_dir, |
| mode='w+', |
| encoding='utf-8') |
| test_stderr = tempfile.TemporaryFile(dir=tempfile_dir, |
| mode='w+', |
| encoding='utf-8') |
| |
| # Note: this could use pipes, but having temp files might make for somewhat |
| # better debuggability. |
| try: |
| started_at = time.time() |
| returncode = subprocess.run(cmdline, |
| stdout=test_stdout, |
| stderr=test_stderr, |
| timeout=NCTEST_TERMINATE_TIMEOUT_SEC).returncode |
| aborted_at = 0 |
| finished_at = time.time() |
| except subprocess.CalledProcessError: |
| returncode = -1 |
| aborted_at = time.time() |
| finished_at = 0 |
| return TestResult(cmdline=' '.join(cmdline), |
| stdout=test_stdout, |
| stderr=test_stderr, |
| name=name, |
| suite_name=config['suite_name'], |
| started_at=started_at, |
| aborted_at=aborted_at, |
| finished_at=finished_at, |
| expectations=expectations, |
| returncode=returncode) |
| |
| |
| def PassTest(resultfile, resultlog, test): |
| """Logs the result of a test run with RunTest(), or a disabled test |
| configuration. |
| |
| Args: |
| resultfile: File object for .cc file that results are written to. |
| resultlog: File object for the log file. |
| test: An instance of the dictionary returned by RunTest(), a |
| configuration from ExtractTestConfigs(). |
| """ |
| resultfile.write(GUNIT_TEMPLATE % (test['suite_name'], test['name'])) |
| |
| # The 'started_at' key is only added if a test has been started. |
| if 'started_at' in test: |
| resultlog.write( |
| LOG_TEMPLATE % |
| (test['suite_name'], test['name'], test['finished_at'] - |
| test['started_at'], test['started_at'], test['finished_at'])) |
| |
| |
| def FailTest(resultfile, test, error, stdout=None, stderr=None): |
| """Logs the result of a test run with by RunTest() |
| |
| Args: |
| resultfile: File object for .cc file that results are written to. |
| test: An instance of the dictionary returned by StartTest() |
| error: The printable reason for the failure. |
| stdout: The test's output to stdout. |
| stderr: The test's output to stderr. |
| """ |
| resultfile.write('\n') |
| resultfile.write('#error %s Failed: %s\n' % (test['name'], error)) |
| resultfile.write('#error compile line: %s\n' % test['cmdline']) |
| if stdout and len(stdout) != 0: |
| resultfile.write('#error %s stdout:\n' % test['name']) |
| for line in stdout.split('\n'): |
| resultfile.write('#error " %s:"\n' % line) |
| |
| if stderr and len(stderr) != 0: |
| resultfile.write('#error %s stderr:"\n' % test['name']) |
| for line in stderr.split('\n'): |
| resultfile.write('#error " %s"\n' % line) |
| |
| |
| def WriteStats(resultlog, suite_name, timings): |
| """Logs the peformance timings for each stage of the script. |
| |
| Args: |
| resultlog: File object for the log file. |
| suite_name: The name of the GUnit suite this test belongs to. |
| timings: Dictionary with timestamps for each stage of the script run. |
| """ |
| stats_template = """ |
| TEST(%s): Started %f, Ended %f, Total %fs, Extract %fs, Compile %fs, Process %fs |
| """ |
| total_secs = timings['results_processed'] - timings['started'] |
| extract_secs = timings['extract_done'] - timings['started'] |
| compile_secs = timings['compile_done'] - timings['extract_done'] |
| process_secs = timings['results_processed'] - timings['compile_done'] |
| resultlog.write(stats_template % |
| (suite_name, timings['started'], timings['results_processed'], |
| total_secs, extract_secs, compile_secs, process_secs)) |
| |
| |
| def ExtractTestOutputAndCleanup(test: TestResult) -> Tuple[str, str]: |
| """Test output is in temp files. Read those and delete them. |
| Returns: A tuple (stderr, stdout). |
| """ |
| def ReadStreamAndClose(stream: IO[str]) -> str: |
| with stream: |
| stream.seek(0) |
| return stream.read() |
| |
| return (ReadStreamAndClose(test['stdout']), |
| ReadStreamAndClose(test['stderr'])) |
| |
| |
| def ProcessTestResult(sourcefile_path: str, |
| resultfile: IO[str], |
| resultlog: IO[str], |
| test: TestResult, |
| includes: Optional[Set[str]] = None) -> None: |
| """Interprets and logs the result of a test run by RunTest() |
| |
| Args: |
| sourcefile_path: Path to the source .cc file derived from the .nc file. |
| resultfile: File object for .cc file that results are written to. |
| resultlog: File object for the log file. |
| test: The dictionary from RunTest() to process. |
| includes: Must either be a set or None. If a set, the driver will scrape |
| stdout for the /showIncludes format and insert any headers found |
| into `includes`. |
| """ |
| (stdout, stderr) = ExtractTestOutputAndCleanup(test) |
| |
| if includes is not None: |
| # /showIncludes format: |
| # Note: including file: third_party/libc++/src/include/stdio.h |
| # Note: including file: third_party/libc++/src/include/__config |
| # Note: including file: buildtools/third_party/libc++/__config_site |
| # Note: including file: third_party/libc++/src/include/stdint.h |
| INCLUDE_PREFIX = 'Note: including file: ' |
| includes.update( |
| map( |
| lambda x: os.path.relpath(x[len(INCLUDE_PREFIX):].strip()), |
| filter( |
| lambda x: x.startswith(INCLUDE_PREFIX), |
| stdout.splitlines(), |
| ), |
| )) |
| |
| if test['aborted_at'] != 0: |
| FailTest( |
| resultfile, test, "Compile timed out. Started %f ended %f." % |
| (test['started_at'], test['aborted_at'])) |
| return |
| |
| if test['returncode'] == 0: |
| # Handle failure due to successful compile. |
| FailTest(resultfile, test, 'Unexpected successful compilation.', stdout, |
| stderr) |
| return |
| else: |
| # Check the output has the right expectations. If there are no |
| # expectations, then we just consider the output "matched" by default. |
| if len(test['expectations']) == 0: |
| PassTest(resultfile, resultlog, test) |
| return |
| |
| # Otherwise test against all expectations. |
| for regexp in test['expectations']: |
| if (regexp.search(stdout) is not None |
| or regexp.search(stderr) is not None): |
| PassTest(resultfile, resultlog, test) |
| return |
| expectation_str = ', '.join( |
| ["r'%s'" % regexp.pattern for regexp in test['expectations']]) |
| FailTest(resultfile, test, |
| 'Expectations [%s] did not match output.' % expectation_str, |
| stdout, stderr) |
| return |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(prog=sys.argv[0]) |
| parser.add_argument('compiler') |
| parser.add_argument('parallelism', type=int) |
| parser.add_argument('sourcefile') |
| parser.add_argument('resultfile') |
| parser.add_argument('--depfile', default='') |
| parser.add_argument('compiler_options', nargs=argparse.REMAINDER) |
| |
| # Force us into the "C" locale so the compiler doesn't localize its output. |
| # In particular, this stops gcc from using smart quotes when in english UTF-8 |
| # locales. This makes the expectation writing much easier. |
| os.environ['LC_ALL'] = 'C' |
| |
| args = parser.parse_args() |
| compiler = args.compiler |
| parallelism = args.parallelism |
| sourcefile_path = args.sourcefile |
| resultfile_path = args.resultfile |
| cflags = args.compiler_options |
| |
| timings = {'started': time.time()} |
| |
| ValidateInput(compiler, parallelism, sourcefile_path, cflags, resultfile_path) |
| |
| # Convert filename from underscores to CamelCase. |
| words = os.path.splitext(os.path.basename(sourcefile_path))[0].split('_') |
| words = [w.capitalize() for w in words] |
| suite_name = 'NoCompile' + ''.join(words) |
| |
| with io.StringIO() as resultfile, io.StringIO() as resultlog: |
| resultfile.write(RESULT_FILE_HEADER % sourcefile_path) |
| |
| test_configs = ExtractTestConfigs(sourcefile_path, suite_name, resultfile, |
| resultlog) |
| timings['extract_done'] = time.time() |
| |
| # Run the no-compile tests, but ensure we do not run more than |parallelism| |
| # tests at once. |
| timings['header_written'] = time.time() |
| finished_tests = [] |
| |
| includes = set() if args.depfile else None |
| |
| with concurrent.futures.ThreadPoolExecutor( |
| max_workers=parallelism) as executor: |
| finished_tests = executor.map( |
| functools.partial(RunTest, compiler, os.path.dirname(resultfile_path), |
| cflags), test_configs) |
| |
| timings['compile_done'] = time.time() |
| |
| finished_tests = sorted(finished_tests, key=lambda test: test['name']) |
| for test in finished_tests: |
| if test['name'] == 'NCTEST_SMOKE': |
| (stdout, stderr) = ExtractTestOutputAndCleanup(test) |
| return_code = test['returncode'] |
| if return_code != 0: |
| sys.stdout.write(stdout) |
| sys.stderr.write(stderr) |
| continue |
| ProcessTestResult(sourcefile_path, resultfile, resultlog, test, |
| includes) |
| timings['results_processed'] = time.time() |
| |
| WriteStats(resultlog, suite_name, timings) |
| |
| with open(resultfile_path + '.log', 'w') as fd: |
| fd.write(resultlog.getvalue()) |
| if return_code == 0: |
| with open(resultfile_path, 'w') as fd: |
| fd.write(resultfile.getvalue()) |
| |
| # Even if includes is empty, write a depfile if it was requested. |
| if args.depfile: |
| action_helpers.write_depfile(args.depfile, resultfile_path, includes) |
| |
| if return_code != 0: |
| print("No-compile driver failure with return_code %d. Result log:" % |
| return_code) |
| print(resultlog.getvalue()) |
| |
| sys.exit(return_code) |
| |
| |
| if __name__ == '__main__': |
| main() |
