| #!/usr/bin/env python |
| |
| ##===--- iwyu_test_util.py - include-what-you-use test framework ----------===## |
| # |
| # The LLVM Compiler Infrastructure |
| # |
| # This file is distributed under the University of Illinois Open Source |
| # License. See LICENSE.TXT for details. |
| # |
| ##===----------------------------------------------------------------------===## |
| |
| """Utilities for writing tests for IWYU. |
| |
| This script has been tested with python 2.7, 3.1.3 and 3.2. |
| In order to support all of these platforms there are a few unusual constructs: |
| * print statements require parentheses |
| * standard output must be decoded as utf-8 |
| * range() must be used in place of xrange() |
| * _PortableNext() is used to obtain next iterator value |
| |
| There is more detail on some of these issues at: |
| http://diveintopython3.org/porting-code-to-python-3-with-2to3.html |
| """ |
| |
| __author__ = 'wan@google.com (Zhanyong Wan)' |
| |
| import difflib |
| import operator |
| import os |
| import re |
| import subprocess |
| import sys |
| |
| # These are the warning/error lines that iwyu.cc produces when --verbose >= 3 |
| _EXPECTED_DIAGNOSTICS_RE = re.compile(r'^\s*//\s*IWYU:\s*(.*)$') |
| _ACTUAL_DIAGNOSTICS_RE = re.compile(r'^(.*?):(\d+):\d+:\s*' |
| r'(?:warning|error|fatal error):\s*(.*)$') |
| |
| # This is the final summary output that iwyu.cc produces when --verbose >= 1 |
| # The summary for a given source file should appear in that source file, |
| # surrounded by '/**** IWYU_SUMMARY' and '***** IWYU_SUMMARY */'. |
| # The leading summary line may also have an expected exit-code in parentheses |
| # after the summary marker: '/**** IWYU_SUMMARY(10)'. |
| _EXPECTED_SUMMARY_START_RE = re.compile(r'/\*+\s*IWYU_SUMMARY') |
| _EXPECTED_SUMMARY_EXIT_CODE_RE = re.compile(r'/\*+\s*IWYU_SUMMARY\((\d+)\)') |
| _EXPECTED_SUMMARY_END_RE = re.compile(r'\**\s*IWYU_SUMMARY\s*\*+/') |
| _ACTUAL_SUMMARY_START_RE = re.compile(r'^(.*?) should add these lines:$') |
| _ACTUAL_SUMMARY_END_RE = re.compile(r'^---$') |
| _ACTUAL_REMOVAL_LIST_START_RE = re.compile(r'.* should remove these lines:$') |
| _NODIFFS_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$') |
| |
| # This is an IWYU_ARGS line that specifies launch arguments for a test in its |
| # source file. Example: |
| # // IWYU_ARGS: -Xiwyu --mapping_file=... -I . |
| _IWYU_TEST_RUN_ARGS_RE = re.compile(r'^//\sIWYU_ARGS:\s(.*)$') |
| |
| |
| def _PortableNext(iterator): |
| if hasattr(iterator, 'next'): |
| iterator.next() # Python 2.4-2.6 |
| else: |
| next(iterator) # Python 3 |
| |
| |
| def _Which(program, paths): |
| """Searches specified paths for program.""" |
| if sys.platform == 'win32' and not program.lower().endswith('.exe'): |
| program += '.exe' |
| |
| for path in paths: |
| candidate = os.path.join(os.path.normpath(path), program) |
| if os.path.isfile(candidate): |
| return candidate |
| |
| return None |
| |
| |
| _IWYU_PATH = None |
| _SYSTEM_PATHS = [p.strip('"') for p in os.environ["PATH"].split(os.pathsep)] |
| _IWYU_PATHS = [ |
| '../../../../Debug+Asserts/bin', |
| '../../../../Release+Asserts/bin', |
| '../../../../Release/bin', |
| '../../../../build/Debug+Asserts/bin', |
| '../../../../build/Release+Asserts/bin', |
| '../../../../build/Release/bin', |
| # Linux/Mac OS X default out-of-tree paths. |
| '../../../../../build/Debug+Asserts/bin', |
| '../../../../../build/Release+Asserts/bin', |
| '../../../../../build/Release/bin', |
| # Windows default out-of-tree paths. |
| '../../../../../build/bin/Debug', |
| '../../../../../build/bin/Release', |
| '../../../../../build/bin/MinSizeRel', |
| '../../../../../build/bin/RelWithDebInfo', |
| ] |
| |
| |
| def SetIwyuPath(iwyu_path): |
| """Set the path to the IWYU executable under test. |
| """ |
| global _IWYU_PATH |
| _IWYU_PATH = iwyu_path |
| |
| |
| def _GetIwyuPath(): |
| """Returns the path to IWYU or raises IOError if it cannot be found.""" |
| global _IWYU_PATH |
| |
| if not _IWYU_PATH: |
| iwyu_paths = _IWYU_PATHS + _SYSTEM_PATHS |
| _IWYU_PATH = _Which('include-what-you-use', iwyu_paths) |
| if not _IWYU_PATH: |
| raise IOError('Failed to locate IWYU.\nSearched\n %s' % |
| '\n '.join(iwyu_paths)) |
| |
| return _IWYU_PATH |
| |
| |
| def _ShellQuote(arg): |
| if ' ' in arg: |
| arg = '"' + arg + '"' |
| return arg |
| |
| |
| def _GetCommandOutput(command): |
| p = subprocess.Popen(command, |
| shell=True, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT) |
| stdout, _ = p.communicate() |
| lines = stdout.decode("utf-8").splitlines(True) |
| lines = [line.replace(os.linesep, '\n') for line in lines] |
| return p.returncode, lines |
| |
| |
| def _GetMatchingLines(regex, file_names): |
| """Returns a map: file location => string matching `regex`. |
| |
| File location is a tuple (file_name, line number starting from 1).""" |
| |
| loc_to_line = {} |
| for file_name in file_names: |
| with open(file_name) as fileobj: |
| for line_num, line in enumerate(fileobj): |
| m = regex.match(line) |
| if m: |
| loc_to_line[file_name, line_num + 1] = m.group() |
| return loc_to_line |
| |
| |
| def _GetExpectedDiagnosticRegexes(spec_loc_to_line): |
| """Returns a map: source file location => list of regexes for that line.""" |
| |
| # Maps a source file line location to a list of regexes for diagnostics |
| # that should be generated for that line. |
| expected_diagnostic_regexes = {} |
| regexes = [] |
| for loc in sorted(spec_loc_to_line.keys()): |
| line = spec_loc_to_line[loc] |
| m = _EXPECTED_DIAGNOSTICS_RE.match(line.strip()) |
| assert m is not None, "Input should contain only matching lines." |
| regex = m.group(1) |
| if not regex: |
| # Allow the regex to be omitted if we are uninterested in the |
| # diagnostic message. |
| regex = r'.*' |
| regexes.append(re.compile(regex)) |
| # Do we have a spec on the next line? |
| path, line_num = loc |
| next_line_loc = path, line_num + 1 |
| if next_line_loc not in spec_loc_to_line: |
| expected_diagnostic_regexes[next_line_loc] = regexes |
| regexes = [] |
| |
| return expected_diagnostic_regexes |
| |
| |
| def _GetActualDiagnostics(actual_output): |
| """Returns a map: source file location => list of diagnostics on that line. |
| |
| The elements of the list are unique and sorted.""" |
| |
| actual_diagnostics = {} |
| for line in actual_output: |
| m = _ACTUAL_DIAGNOSTICS_RE.match(line.strip()) |
| if m: |
| path, line_num, message = m.groups() |
| loc = path, int(line_num) |
| actual_diagnostics[loc] = actual_diagnostics.get(loc, []) + [message] |
| |
| locs = actual_diagnostics.keys() |
| for loc in locs: |
| actual_diagnostics[loc] = sorted(set(actual_diagnostics[loc])) |
| |
| return actual_diagnostics |
| |
| |
| def _StripCommentFromLine(line): |
| """Removes the "// ..." comment at the end of the given line.""" |
| return re.sub(r'\s*//.*$', '', line) |
| |
| |
| def _NormalizeSummaryLineNumbers(line): |
| """Replaces the comment '// lines <number>-<number>' with '// lines XX-YY'. |
| |
| Because line numbers in the source code often change, it's a pain to |
| keep the '// lines <number>-<number>' comments accurate in our |
| 'golden' output. Instead, we normalize these iwyu comments to just |
| say how many line numbers are listed by mapping the output to |
| '// lines XX-XX' (for one-line spans) or '// lines XX-XX+<number>'. |
| For instance, '// lines 12-12' would map to '// lines XX-XX', while |
| '// lines 12-14' would map to '//lines XX-XX+2'. |
| |
| Arguments: |
| line: the line to be normalized. |
| |
| Returns: |
| A new line with the '// lines' comment, if any, normalized as |
| described above. If no '// lines' comment is present, returns |
| the original line. |
| """ |
| m = re.search('// lines ([0-9]+)-([0-9]+)', line) |
| if not m: |
| return line |
| if m.group(1) == m.group(2): |
| return line[:m.start()] + '// lines XX-XX\n' |
| else: |
| num_lines = int(m.group(2)) - int(m.group(1)) |
| return line[:m.start()] + '// lines XX-XX+%d\n' % num_lines |
| |
| |
| def _NormalizeSummaryLine(line): |
| """Alphabetically sorts the symbols in the '// for XXX, YYY, ZZZ' comments. |
| |
| Most iwyu summary lines have the form |
| #include <foo.h> // for XXX, YYY, ZZZ |
| XXX, YYY, ZZZ are symbols that this file uses from foo.h. They are |
| sorted in frequency order, but that changes so often as the test is |
| augmented, that it's impractical to test. We just sort the symbols |
| alphabetically and compare that way. This means we never test the |
| frequency ordering here, but that's a small price to pay for easier |
| testing development. |
| |
| We also always move the '// for' comment to be exactly two spaces |
| after the '#include' text. Again, this means we don't test the |
| indenting correctly (though iwyu_output_test.cc does), but allows us |
| to rename filenames without having to reformat each test. This is |
| particularly important when opensourcing, since the filenames will |
| be different in opensource-land than they are inside google. |
| |
| Arguments: |
| line: one line of the summary output |
| |
| Returns: |
| A normalized form of 'line', with the 'why' symbols sorted and |
| whitespace before the 'why' comment collapsed. |
| """ |
| m = re.match(r'(.*?)\s* // for (.*)', line) |
| if not m: |
| return line |
| symbols = m.group(2).strip().split(', ') |
| symbols.sort() |
| return '%s // for %s\n' % (m.group(1), ', '.join(symbols)) |
| |
| |
| def _GetExpectedSummaries(files): |
| """Returns a map: source file => list of iwyu summary lines.""" |
| |
| expected_summaries = {} |
| for f in files: |
| in_summary = False |
| fh = open(f) |
| for line in fh: |
| if _EXPECTED_SUMMARY_START_RE.match(line): |
| in_summary = True |
| expected_summaries[f] = [] |
| elif _EXPECTED_SUMMARY_END_RE.match(line): |
| in_summary = False |
| elif re.match(r'^\s*//', line): |
| pass # ignore comment lines |
| elif in_summary: |
| expected_summaries[f].append(line) |
| fh.close() |
| |
| # Get rid of blank lines at the beginning and end of the each summary. |
| for loc in expected_summaries: |
| while expected_summaries[loc] and expected_summaries[loc][-1] == '\n': |
| expected_summaries[loc].pop() |
| while expected_summaries[loc] and expected_summaries[loc][0] == '\n': |
| expected_summaries[loc].pop(0) |
| |
| return expected_summaries |
| |
| |
| def _GetExpectedExitCode(main_file): |
| with open(main_file, 'r') as fh: |
| for line in fh: |
| m = _EXPECTED_SUMMARY_EXIT_CODE_RE.match(line) |
| if m: |
| res = int(m.group(1)) |
| return res |
| return None |
| |
| |
| def _GetActualSummaries(output): |
| """Returns a map: source file => list of iwyu summary lines.""" |
| |
| actual_summaries = {} |
| file_being_summarized = None |
| in_addition_section = False # Are we in the "should add these lines" section? |
| for line in output: |
| # For files with no diffs, we print a different (one-line) summary. |
| m = _NODIFFS_RE.match(line) |
| if m: |
| actual_summaries[m.group(1)] = [line] |
| continue |
| |
| m = _ACTUAL_SUMMARY_START_RE.match(line) |
| if m: |
| file_being_summarized = m.group(1) |
| in_addition_section = True |
| actual_summaries[file_being_summarized] = [line] |
| elif _ACTUAL_SUMMARY_END_RE.match(line): |
| file_being_summarized = None |
| elif file_being_summarized: |
| if _ACTUAL_REMOVAL_LIST_START_RE.match(line): |
| in_addition_section = False |
| # Replace any line numbers in comments with something more stable. |
| line = _NormalizeSummaryLineNumbers(line) |
| if in_addition_section: |
| # Each #include in the "should add" list will appear later in |
| # the full include list. There's no need to verify its symbol |
| # list twice. Therefore we remove the symbol list here for |
| # easy test maintenance. |
| line = _StripCommentFromLine(line) |
| else: |
| line = _NormalizeSummaryLine(line) |
| actual_summaries[file_being_summarized].append(line) |
| |
| return actual_summaries |
| |
| |
| def _VerifyDiagnosticsAtLoc(loc_str, regexes, diagnostics): |
| """Verify the diagnostics at the given location; return a list of failures.""" |
| |
| # Find out which regexes match a diagnostic and vice versa. |
| matching_regexes = [[] for unused_i in range(len(diagnostics))] |
| matched_diagnostics = [[] for unused_i in range(len(regexes))] |
| for (r_index, regex) in enumerate(regexes): |
| for (d_index, diagnostic) in enumerate(diagnostics): |
| if regex.search(diagnostic): |
| matching_regexes[d_index].append(r_index) |
| matched_diagnostics[r_index].append(d_index) |
| |
| failure_messages = [] |
| |
| # Collect unmatched diagnostics and multiply matched diagnostics. |
| for (d_index, r_indexes) in enumerate(matching_regexes): |
| if not r_indexes: |
| failure_messages.append('Unexpected diagnostic:\n%s\n' |
| % diagnostics[d_index]) |
| elif len(r_indexes) > 1: |
| failure_messages.append( |
| 'The diagnostic message:\n%s\n' |
| 'matches multiple regexes:\n%s' |
| % (diagnostics[d_index], |
| '\n'.join([regexes[r_index].pattern for r_index in r_indexes]))) |
| |
| # Collect unmatched regexes and regexes with multiple matches. |
| for (r_index, d_indexes) in enumerate(matched_diagnostics): |
| if not d_indexes: |
| failure_messages.append('Unmatched regex:\n%s\n' |
| % regexes[r_index].pattern) |
| elif len(d_indexes) > 1: |
| failure_messages.append( |
| 'The regex:\n%s\n' |
| 'matches multiple diagnostics:\n%s' |
| % (regexes[r_index].pattern, |
| '\n'.join([diagnostics[d_index] for d_index in d_indexes]))) |
| |
| return ['%s %s' % (loc_str, message) for message in failure_messages] |
| |
| |
| def _CompareExpectedAndActualDiagnostics(expected_diagnostic_regexes, |
| actual_diagnostics): |
| """Verify that the diagnostics are as expected; return a list of failures.""" |
| |
| failures = [] |
| for loc in sorted(set(actual_diagnostics.keys()) | |
| set(expected_diagnostic_regexes.keys())): |
| # Find all regexes and actual diagnostics for the given location. |
| regexes = expected_diagnostic_regexes.get(loc, []) |
| diagnostics = actual_diagnostics.get(loc, []) |
| failures += _VerifyDiagnosticsAtLoc('\n%s:%s:' % loc, regexes, diagnostics) |
| |
| return failures |
| |
| |
| def _CompareExpectedAndActualSummaries(expected_summaries, actual_summaries): |
| """Verify that the summaries are as expected; return a list of failures.""" |
| |
| failures = [] |
| for loc in sorted(set(actual_summaries.keys()) | |
| set(expected_summaries.keys())): |
| this_failure = difflib.unified_diff(expected_summaries.get(loc, []), |
| actual_summaries.get(loc, [])) |
| try: |
| _PortableNext(this_failure) # read past the 'what files are this' header |
| failures.append('\n') |
| failures.append('Unexpected summary diffs for %s:\n' % loc) |
| failures.extend(this_failure) |
| failures.append('---\n') |
| except StopIteration: |
| pass # empty diff |
| return failures |
| |
| |
| def _GetLaunchArguments(cc_file): |
| """Gets IWYU launch arguments for a source file from its contents.""" |
| args = '' |
| with open(cc_file) as it: |
| # Find the first '// IWYU_ARGS: ' line. |
| for lineno, line in enumerate(it): |
| m = _IWYU_TEST_RUN_ARGS_RE.match(line) |
| if m: |
| args = m.group(1) |
| break |
| |
| for line in it: |
| # Consume all comment lines until we hit one that doesn't have a |
| # multi-line continuation. |
| if not line.startswith('// ') or not args.endswith('\\'): |
| break |
| line = line[3:].strip() |
| args = args[:-1] + ' ' + line |
| |
| if args.endswith('\\'): |
| raise SyntaxError('%s:%s syntax error in multiline IWYU_ARGS' % |
| (cc_file, lineno)) |
| |
| return args |
| |
| |
| def TestIwyuOnRelativeFile(cc_file, cpp_files_to_check, verbose=False): |
| """Checks running IWYU on the given .cc file. |
| |
| Args: |
| cc_file: The name of the file to test, relative to the current dir. |
| cpp_files_to_check: A list of filenames for the files |
| to check the diagnostics on, relative to the current dir. |
| verbose: Whether to display verbose output. |
| """ |
| verbosity_flags = [] |
| env_verbose_level = os.getenv('IWYU_VERBOSE') |
| if env_verbose_level: |
| verbosity_flags = ['-Xiwyu', '--verbose=' + env_verbose_level] |
| |
| cmd = '%s %s %s %s %s' % ( |
| _ShellQuote(_GetIwyuPath()), |
| # Require verbose level 3 so that we can verify the individual diagnostics. |
| # We allow the level to be overriden by |
| # * IWYU_ARGS comment in a test file; |
| # * iwyu_flags; |
| # * IWYU_VERBOSE environment variable; |
| '-Xiwyu --verbose=3', |
| _GetLaunchArguments(cc_file), |
| ' '.join(verbosity_flags), |
| cc_file) |
| if verbose: |
| print('>>> Running %s' % cmd) |
| exit_code, output = _GetCommandOutput(cmd) |
| print(''.join(output)) |
| sys.stdout.flush() # don't commingle this output with the failure output |
| |
| # Verify exit code if requested |
| expected_exit_code = _GetExpectedExitCode(cc_file) |
| if expected_exit_code is not None and exit_code != expected_exit_code: |
| raise AssertionError('Unexpected exit code, wanted %d, was %d' % |
| (expected_exit_code, exit_code)) |
| |
| expected_diagnostics = _GetMatchingLines( |
| _EXPECTED_DIAGNOSTICS_RE, cpp_files_to_check) |
| failures = _CompareExpectedAndActualDiagnostics( |
| _GetExpectedDiagnosticRegexes(expected_diagnostics), |
| _GetActualDiagnostics(output)) |
| |
| # Also figure out if the end-of-parsing suggestions match up. |
| failures += _CompareExpectedAndActualSummaries( |
| _GetExpectedSummaries(cpp_files_to_check), |
| _GetActualSummaries(output)) |
| |
| if failures: |
| raise AssertionError(''.join(failures)) |