scripts/test_tool.py

#!/usr/bin/env vpython3
# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Test harness for chromium clang tools."""

from __future__ import print_function

import argparse
import difflib
import glob
import json
import os
import os.path
import re
import shutil
import subprocess
import sys


def _RunGit(args):
  if sys.platform == 'win32':
    args = ['git.bat'] + args
  else:
    args = ['git'] + args
  subprocess.check_call(args)


def _GenerateCompileCommands(files, include_paths):
  """Returns a JSON string containing a compilation database for the input."""
  # Note: in theory, backslashes in the compile DB should work but the tools
  # that write compile DBs and the tools that read them don't agree on the
  # escaping convention: https://llvm.org/bugs/show_bug.cgi?id=19687
  files = [f.replace('\\', '/') for f in files]
  include_path_flags = ' '.join('-I %s' % include_path.replace('\\', '/')
                                for include_path in include_paths)
  return json.dumps([{
      'directory':
      os.path.dirname(f),
      'command':
      'clang++ -std=c++20 -fsyntax-only %s %s' %
      (include_path_flags, os.path.basename(f)),
      'file':
      os.path.basename(f)
  } for f in files],
                    indent=2)


def _NumberOfTestsToString(tests):
  """Returns an English describing the number of tests."""
  return '%d test%s' % (tests, 's' if tests != 1 else '')


def _ApplyTool(tools_clang_scripts_directory, tool_to_test, tool_path,
               tool_args, test_directory_for_tool, actual_files, apply_edits,
               extract_edits_path):
  try:
    # Stage the test files in the git index. If they aren't staged, then
    # run_tool.py will skip them when applying replacements.
    args = ['add']
    args.extend(actual_files)
    _RunGit(args)

    # Launch the following pipeline if |apply_edits| is True:
    #     run_tool.py ... | extract_edits.py | apply_edits.py ...
    # Otherwise just the first step is done and the result is written to
    #   actual_files[0].
    processes = []
    args = ['python',
            os.path.join(tools_clang_scripts_directory, 'run_tool.py')]
    extra_run_tool_args_path = os.path.join(test_directory_for_tool,
                                            'run_tool.args')
    if os.path.exists(extra_run_tool_args_path):
      with open(extra_run_tool_args_path, 'r') as extra_run_tool_args_file:
        extra_run_tool_args = extra_run_tool_args_file.readlines()
        args.extend([arg.strip() for arg in extra_run_tool_args])
    args.extend(['--tool', tool_to_test, '-p', test_directory_for_tool])

    if tool_path:
      args.extend(['--tool-path', tool_path])
    if tool_args:
      for arg in tool_args:
        args.append('--tool-arg=%s' % arg)

    args.extend(actual_files)
    processes.append(subprocess.Popen(args, stdout=subprocess.PIPE))

    if apply_edits:
      if not extract_edits_path:
        args = [
            'python',
            os.path.join(tools_clang_scripts_directory, 'extract_edits.py')
        ]
        processes.append(
            subprocess.Popen(args,
                             stdin=processes[-1].stdout,
                             stdout=subprocess.PIPE))
      else:
        args = ['python', os.path.join(extract_edits_path, 'extract_edits.py')]
        processes.append(
            subprocess.Popen(args,
                             stdin=processes[-1].stdout,
                             stdout=subprocess.PIPE))

      args = [
          'python',
          os.path.join(tools_clang_scripts_directory, 'apply_edits.py'), '-p',
          test_directory_for_tool
      ]
      args.extend(actual_files)  # Limit edits to the test files.
      processes.append(subprocess.Popen(
          args, stdin=processes[-1].stdout, stdout=subprocess.PIPE))

    # Wait for the pipeline to finish running + check exit codes.
    stdout, _ = processes[-1].communicate()
    for process in processes:
      process.wait()
      if process.returncode != 0:
        print('Failure while running the tool.')
        return process.returncode

    if apply_edits:
      # Reformat the resulting edits via: git cl format.
      args = ['cl', 'format']
      args.extend(actual_files)
      _RunGit(args)
    else:
      with open(actual_files[0], 'w') as output_file:
        output_file.write(stdout.decode('utf-8'))

    return 0

  finally:
    # No matter what, unstage the git changes we made earlier to avoid polluting
    # the index.
    args = ['reset', '--quiet', 'HEAD']
    args.extend(actual_files)
    _RunGit(args)


def _NormalizePathInRawOutput(path, test_dir):
  if not os.path.isabs(path):
    path = os.path.join(test_dir, path)

  return os.path.relpath(path, test_dir)


def _NormalizeSingleRawOutputLine(output_line, test_dir):
  if not re.match('^[^:]+(:::.*){4,4}$', output_line):
    return output_line

  edit_type, path, offset, length, replacement = output_line.split(':::', 4)
  path = _NormalizePathInRawOutput(path, test_dir)
  return "%s:::%s:::%s:::%s:::%s" % (edit_type, path, offset, length,
                                     replacement)


def _NormalizeRawOutput(output_lines, test_dir):
  return list(
      map(lambda line: _NormalizeSingleRawOutputLine(line, test_dir),
          output_lines))


def main(argv):
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--apply-edits',
      action='store_true',
      help='Applies the edits to the original test files and compares the '
           'reformatted new files with the expected files.')
  parser.add_argument(
      '--tool-arg', nargs='?', action='append',
      help='optional arguments passed to the tool')
  parser.add_argument(
      '--tool-path', nargs='?',
      help='optional path to the tool directory')
  parser.add_argument('tool_name',
                      nargs=1,
                      help='Clang tool to be tested.')
  parser.add_argument(
      '--test-filter', default='*', help='optional glob filter for test names')
  parser.add_argument('--extract-edits-path',
                      nargs='?',
                      help='optional path to the extract_edits script\
      [e.g. if custom filtering or post-processing of edits is needed]')
  args = parser.parse_args(argv)
  tool_to_test = args.tool_name[0]
  print('\nTesting %s\n' % tool_to_test)
  tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__))
  tools_clang_directory = os.path.dirname(tools_clang_scripts_directory)
  test_directory_for_tool = os.path.join(
      tools_clang_directory, tool_to_test, 'tests')
  compile_database = os.path.join(test_directory_for_tool,
                                  'compile_commands.json')
  source_files = glob.glob(
      os.path.join(test_directory_for_tool,
                   '%s-original.cc' % args.test_filter))
  ext = 'cc' if args.apply_edits else 'txt'
  actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc'])
                  for source_file in source_files]
  expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.' + ext])
                    for source_file in source_files]
  if not args.apply_edits and len(actual_files) != 1:
    print('Only one test file is expected for testing without apply-edits.')
    return 1

  include_paths = []
  include_paths.append(
      os.path.realpath(os.path.join(tools_clang_directory, '../..')))
  # Many gtest and gmock headers expect to have testing/gtest/include and/or
  # testing/gmock/include in the include search path.
  include_paths.append(
      os.path.realpath(os.path.join(tools_clang_directory,
                                    '../..',
                                    'testing/gtest/include')))
  include_paths.append(
      os.path.realpath(os.path.join(tools_clang_directory,
                                    '../..',
                                    'testing/gmock/include')))

  include_paths.append(
      os.path.realpath(
          os.path.join(tools_clang_directory, '../..',
                       'third_party/googletest/src/googletest/include')))

  include_paths.append(
      os.path.realpath(
          os.path.join(tools_clang_directory, '../..',
                       'third_party/googletest/src/googlemock/include')))

  if len(actual_files) == 0:
    print('Tool "%s" does not have compatible test files.' % tool_to_test)
    return 1

  # Set up the test environment.
  for source, actual in zip(source_files, actual_files):
    shutil.copyfile(source, actual)
  # Generate a temporary compilation database to run the tool over.
  with open(compile_database, 'w') as f:
    f.write(_GenerateCompileCommands(actual_files, include_paths))

  # Run the tool.
  os.chdir(test_directory_for_tool)
  exitcode = _ApplyTool(tools_clang_scripts_directory, tool_to_test,
                        args.tool_path, args.tool_arg, test_directory_for_tool,
                        actual_files, args.apply_edits, args.extract_edits_path)
  if (exitcode != 0):
    return exitcode

  # Compare actual-vs-expected results.
  passed = 0
  failed = 0
  for expected, actual in zip(expected_files, actual_files):
    print('[ RUN      ] %s' % os.path.relpath(actual))
    expected_output = actual_output = None
    with open(expected, 'r') as f:
      expected_output = f.readlines()
    with open(actual, 'r') as f:
      actual_output =  f.readlines()
    if not args.apply_edits:
      actual_output = _NormalizeRawOutput(actual_output,
                                          test_directory_for_tool)
      expected_output = _NormalizeRawOutput(expected_output,
                                            test_directory_for_tool)
    if actual_output != expected_output:
      failed += 1
      lines = difflib.unified_diff(expected_output, actual_output,
                                   fromfile=os.path.relpath(expected),
                                   tofile=os.path.relpath(actual))
      sys.stdout.writelines(lines)
      print('[  FAILED  ] %s' % os.path.relpath(actual))
      # Don't clean up the file on failure, so the results can be referenced
      # more easily.
      continue
    print('[       OK ] %s' % os.path.relpath(actual))
    passed += 1
    os.remove(actual)

  if failed == 0:
    os.remove(compile_database)

  print('[==========] %s ran.' % _NumberOfTestsToString(len(source_files)))
  if passed > 0:
    print('[  PASSED  ] %s.' % _NumberOfTestsToString(passed))
  if failed > 0:
    print('[  FAILED  ] %s.' % _NumberOfTestsToString(failed))
    return 1


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))