ios/tools/coverage/coverage.py

#!/usr/bin/python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script to generate code coverage report for iOS.

  The generated code coverage report excludes test files, and test files are
  identified by postfixes: ['unittest.cc', 'unittest.mm', 'egtest.mm'].
  TODO(crbug.com/763957): Make test file identifiers configurable.

  NOTE: This script must be called from the root of checkout. It may take up to
        a few minutes to generate a report for targets that depend on Chrome,
        such as ios_chrome_unittests. To simply play with this tool, you are
        suggested to start with 'url_unittests'.

  ios/tools/coverage/coverage.py -p path1 -p path2 target
    Generate code coverage report for |target| for |path1| and |path2|.

  ios/tools/coverage/coverage.py target -p path --reuse-profdata
  out/Coverage-iphonesimulator/coverage.profdata
    Skip running tests and reuse the specified profile data file to generate
    code coverage report.

  For more options, please refer to ios/tools/coverage/coverage.py -h

"""

import sys

import argparse
import collections
import ConfigParser
import os
import subprocess

BUILD_DIRECTORY = 'out/Coverage-iphonesimulator'
DEFAULT_GOMA_JOBS = 50

# Name of the final profdata file, and this file needs to be passed to
# "llvm-cov" command in order to call "llvm-cov show" to inspect the
# line-by-line coverage of specific files.
PROFDATA_FILE_NAME = 'coverage.profdata'

# The code coverage profraw data file is generated by running the tests with
# coverage configuration, and the path to the file is part of the log that can
# be identified with the following identifier.
PROFRAW_LOG_IDENTIFIER = 'Coverage data at '

# Only test targets with the following postfixes are considered to be valid.
VALID_TEST_TARGET_POSTFIXES = ['unittests', 'inttests', 'egtests']

# Used to determine if a test target is an earl grey test.
EARL_GREY_TEST_TARGET_POSTFIX = 'egtests'

# Used to determine if a file is a test file. The coverage of test files should
# be excluded from code coverage report.
TEST_FILES_POSTFIXES = ['unittest.mm', 'unittest.cc', 'egtest.mm']


def _CreateCoverageProfileDataForTarget(target, jobs_count=None,
                                        gtest_filter=None):
  """Builds and runs target to generate the coverage profile data.

  Args:
    target: A string representing the name of the target to be tested.
    jobs_count: Number of jobs to run in parallel for building. If None, a
                default value is derived based on CPUs availability.
    gtest_filter: If present, only run unit tests whose full name matches the
                  filter.

  Returns:
    A string representing the absolute path to the generated profdata file.
  """
  _BuildTargetWithCoverageConfiguration(target, jobs_count)
  profraw_path = _GetProfileRawDataPathByRunningTarget(target, gtest_filter)
  profdata_path = _CreateCoverageProfileDataFromProfRawData(profraw_path)

  print 'Code coverage profile data is created as: ' + profdata_path
  return profdata_path


def _DisplayLineCoverageReport(target, profdata_path, paths):
  """Generates and displays line coverage report.

  The output has the following format:
  Line Coverage Report for the Following Directories:
  dir1:
    Total Lines: 10 Executed Lines: 5 Missed Lines: 5  Coverage: 50%
  dir2:
    Total Lines: 20 Executed Lines: 2 Missed lines: 18 Coverage: 10%
  In Aggregate:
    Total Lines: 30 Executed Lines: 7 Missed Lines: 23 Coverage: 23%

  Args:
    target: A string representing the name of the target to be tested.
    profdata_path: A string representing the path to the profdata file.
    paths: A list of directories to generate code coverage for.
  """
  print 'Generating code coverge report'
  raw_line_coverage_report = _GenerateLineCoverageReport(target, profdata_path)
  line_coverage_report_with_test = _FilterLineCoverageReport(
      raw_line_coverage_report, paths)
  line_coverage_report = _ExcludeTestFiles(line_coverage_report_with_test)

  coverage_by_path = collections.defaultdict(
      lambda: collections.defaultdict(lambda: 0))
  for file_name in line_coverage_report:
    total_lines = line_coverage_report[file_name]['total_lines']
    executed_lines = line_coverage_report[file_name]['executed_lines']

    matched_paths = _MatchFilePathWithDirectories(file_name, paths)
    for matched_path in matched_paths:
      coverage_by_path[matched_path]['total_lines'] += total_lines
      coverage_by_path[matched_path]['executed_lines'] += executed_lines

    if matched_paths:
      coverage_by_path['aggregate']['total_lines'] += total_lines
      coverage_by_path['aggregate']['executed_lines'] += executed_lines

  print '\nLine Coverage Report for Following Directories: ' + str(paths)
  for path in paths:
    print path + ':'
    _PrintLineCoverageStats(coverage_by_path[path]['total_lines'],
                            coverage_by_path[path]['executed_lines'])

  if len(paths) > 1:
    print 'In Aggregate:'
    _PrintLineCoverageStats(coverage_by_path['aggregate']['total_lines'],
                            coverage_by_path['aggregate']['executed_lines'])


def _GenerateLineCoverageReport(target, profdata_path):
  """Generate code coverage report using llvm-cov report.

  The officially suggested command to export code coverage data is to use
  "llvm-cov export", which returns comprehensive code coverage data in a json
  object, however, due to the large size and complicated dependencies of
  Chrome, "llvm-cov export" takes too long to run, and for example, it takes 5
  minutes for ios_chrome_unittests. Therefore, this script gets code coverage
  data by calling "llvm-cov report", which is significantly faster and provides
  the same data.

  The raw code coverage report returned from "llvm-cov report" has the following
  format:
  Filename\tRegions\tMissed Regions\tCover\tFunctions\tMissed Functions\t
  Executed\tInstantiations\tMissed Insts.\tLines\tMissed Lines\tCover
  ------------------------------------------------------------------------------
  ------------------------------------------------------------------------------
  base/at_exit.cc\t89\t85\t4.49%\t7\t6\t14.29%\t7\t6\t14.29%\t107\t99\t7.48%
  url/pathurl.cc\t89\t85\t4.49%\t7\t6\t14.29%\t7\t6\t14.29%\t107\t99\t7.48%
  ------------------------------------------------------------------------------
  ------------------------------------------------------------------------------
  In Total\t89\t85\t4.49%\t7\t6\t14.29%\t7\t6\t14.29%\t107\t99\t7.48%

  Args:
    target: A string representing the name of the target to be tested.
    profdata_path: A string representing the path to the profdata file.

  Returns:
    A json object whose format can be found at:

    Root: dict => A dictionary of objects describing line covearge for files
    -- file_name: dict => Line coverage summary.
    ---- total_lines: int => Number of total lines.
    ---- executed_lines: int => Number of executed lines.
  """
  application_path = _GetApplicationBundlePath(target)
  binary_path = os.path.join(application_path, target)
  cmd = ['xcrun', 'llvm-cov', 'report', '-instr-profile', profdata_path,
         '-arch=x86_64', binary_path]
  std_out = subprocess.check_output(cmd)
  std_out_by_lines = std_out.split('\n')

  # Strip out the unrelated lines. The 1st line is the header and the 2nd line
  # is a '-' separator line. The last line is an empty line break, the second
  # to last line is the in total coverage and the third to last line is a '-'
  # separator line.
  coverage_content_by_files = std_out_by_lines[2: -3]

  # The 3rd to last column contains the total number of lines.
  total_lines_index = -3

  # The 2nd to last column contains the missed number of lines.
  missed_lines_index = -2

  line_coverage_report = collections.defaultdict(
      lambda: collections.defaultdict(lambda: 0))
  for coverage_content in coverage_content_by_files:
    coverage_data = coverage_content.split()
    file_name = coverage_data[0]

    # TODO(crbug.com/765818): llvm-cov has a bug that proceduces invalid data in
    # the report, and the following hack works it around. Remove the hack once
    # the bug is fixed.
    try:
      total_lines = int(coverage_data[total_lines_index])
      missed_lines = int(coverage_data[missed_lines_index])
    except ValueError as e:
      continue

    executed_lines = total_lines - missed_lines

    line_coverage_report[file_name]['total_lines'] = total_lines
    line_coverage_report[file_name]['executed_lines'] = executed_lines

  return line_coverage_report


def _FilterLineCoverageReport(raw_report, paths):
  """Filter line coverage report to only include directories in |paths|.

  Args:
    raw_report: A json object with the following format:
      Root: dict => A dictionary of objects describing line covearge for files
      -- file_name: dict => Line coverage summary.
      ---- total_lines: int => Number of total lines.
      ---- executed_lines: int => Number of executed lines.
    paths: A list of directories to generate code coverage for.

  Returns:
    A json object with the following format:

    Root: dict => A dictionary of objects describing line covearge for files
    -- file_name: dict => Line coverage summary.
    ---- total_lines: int => Number of total lines.
    ---- executed_lines: int => Number of executed lines.
  """
  filtered_report = {}
  for file_name in raw_report:
    if _MatchFilePathWithDirectories(file_name, paths):
      filtered_report[file_name] = raw_report[file_name]

  return filtered_report


def _ExcludeTestFiles(line_coverage_report_with_test):
  """Exclude test files from code coverage report.

  Test files are identified by |TEST_FILES_POSTFIXES|.

  Args:
    line_coverage_report_with_test: A json object with the following format:
      Root: dict => A dictionary of objects describing line covearge for files
      -- file_name: dict => Line coverage summary.
      ---- total_lines: int => Number of total lines.
      ---- executed_lines: int => Number of executed lines.

  Returns:
    A coverage report with test files excluded, as a json object in the format
    below:

    Root: dict => A dictionary of objects describing line covearge for files
    -- file_name: dict => Line coverage summary.
    ---- total_lines: int => Number of total lines.
    ---- executed_lines: int => Number of executed lines.
  """
  line_coverage_report = {}
  for file_name in line_coverage_report_with_test:
    if any(file_name.endswith(postfix) for postfix in TEST_FILES_POSTFIXES):
      continue

    line_coverage_report[file_name] = line_coverage_report_with_test[file_name]

  return line_coverage_report


def _PrintLineCoverageStats(total_lines, executed_lines):
  """Print line coverage statistics.

  The format is as following:
    Total Lines: 20 Executed Lines: 2 missed lines: 18 Coverage: 10%

  Args:
    total_lines: number of lines in total.
    executed_lines: number of lines that are executed.
  """
  missed_lines = total_lines - executed_lines
  coverage = float(executed_lines) / total_lines if total_lines > 0 else None
  percentage_coverage = '{}%'.format(int(coverage * 100)) if coverage else None

  output = ('\tTotal Lines: {}\tExecuted Lines: {}\tMissed Lines: {}\t'
            'Coverage: {}\n')
  print output.format(total_lines, executed_lines, missed_lines,
                      percentage_coverage or 'NA')


def _MatchFilePathWithDirectories(file_path, directories):
  """Returns the directories that contains the file.

  Args:
    file_path: the absolute path of a file that is to be matched.
    directories: A list of directories that are relative to source root.

  Returns:
    A list of directories that contains the file.
  """
  matched_directories = []
  src_root = _GetSrcRootPath()
  relative_file_path = os.path.relpath(file_path, src_root)
  for directory in directories:
    if relative_file_path.startswith(directory):
      matched_directories.append(directory)

  return matched_directories


def _BuildTargetWithCoverageConfiguration(target, jobs_count=None):
  """Builds target with coverage configuration.

  This function requires current working directory to be the root of checkout.

  Args:
    target: A string representing the name of the target to be tested.
    jobs_count: Number of jobs to run in parallel for compilation. If None, a
                default value is derived based on CPUs availability.
  """
  print 'Building ' + target

  src_root = _GetSrcRootPath()
  build_dir_path = os.path.join(src_root, BUILD_DIRECTORY)

  cmd = ['ninja', '-C', build_dir_path]
  if jobs_count:
    cmd.append('-j' + str(jobs_count))

  cmd.append(target)
  subprocess.check_call(cmd)


def _GetProfileRawDataPathByRunningTarget(target, gtest_filter=None):
  """Runs target and returns the path to the generated profraw data file.

  The output log of running the test target has no format, but it is guaranteed
  to have a single line containing the path to the generated profraw data file.

  Args:
    target: A string representing the name of the target to be tested.
    gtest_filter: If present, only run unit tests whose full name matches the
                  filter.

  Returns:
    A string representing the absolute path to the generated profraw data file.
  """
  logs = _RunTestTargetWithCoverageConfiguration(target, gtest_filter)
  for log in logs:
    if PROFRAW_LOG_IDENTIFIER in log:
      profraw_path = log.split(PROFRAW_LOG_IDENTIFIER)[1][:-1]
      return os.path.abspath(profraw_path)

  assert False, ('No profraw data file is generated, did you call '
                 'coverage_util::ConfigureCoverageReportPath() in test setup? '
                 'Please refer to base/test/test_support_ios.mm for example.')


def _RunTestTargetWithCoverageConfiguration(target, gtest_filter=None):
  """Runs tests to generate the profraw data file.

  This function requires current working directory to be the root of checkout.

  Args:
    target: A string representing the name of the target to be tested.
    gtest_filter: If present, only run unit tests whose full name matches the
                  filter.

  Returns:
    A list of lines/strings created from the output log by breaking lines. The
    log has no format, but it is guaranteed to have a single line containing the
    path to the generated profraw data file.
  """
  iossim_path = _GetIOSSimPath()
  application_path = _GetApplicationBundlePath(target)

  cmd = [iossim_path]

  # For iossim arguments, please refer to src/testing/iossim/iossim.mm.
  if gtest_filter:
    cmd.append('-c --gtest_filter=' + gtest_filter)

  cmd.append(application_path)
  if _TargetIsEarlGreyTest(target):
    cmd.append(_GetXCTestBundlePath(target))

  print 'Running {} with command: {}'.format(target, ' '.join(cmd))

  logs_chracters = subprocess.check_output(cmd)

  return ''.join(logs_chracters).split('\n')


def _CreateCoverageProfileDataFromProfRawData(profraw_path):
  """Returns the path to the profdata file by merging profraw data file.

  Args:
    profraw_path: A string representing the absolute path to the profraw data
                  file that is to be merged.

  Returns:
    A string representing the absolute path to the generated profdata file.

  Raises:
    CalledProcessError: An error occurred merging profraw data files.
  """
  print 'Creating the profile data file'

  src_root = _GetSrcRootPath()
  profdata_path = os.path.join(src_root, BUILD_DIRECTORY,
                               PROFDATA_FILE_NAME)
  try:
    cmd = ['xcrun', 'llvm-profdata', 'merge', '-o', profdata_path, profraw_path]
    subprocess.check_call(cmd)
  except subprocess.CalledProcessError as error:
    print 'Failed to merge profraw to create profdata.'
    raise error

  return profdata_path


def _GetSrcRootPath():
  """Returns the absolute path to the root of checkout.

  Returns:
    A string representing the absolute path to the root of checkout.
  """
  return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir,
                                      os.pardir, os.pardir))


def _GetApplicationBundlePath(target):
  """Returns the path to the generated application bundle after building.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A string representing the path to the generated application bundle.
  """
  src_root = _GetSrcRootPath()
  application_bundle_name = target + '.app'
  return os.path.join(src_root, BUILD_DIRECTORY, application_bundle_name)


def _GetXCTestBundlePath(target):
  """Returns the path to the xctest bundle after building.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A string representing the path to the generated xctest bundle.
  """
  application_path = _GetApplicationBundlePath(target)
  xctest_bundle_name = target + '_module.xctest'
  return os.path.join(application_path, 'PlugIns', xctest_bundle_name)


def _GetIOSSimPath():
  """Returns the path to the iossim executable file after building.

  Returns:
    A string representing the path to the iossim executable file.
  """
  src_root = _GetSrcRootPath()
  iossim_path = os.path.join(src_root, BUILD_DIRECTORY, 'iossim')
  return iossim_path


def _IsGomaConfigured():
  """Returns True if goma is enabled in the gn build settings.

  Returns:
    A boolean indicates whether goma is configured for building or not.
  """
  # Load configuration.
  settings = ConfigParser.SafeConfigParser()
  settings.read(os.path.expanduser('~/.setup-gn'))
  return settings.getboolean('goma', 'enabled')


def _TargetIsEarlGreyTest(target):
  """Returns true if the target is an earl grey test.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A boolean indicates whether the target is an earl grey test or not.
  """
  return target.endswith(EARL_GREY_TEST_TARGET_POSTFIX)


def _TargetNameIsValidTestTarget(target):
  """Returns True if the target name has a valid postfix.

  The list of valid target name postfixes are defined in
  VALID_TEST_TARGET_POSTFIXES.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A boolean indicates whether the target is a valid test target.
  """
  return (any(target.endswith(postfix) for postfix in
              VALID_TEST_TARGET_POSTFIXES))


def _ParseCommandArguments():
  """Add and parse relevant arguments for tool commands.

  Returns:
    A dictionanry representing the arguments.
  """
  arg_parser = argparse.ArgumentParser()
  arg_parser.usage = __doc__

  arg_parser.add_argument('-p', '--path', action='append', required=True,
                          help='Directories to get code coverage for.')

  arg_parser.add_argument('-j', '--jobs', type=int, default=None,
                          help='Run N jobs to build in parallel. If not '
                               'specified, a default value will be derived '
                               'based on CPUs availability. Please refer to '
                               '\'ninja -h\' for more details.')

  arg_parser.add_argument('-r', '--reuse-profdata',
                          help='Skip building test target and running tests '
                               'and re-use the specified profile data file.')

  arg_parser.add_argument('--gtest_filter', type=str,
                          help='Only run unit tests whose full name matches '
                               'the filter.')

  arg_parser.add_argument('target', nargs='+',
                          help='The name of the test target to run.')

  args = arg_parser.parse_args()
  return args


def _AssertCoverageBuildDirectoryExists():
  """Asserts that the build directory with converage configuration exists."""
  src_root = _GetSrcRootPath()
  build_dir_path = os.path.join(src_root, BUILD_DIRECTORY)
  assert os.path.exists(build_dir_path), (build_dir_path + " doesn't exist."
                                          'Hint: run gclient runhooks or '
                                          'ios/build/tools/setup-gn.py.')


def _AssertPathsExist(paths):
  """Asserts that paths specified in |paths| exist.

  Args:
    paths: A list of directories.
  """
  src_root = _GetSrcRootPath()
  for path in paths:
    abspath = os.path.join(src_root, path)
    assert os.path.exists(abspath), (('Path: {} doesn\'t exist.\n A valid '
                                      'path must exist and be relative to the '
                                      'root of source, which is {} \nFor '
                                      'example, \'ios/\' is a valid path.').
                                     format(abspath, src_root))


def Main():
  """Executes tool commands."""
  args = _ParseCommandArguments()
  targets = args.target
  assert len(targets) == 1, ('targets: ' + str(targets) + ' are detected, '
                             'however, only a single target is supported now.')

  target = targets[0]
  if not _TargetNameIsValidTestTarget(target):
    assert False, ('target: ' + str(target) + ' is detected, however, only '
                   'target name with the following postfixes are supported: ' +
                   str(VALID_TEST_TARGET_POSTFIXES))

  jobs = args.jobs
  if not jobs and _IsGomaConfigured():
    jobs = DEFAULT_GOMA_JOBS

  _AssertCoverageBuildDirectoryExists()
  _AssertPathsExist(args.path)

  profdata_path = args.reuse_profdata
  if profdata_path:
    assert os.path.exists(profdata_path), ('The provided profile data file: {} '
                                           'doesn\'t exist.').format(
                                               profdata_path)
  else:
    profdata_path = _CreateCoverageProfileDataForTarget(target, jobs,
                                                        args.gtest_filter)

  _DisplayLineCoverageReport(target, profdata_path, args.path)

if __name__ == '__main__':
  sys.exit(Main())