ios/tools/coverage/coverage.py

#!/usr/bin/python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script to generate code coverage reports for iOS.

  NOTE: This script must be called from the root of checkout. It may take up to
        a few minutes to generate a report for targets that depend on Chrome,
        such as ios_chrome_unittests. To simply play with this tool, you are
        suggested to start with 'url_unittests'.

  ios/tools/coverage/coverage.py target
    Generate code coverage report for |target| and restrict the results to ios/.

  ios/tools/coverage/coverage.py -f path1 -f path2 target
    Generate code coverage report for |target| and restrict the results to
    |path1| and |path2|.

  For more info, please refer to ios/tools/coverage/coverage.py -h

"""

import sys

import argparse
import ConfigParser
import os
import subprocess

BUILD_DIRECTORY = 'out/Coverage-iphonesimulator'
DEFAULT_GOMA_JOBS = 50

# Name of the final profdata file, and this file needs to be passed to
# "llvm-cov" command in order to call "llvm-cov show" to inspect the
# line-by-line coverage of specific files.
PROFDATA_FILE_NAME = 'coverage.profdata'

# The code coverage profraw data file is generated by running the tests with
# coverage configuration, and the path to the file is part of the log that can
# be identified with the following identifier.
PROFRAW_LOG_IDENTIFIER = 'Coverage data at '

# By default, code coverage results are restricted to 'ios/' directory.
# If the filter arguments are defined, they will override the default values.
# Having default values are important because otherwise the code coverage data
# returned by "llvm-cove" will include completely unrelated directories such as
# 'base/' and 'url/'.
DEFAULT_FILTERS = ['ios/']

# Only test targets with the following postfixes are considered to be valid.
VALID_TEST_TARGET_POSTFIXES = ['unittests', 'inttests', 'egtests']

# Used to determine if a test target is an earl grey test.
EARL_GREY_TEST_TARGET_POSTFIX = 'egtests'


def CreateCoverageProfileDataForTarget(target, jobs_count=None):
  """Builds and runs target to generate the coverage profile data.

  Args:
    target: A string representing the name of the target.
    jobs_count: Number of jobs to run in parallel for building. If None, a
                default value is derived based on CPUs availability.

  Returns:
    A string representing the absolute path to the generated profdata file.
  """
  _BuildTargetWithCoverageConfiguration(target, jobs_count)
  profraw_path = _GetProfileRawDataPathByRunningTarget(target)
  profdata_path = _CreateCoverageProfileDataFromProfRawData(profraw_path)

  print 'Code coverage profile data is created as: ' + profdata_path
  return profdata_path


def _BuildTargetWithCoverageConfiguration(target, jobs_count):
  """Builds target with coverage configuration.

  This function requires current working directory to be the root of checkout.

  Args:
    target: A string representing the name of the target to be tested.
    jobs_count: Number of jobs to run in parallel for compilation. If None, a
                default value is derived based on CPUs availability.
  """
  print 'Building ' + target

  default_root = _GetSrcRootPath()
  build_dir_path = os.path.join(default_root, BUILD_DIRECTORY)

  cmd = ['ninja', '-C', build_dir_path]
  if jobs_count:
    cmd.append('-j' + str(jobs_count))

  cmd.append(target)
  subprocess.check_call(cmd)


def _GetProfileRawDataPathByRunningTarget(target):
  """Runs target and returns the path to the generated profraw data file.

  The output log of running the test target has no format, but it is guaranteed
  to have a single line containing the path to the generated profraw data file.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A string representing the absolute path to the generated profraw data file.
  """
  logs = _RunTestTargetWithCoverageConfiguration(target)
  for log in logs:
    if PROFRAW_LOG_IDENTIFIER in log:
      profraw_path = log.split(PROFRAW_LOG_IDENTIFIER)[1][:-1]
      return os.path.abspath(profraw_path)

  assert False, ('No profraw data file is generated, did you call '
                 'coverage_util::ConfigureCoverageReportPath() in test setup? '
                 'Please refer to base/test/test_support_ios.mm for example.')


def _RunTestTargetWithCoverageConfiguration(target):
  """Runs tests to generate the profraw data file.

  This function requires current working directory to be the root of checkout.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A list of lines/strings created from the output log by breaking lines. The
    log has no format, but it is guaranteed to have a single line containing the
    path to the generated profraw data file.
  """
  print 'Running ' + target

  iossim_path = _GetIOSSimPath()
  application_path = _GetApplicationBundlePath(target)

  cmd = [iossim_path, application_path]
  if _TargetIsEarlGreyTest(target):
    cmd.append(_GetXCTestBundlePath(target))

  logs_chracters = subprocess.check_output(cmd)
  return ''.join(logs_chracters).split('\n')


def _CreateCoverageProfileDataFromProfRawData(profraw_path):
  """Returns the path to the profdata file by merging profraw data file.

  Args:
    profraw_path: A string representing the absolute path to the profraw data
                  file that is to be merged.

  Returns:
    A string representing the absolute path to the generated profdata file.

  Raises:
    CalledProcessError: An error occurred merging profraw data files.
  """
  print 'Creating the profile data file'

  default_root = _GetSrcRootPath()
  profdata_path = os.path.join(default_root, BUILD_DIRECTORY,
                               PROFDATA_FILE_NAME)
  try:
    cmd = ['xcrun', 'llvm-profdata', 'merge', '-o', profdata_path, profraw_path]
    subprocess.check_call(cmd)
  except subprocess.CalledProcessError as error:
    print 'Failed to merge profraw to create profdata.'
    raise error

  return profdata_path


def _GetSrcRootPath():
  """Returns the absolute path to the root of checkout.

  Returns:
    A string representing the absolute path to the root of checkout.
  """
  return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir,
                                      os.pardir, os.pardir))


def _GetApplicationBundlePath(target):
  """Returns the path to the generated application bundle after building.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A string representing the path to the generated application bundle.
  """
  default_root = _GetSrcRootPath()
  application_bundle_name = target + '.app'
  return os.path.join(default_root, BUILD_DIRECTORY, application_bundle_name)


def _GetXCTestBundlePath(target):
  """Returns the path to the xctest bundle after building.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A string representing the path to the generated xctest bundle.
  """
  application_path = _GetApplicationBundlePath(target);
  xctest_bundle_name = target + '_module.xctest'
  return os.path.join(application_path, 'PlugIns', xctest_bundle_name)


def _GetIOSSimPath():
  """Returns the path to the iossim executable file after building.

  Returns:
    A string representing the path to the iossim executable file.
  """
  default_root = _GetSrcRootPath()
  iossim_path = os.path.join(default_root, BUILD_DIRECTORY, 'iossim')
  return iossim_path


def _IsGomaConfigured():
  """Returns True if goma is enabled in the gn build settings.

  Returns:
    A boolean indicates whether goma is configured for building or not.
  """
  # Load configuration.
  settings = ConfigParser.SafeConfigParser()
  settings.read(os.path.expanduser('~/.setup-gn'))
  return settings.getboolean('goma', 'enabled')


def _TargetIsEarlGreyTest(target):
  """Returns true if the target is an earl grey test.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A boolean indicates whether the target is an earl grey test or not.
  """
  return target.endswith(EARL_GREY_TEST_TARGET_POSTFIX)


def _TargetNameIsValidTestTarget(target):
  """Returns True if the target name has a valid postfix.

  The list of valid target name postfixes are defined in
  VALID_TEST_TARGET_POSTFIXES.

  Args:
    target: A string representing the name of the target to be tested.

  Returns:
    A boolean indicates whether the target is a valid test target.
  """
  return (any(target.endswith(postfix) for postfix in
              VALID_TEST_TARGET_POSTFIXES))


def _ParseCommandArguments():
  """Add and parse relevant arguments for tool commands.

  Returns:
    A dictionanry representing the arguments.
  """
  arg_parser = argparse.ArgumentParser()
  arg_parser.usage = __doc__

  arg_parser.add_argument('-f', '--filter', type=str, action='append',
                          help='Paths used to restrict code coverage results '
                               'to specific directories, and the default value '
                               'is \'ios/\'. \n'
                               'NOTE: if this value is defined, it will '
                               'override instead of appeding to the defaults.')

  arg_parser.add_argument('-j', '--jobs', type=int, default=None,
                          help='Run N jobs to build in parallel. If not '
                               'specified, a default value will be derived '
                               'based on CPUs availability. Please refer to '
                               '\'ninja -h\' for more details.')

  arg_parser.add_argument('target', nargs='+',
                          help='The name of the test target to run.')

  args = arg_parser.parse_args()
  return args


def _AssertCoverageBuildDirectoryExists():
  """Asserts that the build directory with converage configuration exists."""
  default_root = _GetSrcRootPath()
  build_dir_path = os.path.join(default_root, BUILD_DIRECTORY)
  assert os.path.exists(build_dir_path), (build_dir_path + " doesn't exist."
                                          'Hint: run gclient runhooks or '
                                          'ios/build/tools/setup-gn.py.')


def Main():
  """Executes tool commands."""
  args = _ParseCommandArguments()
  targets = args.target
  assert len(targets) == 1, ('targets: ' + str(targets) + ' are detected, '
                             'however, only a single target is supported now.')

  target = targets[0]
  if not _TargetNameIsValidTestTarget(target):
    assert False, ('target: ' + str(target) + ' is detected, however, only '
                   'target name with the following postfixes are supported: ' +
                   str(VALID_TEST_TARGET_POSTFIXES))

  jobs = args.jobs
  if not jobs and _IsGomaConfigured():
    jobs = DEFAULT_GOMA_JOBS

  _AssertCoverageBuildDirectoryExists()

  CreateCoverageProfileDataForTarget(target, jobs)

if __name__ == '__main__':
  sys.exit(Main())