blob: 1f98b13ee373159fa50d059cd310bb1482d30465 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Tests for code coverage tools."""
import os
import re
import shutil
import subprocess
import sys
import unittest
import coverage_utils
def _RecursiveDirectoryListing(dirpath):
"""Returns a list of relative paths to all files in a given directory."""
result = []
for root, _, files in os.walk(dirpath):
for f in files:
result.append(os.path.relpath(os.path.join(root, f), dirpath))
return result
def _ReadFile(filepath):
"""Returns contents of a given file."""
with open(filepath) as f:
return f.read()
class CoverageTest(unittest.TestCase):
def setUp(self):
self.maxDiff = 1000
self.COVERAGE_TOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
self.COVERAGE_SCRIPT = os.path.join(self.COVERAGE_TOOLS_DIR, 'coverage.py')
self.COVERAGE_UTILS = os.path.join(self.COVERAGE_TOOLS_DIR,
'coverage_utils.py')
self.CHROMIUM_SRC_DIR = os.path.dirname(
os.path.dirname(self.COVERAGE_TOOLS_DIR))
self.BUILD_DIR = os.path.join(self.CHROMIUM_SRC_DIR, 'out',
'code_coverage_tools_test')
self.REPORT_DIR_1 = os.path.join(self.BUILD_DIR, 'report1')
self.REPORT_DIR_1_NO_COMPONENTS = os.path.join(self.BUILD_DIR,
'report1_no_components')
self.REPORT_DIR_2 = os.path.join(self.BUILD_DIR, 'report2')
self.REPORT_DIR_3 = os.path.join(self.BUILD_DIR, 'report3')
self.REPORT_DIR_4 = os.path.join(self.BUILD_DIR, 'report4')
self.LLVM_COV = os.path.join(self.CHROMIUM_SRC_DIR, 'third_party',
'llvm-build', 'Release+Asserts', 'bin',
'llvm-cov')
self.PYTHON = 'python3'
self.PLATFORM = coverage_utils.GetHostPlatform()
if self.PLATFORM == 'win32':
self.LLVM_COV += '.exe'
self.PYTHON += '.exe'
# Even though 'is_component_build=false' is recommended, we intentionally
# use 'is_component_build=true' to test handling of shared libraries.
self.GN_ARGS = ('use_clang_coverage=true '
'dcheck_always_on=true '
'ffmpeg_branding=\"ChromeOS\" '
'is_component_build=true '
'is_debug=false '
'proprietary_codecs=true '
'use_libfuzzer=true')
shutil.rmtree(self.BUILD_DIR, ignore_errors=True)
gn_gen_cmd = ['gn', 'gen', self.BUILD_DIR, '--args=%s' % self.GN_ARGS]
self.run_cmd(gn_gen_cmd)
build_cmd = [
'autoninja', '-C', self.BUILD_DIR, 'crypto_unittests',
'libpng_read_fuzzer'
]
self.run_cmd(build_cmd)
def tearDown(self):
shutil.rmtree(self.BUILD_DIR, ignore_errors=True)
def run_cmd(self, cmd):
return subprocess.check_output(cmd, cwd=self.CHROMIUM_SRC_DIR)
def verify_component_view(self, filepath):
"""Asserts that a given component view looks correct."""
# There must be several Blink and Internals components.
with open(filepath) as f:
data = f.read()
counts = data.count('Blink') + data.count('Internals')
self.assertGreater(counts, 5)
def verify_directory_view(self, filepath):
"""Asserts that a given directory view looks correct."""
# Directory view page does a redirect to another page, extract its URL.
with open(filepath) as f:
data = f.read()
url = re.search(r'.*refresh.*url=([a-zA-Z0-9_\-\/.]+).*', data).group(1)
directory_view_path = os.path.join(os.path.dirname(filepath), url)
# There must be at least 'crypto' and 'third_party' directories.
with open(directory_view_path) as f:
data = f.read()
self.assertTrue('crypto' in data and 'third_party' in data)
def verify_file_view(self, filepath):
"""Asserts that a given file view looks correct."""
# There must be hundreds of '.*crypto.*' files and 10+ of '.*libpng.*'.
with open(filepath) as f:
data = f.read()
self.assertGreater(data.count('crypto'), 100)
self.assertGreater(data.count('libpng'), 10)
def verify_lcov_file(self, filepath):
"""Asserts that a given lcov file looks correct."""
with open(filepath) as f:
data = f.read()
self.assertGreater(data.count('SF:'), 100)
self.assertGreater(data.count('crypto'), 100)
self.assertGreater(data.count('libpng'), 10)
def test_different_workflows_and_cross_check_the_results(self):
"""Test a few different workflows and assert that the results are the same
and look legit.
"""
# Testcase 1. End-to-end report generation using coverage.py script. This is
# the workflow of a regular user.
cmd = [
self.COVERAGE_SCRIPT,
'crypto_unittests',
'libpng_read_fuzzer',
'-v',
'-b',
self.BUILD_DIR,
'-o',
self.REPORT_DIR_1,
'-c'
'%s/crypto_unittests' % self.BUILD_DIR,
'-c',
'%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
]
self.run_cmd(cmd)
output_dir = os.path.join(self.REPORT_DIR_1, self.PLATFORM)
self.verify_component_view(
os.path.join(output_dir, 'component_view_index.html'))
self.verify_directory_view(
os.path.join(output_dir, 'directory_view_index.html'))
self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
# Also try generating a report without components view. Useful for cross
# checking with the report produced in the testcase #3.
cmd = [
self.COVERAGE_SCRIPT,
'crypto_unittests',
'libpng_read_fuzzer',
'-v',
'-b',
self.BUILD_DIR,
'-o',
self.REPORT_DIR_1_NO_COMPONENTS,
'-c'
'%s/crypto_unittests' % self.BUILD_DIR,
'-c',
'%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
'--no-component-view',
]
self.run_cmd(cmd)
output_dir = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM)
self.verify_directory_view(
os.path.join(output_dir, 'directory_view_index.html'))
self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
self.assertFalse(
os.path.exists(os.path.join(output_dir, 'component_view_index.html')))
# Testcase #2. Run the script for post processing in Chromium tree. This is
# the workflow of the code coverage bots.
instr_profile_path = os.path.join(self.REPORT_DIR_1, self.PLATFORM,
'coverage.profdata')
cmd = [
self.COVERAGE_SCRIPT,
'crypto_unittests',
'libpng_read_fuzzer',
'-v',
'-b',
self.BUILD_DIR,
'-p',
instr_profile_path,
'-o',
self.REPORT_DIR_2,
]
self.run_cmd(cmd)
# Verify that the output dirs are the same except of the expected diff.
report_1_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_1))
report_2_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_2))
logs_subdir = os.path.join(self.PLATFORM, 'logs')
self.assertEqual(
set([
os.path.join(self.PLATFORM, 'coverage.profdata'),
os.path.join(logs_subdir, 'crypto_unittests_output.log'),
os.path.join(logs_subdir, 'libpng_read_fuzzer_output.log'),
]), report_1_listing - report_2_listing)
output_dir = os.path.join(self.REPORT_DIR_2, self.PLATFORM)
self.verify_component_view(
os.path.join(output_dir, 'component_view_index.html'))
self.verify_directory_view(
os.path.join(output_dir, 'directory_view_index.html'))
self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
# Verify that the file view pages are binary equal.
report_1_file_view_data = _ReadFile(
os.path.join(self.REPORT_DIR_1, self.PLATFORM, 'file_view_index.html'))
report_2_file_view_data = _ReadFile(
os.path.join(self.REPORT_DIR_2, self.PLATFORM, 'file_view_index.html'))
self.assertEqual(report_1_file_view_data, report_2_file_view_data)
# Testcase #3, run coverage_utils.py on manually produced report and summary
# file. This is the workflow of OSS-Fuzz code coverage job.
objects = [
'-object=%s' % os.path.join(self.BUILD_DIR, 'crypto_unittests'),
'-object=%s' % os.path.join(self.BUILD_DIR, 'libpng_read_fuzzer'),
]
cmd = [
self.PYTHON,
self.COVERAGE_UTILS,
'-v',
'shared_libs',
'-build-dir=%s' % self.BUILD_DIR,
] + objects
shared_libraries = self.run_cmd(cmd)
objects.extend(shared_libraries.split())
instr_profile_path = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS,
self.PLATFORM, 'coverage.profdata')
cmd = [
self.LLVM_COV,
'show',
'-format=html',
'-output-dir=%s' % self.REPORT_DIR_3,
'-instr-profile=%s' % instr_profile_path,
] + objects
if self.PLATFORM in ['linux', 'mac']:
cmd.extend(['-Xdemangler', 'c++filt', '-Xdemangler', '-n'])
self.run_cmd(cmd)
cmd = [
self.LLVM_COV,
'export',
'-summary-only',
'-instr-profile=%s' % instr_profile_path,
] + objects
summary_output = self.run_cmd(cmd)
summary_path = os.path.join(self.REPORT_DIR_3, 'summary.json')
with open(summary_path, 'wb') as f:
f.write(summary_output)
cmd = [
self.PYTHON,
self.COVERAGE_UTILS,
'-v',
'post_process',
'-src-root-dir=%s' % self.CHROMIUM_SRC_DIR,
'-summary-file=%s' % summary_path,
'-output-dir=%s' % self.REPORT_DIR_3,
]
self.run_cmd(cmd)
output_dir = os.path.join(self.REPORT_DIR_3, self.PLATFORM)
self.verify_directory_view(
os.path.join(output_dir, 'directory_view_index.html'))
self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
self.assertFalse(
os.path.exists(os.path.join(output_dir, 'component_view_index.html')))
# Verify that the file view pages are binary equal.
report_1_file_view_data_no_component = _ReadFile(
os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM,
'file_view_index.html'))
report_3_file_view_data = _ReadFile(
os.path.join(self.REPORT_DIR_3, self.PLATFORM, 'file_view_index.html'))
self.assertEqual(report_1_file_view_data_no_component,
report_3_file_view_data)
# Testcase 4. Export coverage data in lcov format using coverage.py script.
cmd = [
self.COVERAGE_SCRIPT,
'crypto_unittests',
'libpng_read_fuzzer',
'--format',
'lcov',
'-v',
'-b',
self.BUILD_DIR,
'-o',
self.REPORT_DIR_4,
'-c'
'%s/crypto_unittests' % self.BUILD_DIR,
'-c',
'%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
]
self.run_cmd(cmd)
output_dir = os.path.join(self.REPORT_DIR_4, self.PLATFORM)
self.verify_lcov_file(os.path.join(output_dir, 'coverage.lcov'))
if __name__ == '__main__':
unittest.main()