| #!/usr/bin/env python |
| # Copyright 2018 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Tests for code coverage tools.""" |
| |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import unittest |
| import coverage_utils |
| |
| |
| def _RecursiveDirectoryListing(dirpath): |
| """Returns a list of relative paths to all files in a given directory.""" |
| result = [] |
| for root, _, files in os.walk(dirpath): |
| for f in files: |
| result.append(os.path.relpath(os.path.join(root, f), dirpath)) |
| return result |
| |
| |
| def _ReadFile(filepath): |
| """Returns contents of a given file.""" |
| with open(filepath) as f: |
| return f.read() |
| |
| |
| class CoverageTest(unittest.TestCase): |
| |
| def setUp(self): |
| self.maxDiff = 1000 |
| self.COVERAGE_TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) |
| self.COVERAGE_SCRIPT = os.path.join(self.COVERAGE_TOOLS_DIR, 'coverage.py') |
| self.COVERAGE_UTILS = os.path.join(self.COVERAGE_TOOLS_DIR, |
| 'coverage_utils.py') |
| |
| self.CHROMIUM_SRC_DIR = os.path.dirname( |
| os.path.dirname(self.COVERAGE_TOOLS_DIR)) |
| self.BUILD_DIR = os.path.join(self.CHROMIUM_SRC_DIR, 'out', |
| 'code_coverage_tools_test') |
| |
| self.REPORT_DIR_1 = os.path.join(self.BUILD_DIR, 'report1') |
| self.REPORT_DIR_1_NO_COMPONENTS = os.path.join(self.BUILD_DIR, |
| 'report1_no_components') |
| self.REPORT_DIR_2 = os.path.join(self.BUILD_DIR, 'report2') |
| self.REPORT_DIR_3 = os.path.join(self.BUILD_DIR, 'report3') |
| self.REPORT_DIR_4 = os.path.join(self.BUILD_DIR, 'report4') |
| |
| self.LLVM_COV = os.path.join(self.CHROMIUM_SRC_DIR, 'third_party', |
| 'llvm-build', 'Release+Asserts', 'bin', |
| 'llvm-cov') |
| |
| self.PYTHON = 'python3' |
| self.PLATFORM = coverage_utils.GetHostPlatform() |
| if self.PLATFORM == 'win32': |
| self.LLVM_COV += '.exe' |
| self.PYTHON += '.exe' |
| |
| # Even though 'is_component_build=false' is recommended, we intentionally |
| # use 'is_component_build=true' to test handling of shared libraries. |
| self.GN_ARGS = ('use_clang_coverage=true ' |
| 'dcheck_always_on=true ' |
| 'ffmpeg_branding=\"ChromeOS\" ' |
| 'is_component_build=true ' |
| 'is_debug=false ' |
| 'proprietary_codecs=true ' |
| 'use_libfuzzer=true') |
| |
| shutil.rmtree(self.BUILD_DIR, ignore_errors=True) |
| |
| gn_gen_cmd = ['gn', 'gen', self.BUILD_DIR, '--args=%s' % self.GN_ARGS] |
| self.run_cmd(gn_gen_cmd) |
| |
| build_cmd = [ |
| 'autoninja', '-C', self.BUILD_DIR, 'crypto_unittests', |
| 'libpng_read_fuzzer' |
| ] |
| self.run_cmd(build_cmd) |
| |
| def tearDown(self): |
| shutil.rmtree(self.BUILD_DIR, ignore_errors=True) |
| |
| def run_cmd(self, cmd): |
| return subprocess.check_output(cmd, cwd=self.CHROMIUM_SRC_DIR) |
| |
| def verify_component_view(self, filepath): |
| """Asserts that a given component view looks correct.""" |
| # There must be several Blink and Internals components. |
| with open(filepath) as f: |
| data = f.read() |
| |
| counts = data.count('Blink') + data.count('Internals') |
| self.assertGreater(counts, 5) |
| |
| def verify_directory_view(self, filepath): |
| """Asserts that a given directory view looks correct.""" |
| # Directory view page does a redirect to another page, extract its URL. |
| with open(filepath) as f: |
| data = f.read() |
| |
| url = re.search(r'.*refresh.*url=([a-zA-Z0-9_\-\/.]+).*', data).group(1) |
| directory_view_path = os.path.join(os.path.dirname(filepath), url) |
| |
| # There must be at least 'crypto' and 'third_party' directories. |
| with open(directory_view_path) as f: |
| data = f.read() |
| |
| self.assertTrue('crypto' in data and 'third_party' in data) |
| |
| def verify_file_view(self, filepath): |
| """Asserts that a given file view looks correct.""" |
| # There must be hundreds of '.*crypto.*' files and 10+ of '.*libpng.*'. |
| with open(filepath) as f: |
| data = f.read() |
| |
| self.assertGreater(data.count('crypto'), 100) |
| self.assertGreater(data.count('libpng'), 10) |
| |
| def verify_lcov_file(self, filepath): |
| """Asserts that a given lcov file looks correct.""" |
| with open(filepath) as f: |
| data = f.read() |
| |
| self.assertGreater(data.count('SF:'), 100) |
| self.assertGreater(data.count('crypto'), 100) |
| self.assertGreater(data.count('libpng'), 10) |
| |
| def test_different_workflows_and_cross_check_the_results(self): |
| """Test a few different workflows and assert that the results are the same |
| |
| and look legit. |
| """ |
| # Testcase 1. End-to-end report generation using coverage.py script. This is |
| # the workflow of a regular user. |
| cmd = [ |
| self.COVERAGE_SCRIPT, |
| 'crypto_unittests', |
| 'libpng_read_fuzzer', |
| '-v', |
| '-b', |
| self.BUILD_DIR, |
| '-o', |
| self.REPORT_DIR_1, |
| '-c' |
| '%s/crypto_unittests' % self.BUILD_DIR, |
| '-c', |
| '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR, |
| ] |
| self.run_cmd(cmd) |
| |
| output_dir = os.path.join(self.REPORT_DIR_1, self.PLATFORM) |
| self.verify_component_view( |
| os.path.join(output_dir, 'component_view_index.html')) |
| self.verify_directory_view( |
| os.path.join(output_dir, 'directory_view_index.html')) |
| self.verify_file_view(os.path.join(output_dir, 'file_view_index.html')) |
| |
| # Also try generating a report without components view. Useful for cross |
| # checking with the report produced in the testcase #3. |
| cmd = [ |
| self.COVERAGE_SCRIPT, |
| 'crypto_unittests', |
| 'libpng_read_fuzzer', |
| '-v', |
| '-b', |
| self.BUILD_DIR, |
| '-o', |
| self.REPORT_DIR_1_NO_COMPONENTS, |
| '-c' |
| '%s/crypto_unittests' % self.BUILD_DIR, |
| '-c', |
| '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR, |
| '--no-component-view', |
| ] |
| self.run_cmd(cmd) |
| |
| output_dir = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM) |
| self.verify_directory_view( |
| os.path.join(output_dir, 'directory_view_index.html')) |
| self.verify_file_view(os.path.join(output_dir, 'file_view_index.html')) |
| self.assertFalse( |
| os.path.exists(os.path.join(output_dir, 'component_view_index.html'))) |
| |
| # Testcase #2. Run the script for post processing in Chromium tree. This is |
| # the workflow of the code coverage bots. |
| instr_profile_path = os.path.join(self.REPORT_DIR_1, self.PLATFORM, |
| 'coverage.profdata') |
| |
| cmd = [ |
| self.COVERAGE_SCRIPT, |
| 'crypto_unittests', |
| 'libpng_read_fuzzer', |
| '-v', |
| '-b', |
| self.BUILD_DIR, |
| '-p', |
| instr_profile_path, |
| '-o', |
| self.REPORT_DIR_2, |
| ] |
| self.run_cmd(cmd) |
| |
| # Verify that the output dirs are the same except of the expected diff. |
| report_1_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_1)) |
| report_2_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_2)) |
| logs_subdir = os.path.join(self.PLATFORM, 'logs') |
| self.assertEqual( |
| set([ |
| os.path.join(self.PLATFORM, 'coverage.profdata'), |
| os.path.join(logs_subdir, 'crypto_unittests_output.log'), |
| os.path.join(logs_subdir, 'libpng_read_fuzzer_output.log'), |
| ]), report_1_listing - report_2_listing) |
| |
| output_dir = os.path.join(self.REPORT_DIR_2, self.PLATFORM) |
| self.verify_component_view( |
| os.path.join(output_dir, 'component_view_index.html')) |
| self.verify_directory_view( |
| os.path.join(output_dir, 'directory_view_index.html')) |
| self.verify_file_view(os.path.join(output_dir, 'file_view_index.html')) |
| |
| # Verify that the file view pages are binary equal. |
| report_1_file_view_data = _ReadFile( |
| os.path.join(self.REPORT_DIR_1, self.PLATFORM, 'file_view_index.html')) |
| report_2_file_view_data = _ReadFile( |
| os.path.join(self.REPORT_DIR_2, self.PLATFORM, 'file_view_index.html')) |
| self.assertEqual(report_1_file_view_data, report_2_file_view_data) |
| |
| # Testcase #3, run coverage_utils.py on manually produced report and summary |
| # file. This is the workflow of OSS-Fuzz code coverage job. |
| objects = [ |
| '-object=%s' % os.path.join(self.BUILD_DIR, 'crypto_unittests'), |
| '-object=%s' % os.path.join(self.BUILD_DIR, 'libpng_read_fuzzer'), |
| ] |
| |
| cmd = [ |
| self.PYTHON, |
| self.COVERAGE_UTILS, |
| '-v', |
| 'shared_libs', |
| '-build-dir=%s' % self.BUILD_DIR, |
| ] + objects |
| |
| shared_libraries = self.run_cmd(cmd) |
| objects.extend(shared_libraries.split()) |
| |
| instr_profile_path = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, |
| self.PLATFORM, 'coverage.profdata') |
| cmd = [ |
| self.LLVM_COV, |
| 'show', |
| '-format=html', |
| '-output-dir=%s' % self.REPORT_DIR_3, |
| '-instr-profile=%s' % instr_profile_path, |
| ] + objects |
| if self.PLATFORM in ['linux', 'mac']: |
| cmd.extend(['-Xdemangler', 'c++filt', '-Xdemangler', '-n']) |
| self.run_cmd(cmd) |
| |
| cmd = [ |
| self.LLVM_COV, |
| 'export', |
| '-summary-only', |
| '-instr-profile=%s' % instr_profile_path, |
| ] + objects |
| summary_output = self.run_cmd(cmd) |
| |
| summary_path = os.path.join(self.REPORT_DIR_3, 'summary.json') |
| with open(summary_path, 'wb') as f: |
| f.write(summary_output) |
| |
| cmd = [ |
| self.PYTHON, |
| self.COVERAGE_UTILS, |
| '-v', |
| 'post_process', |
| '-src-root-dir=%s' % self.CHROMIUM_SRC_DIR, |
| '-summary-file=%s' % summary_path, |
| '-output-dir=%s' % self.REPORT_DIR_3, |
| ] |
| self.run_cmd(cmd) |
| |
| output_dir = os.path.join(self.REPORT_DIR_3, self.PLATFORM) |
| self.verify_directory_view( |
| os.path.join(output_dir, 'directory_view_index.html')) |
| self.verify_file_view(os.path.join(output_dir, 'file_view_index.html')) |
| self.assertFalse( |
| os.path.exists(os.path.join(output_dir, 'component_view_index.html'))) |
| |
| # Verify that the file view pages are binary equal. |
| report_1_file_view_data_no_component = _ReadFile( |
| os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM, |
| 'file_view_index.html')) |
| report_3_file_view_data = _ReadFile( |
| os.path.join(self.REPORT_DIR_3, self.PLATFORM, 'file_view_index.html')) |
| self.assertEqual(report_1_file_view_data_no_component, |
| report_3_file_view_data) |
| |
| # Testcase 4. Export coverage data in lcov format using coverage.py script. |
| cmd = [ |
| self.COVERAGE_SCRIPT, |
| 'crypto_unittests', |
| 'libpng_read_fuzzer', |
| '--format', |
| 'lcov', |
| '-v', |
| '-b', |
| self.BUILD_DIR, |
| '-o', |
| self.REPORT_DIR_4, |
| '-c' |
| '%s/crypto_unittests' % self.BUILD_DIR, |
| '-c', |
| '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR, |
| ] |
| self.run_cmd(cmd) |
| |
| output_dir = os.path.join(self.REPORT_DIR_4, self.PLATFORM) |
| self.verify_lcov_file(os.path.join(output_dir, 'coverage.lcov')) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
