tools/code_coverage/measure_fuzzilli_coverage.py - chromium/src - Git at Google

 #!/usr/bin/env vpython3
 # Copyright 2026 The Chromium Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Run fuzzilli for the specified duration and generate a coverage report

   * Example usage: vpython3 measure_fuzzilli_coverage.py
                    --build-out-dir ~/chromium/src/out/Fuzzilli
                    --fuzzilli-dir ~/fuzzilli
                    --report-out-dir ~/chromium/src/out/report
                    --minutes 20

     Optionally, use --ignore-filename-regex to provide a regular
     expression matching all the file paths to be excluded from the report.
     Use --filters to explicitly state all the directories or files to get
     include in the coverage report.

     The following gn args, for use with `js_in_process_fuzzer`, are a mix
     of args required for coverage and args that improve fuzzing either by
     speed up or increased coverage:
         use_sanitizer_coverage = true
         is_component_build = false
         is_debug = false
         symbol_level = 2
         blink_symbol_level = 0
         use_remoteexec = true
         dcheck_always_on = false
         is_asan = false
         use_chromium_fuzzilli = true
         v8_fuzzilli = true
         v8_static_library = true
         v8_dcheck_always_on = true
         optimize_for_fuzzing = false
         enable_mojom_fuzzer = true
         use_clang_coverage = true
 """

 import argparse
 import glob
 import logging
 from pathlib import Path
 import os
 import signal
 import subprocess
 import sys
 import tempfile

 DEFAULT_FUZZILLI_FLAGS = [
     '--profile=chromiumMojo', '--storagePath=/tmp/fuzzilli_storage',
     '--overwrite', '--engine=hybrid'
 ]
 SCRIPT_DIR = Path(__file__).resolve().parent
 SRC_DIR = SCRIPT_DIR.parents[1]


 class ProcessGroupHandler:
   """Simple class for handling process groups across the program.

      Notably, this class ensures that the associated process group
      is killed upon interrupt by providing an interrupt_handler.
   """

   def __init__(self):
     self.process_group = None

   def SetProcessGroup(self, pg):
     self.process_group = pg

   def KillProcessGroup(self):
     """Kill the process group with a SIGTERM signal. Grant 10 seconds
        to allow for graceful shutdown, following up with a SIGKILL if
        the process group is still active.
     """

     os.killpg(os.getpgid(self.process_group.pid), signal.SIGTERM)
     try:
       self.process_group.wait(timeout=10)
     except subprocess.TimeoutExpired:
       # Kill the process if still active
       os.killpg(os.getpgid(self.process_group.pid), signal.SIGKILL)

   def interrupt_handler(self, sig, frame):
     """Gracefully handle interrupt by killing children processes.

        When running Fuzzilli, many child processes are created. Without
        killing the entire process groups, these processes become orphan
        processes.
     """

     self.KillProcessGroup()
     sys.exit(0)


 def _ParseCommandArguments():
   """Adds and parses relevant arguments for tool commands.

   Returns:
     An argparse.Namespace object representing the arguments.
   """
   arg_parser = argparse.ArgumentParser()
   arg_parser.usage = __doc__

   arg_parser.add_argument(
       '-fd',
       '--fuzzilli-dir',
       type=str,
       required=True,
       help='The absolute path to the directory containing Fuzzilli.')

   arg_parser.add_argument(
       '-b',
       '--build-out-dir',
       type=str,
       required=True,
       help='The absolute path to the Chrome output directory.')

   arg_parser.add_argument(
       '-r',
       '--report-out-dir',
       type=str,
       required=True,
       help='The absolute path to the directory to which to output the report.')

   arg_parser.add_argument(
       '-m',
       '--minutes',
       type=int,
       required=True,
       help='The number of minutes for which to run Fuzzilli.')

   arg_parser.add_argument(
       '-f',
       '--filters',
       action='append',
       required=False,
       help='Directories or files to get code coverage for, and all files under '
       'the directories are included recursively.')

   arg_parser.add_argument(
       '-i',
       '--ignore-filename-regex',
       type=str,
       help='Skip source code files with file paths that match the given '
       'regular expression. For example, use -i=\'.*/out/.*|.*/third_party/.*\' '
       'to exclude files in third_party/ and out/ folders from the report.')

   args = arg_parser.parse_args()
   return args


 def _RunFuzzilli(pg_handler, fuzzilli_dir, build_out_dir, minutes, profraw_dir):
   """Builds and runs Fuzzilli.

      Bypasses the 'swift run' wrapper to ensure signals are handled correctly.
      Launches a process group in order to kill all associated processes
      after running for the desired duration.
   """

   os.environ["LLVM_PROFILE_FILE"] = \
     os.path.join(profraw_dir, "fuzzilli.%4m%c.profraw")

   build_command = ['swift', 'build', '-c', 'release']
   try:
     subprocess.run(build_command, check=True, cwd=fuzzilli_dir)
   except (subprocess.CalledProcessError, FileNotFoundError) as e:
     logging.fatal(e)
     return False

   # Fuzzilli strips executed programs of any environment variables, setting
   # only the environment variables specified in the optionally provided
   # profile. To ensure that js_in_process_fuzzer can see the environment
   # variable, create a temporary script for Fuzzilli to execute, in which the
   # environment variable is set.
   temp_wrapper = tempfile.NamedTemporaryFile(mode='w+',
                                              delete=False,
                                              suffix='.sh')
   try:
     temp_wrapper.write(
         '#!/bin/bash\n'
         f'export LLVM_PROFILE_FILE="{os.environ["LLVM_PROFILE_FILE"]}"\n'
         f'exec {os.path.join(build_out_dir, "js_in_process_fuzzer")} "$@"')
     temp_wrapper.flush()
     temp_wrapper.close()

     # mark wrapper as executable
     os.chmod(temp_wrapper.name, 0o755)
     print(f"The temporary file is located at: {temp_wrapper.name}")

     fuzzilli_executable = os.path.join(fuzzilli_dir,
                                        '.build/release/FuzzilliCli')
     run_command = [
         fuzzilli_executable, *DEFAULT_FUZZILLI_FLAGS, temp_wrapper.name
     ]

     # Use a process group, as this command will create many child processes that
     # continue to live if only the parent is killed
     p = subprocess.Popen(run_command, cwd=fuzzilli_dir, start_new_session=True)
     pg_handler.SetProcessGroup(p)
     try:
       timeout = minutes * 60
       p.wait(timeout=timeout)
       if p.returncode != 0:
         return False
     except subprocess.TimeoutExpired:
       # Kill the entire process group to ensure no orphan child processes
       pg_handler.KillProcessGroup()
   finally:
     if os.path.exists(temp_wrapper.name):
       os.remove(temp_wrapper.name)

   return True


 def _GenerateCoverageReport(build_out_dir, report_out_dir,
                             ignore_filename_regex, filters, profraw_dir):
   if filters is None:
     filters = []

   try:
     # Create the indexed profile to pass into `coverage.py`
     #
     # glob.glob expands the wildcard
     gen_profile_command = [
         'llvm-profdata', 'merge', '-o',
         os.path.join(profraw_dir, 'coverage.profdata')
     ] + glob.glob(os.path.join(profraw_dir, '*.profraw'))
     subprocess.run(gen_profile_command, check=True)

     coverage_command = [
         f'{SRC_DIR}/tools/code_coverage/coverage.py', 'js_in_process_fuzzer',
         '-b', build_out_dir, '-o', report_out_dir, '-p',
         os.path.join(profraw_dir, 'coverage.profdata'), '--no-component-view'
     ]
     for f in filters:
       coverage_command.extend(['-f', f])
     if ignore_filename_regex:
       coverage_command.extend(['-i', ignore_filename_regex])
     subprocess.run(coverage_command, check=True)
   except FileNotFoundError as e:
     logging.fatal(e)
     return False
   except subprocess.CalledProcessError:
     # Don't log here to avoid duplicate logs (as both llvm-profdata and
     # coverage.py already log).
     return False

   return True


 def Main():
   """Execute script."""

   pg_handler = ProcessGroupHandler()
   signal.signal(signal.SIGINT, pg_handler.interrupt_handler)

   args = _ParseCommandArguments()

   # Create a unique directory for storing the generated *.profraw files
   with tempfile.TemporaryDirectory() as profraw_dir:
     profraw_path = Path(profraw_dir)
     print(f'Using {profraw_path} for profraw files.')

     if not _RunFuzzilli(pg_handler, args.fuzzilli_dir, args.build_out_dir,
                         args.minutes, profraw_path):
       sys.exit('Error: Fuzzilli failed to build or run.')

     if not _GenerateCoverageReport(args.build_out_dir, args.report_out_dir,
                                    args.ignore_filename_regex, args.filters,
                                    profraw_path):
       sys.exit('Error: Failed to generate coverage report.')


 if __name__ == '__main__':
   sys.exit(Main())
	#!/usr/bin/env vpython3
	# Copyright 2026 The Chromium Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Run fuzzilli for the specified duration and generate a coverage report

	* Example usage: vpython3 measure_fuzzilli_coverage.py
	--build-out-dir ~/chromium/src/out/Fuzzilli
	--fuzzilli-dir ~/fuzzilli
	--report-out-dir ~/chromium/src/out/report
	--minutes 20

	Optionally, use --ignore-filename-regex to provide a regular
	expression matching all the file paths to be excluded from the report.
	Use --filters to explicitly state all the directories or files to get
	include in the coverage report.

	The following gn args, for use with `js_in_process_fuzzer`, are a mix
	of args required for coverage and args that improve fuzzing either by
	speed up or increased coverage:
	use_sanitizer_coverage = true
	is_component_build = false
	is_debug = false
	symbol_level = 2
	blink_symbol_level = 0
	use_remoteexec = true
	dcheck_always_on = false
	is_asan = false
	use_chromium_fuzzilli = true
	v8_fuzzilli = true
	v8_static_library = true
	v8_dcheck_always_on = true
	optimize_for_fuzzing = false
	enable_mojom_fuzzer = true
	use_clang_coverage = true
	"""

	import argparse
	import glob
	import logging
	from pathlib import Path
	import os
	import signal
	import subprocess
	import sys
	import tempfile

	DEFAULT_FUZZILLI_FLAGS = [
	'--profile=chromiumMojo', '--storagePath=/tmp/fuzzilli_storage',
	'--overwrite', '--engine=hybrid'
	]
	SCRIPT_DIR = Path(__file__).resolve().parent
	SRC_DIR = SCRIPT_DIR.parents[1]


	class ProcessGroupHandler:
	"""Simple class for handling process groups across the program.

	Notably, this class ensures that the associated process group
	is killed upon interrupt by providing an interrupt_handler.
	"""

	def __init__(self):
	self.process_group = None

	def SetProcessGroup(self, pg):
	self.process_group = pg

	def KillProcessGroup(self):
	"""Kill the process group with a SIGTERM signal. Grant 10 seconds
	to allow for graceful shutdown, following up with a SIGKILL if
	the process group is still active.
	"""

	os.killpg(os.getpgid(self.process_group.pid), signal.SIGTERM)
	try:
	self.process_group.wait(timeout=10)
	except subprocess.TimeoutExpired:
	# Kill the process if still active
	os.killpg(os.getpgid(self.process_group.pid), signal.SIGKILL)

	def interrupt_handler(self, sig, frame):
	"""Gracefully handle interrupt by killing children processes.

	When running Fuzzilli, many child processes are created. Without
	killing the entire process groups, these processes become orphan
	processes.
	"""

	self.KillProcessGroup()
	sys.exit(0)


	def _ParseCommandArguments():
	"""Adds and parses relevant arguments for tool commands.

	Returns:
	An argparse.Namespace object representing the arguments.
	"""
	arg_parser = argparse.ArgumentParser()
	arg_parser.usage = __doc__

	arg_parser.add_argument(
	'-fd',
	'--fuzzilli-dir',
	type=str,
	required=True,
	help='The absolute path to the directory containing Fuzzilli.')

	arg_parser.add_argument(
	'-b',
	'--build-out-dir',
	type=str,
	required=True,
	help='The absolute path to the Chrome output directory.')

	arg_parser.add_argument(
	'-r',
	'--report-out-dir',
	type=str,
	required=True,
	help='The absolute path to the directory to which to output the report.')

	arg_parser.add_argument(
	'-m',
	'--minutes',
	type=int,
	required=True,
	help='The number of minutes for which to run Fuzzilli.')

	arg_parser.add_argument(
	'-f',
	'--filters',
	action='append',
	required=False,
	help='Directories or files to get code coverage for, and all files under '
	'the directories are included recursively.')

	arg_parser.add_argument(
	'-i',
	'--ignore-filename-regex',
	type=str,
	help='Skip source code files with file paths that match the given '
	'regular expression. For example, use -i=\'./out/.\|./third_party/.\' '
	'to exclude files in third_party/ and out/ folders from the report.')

	args = arg_parser.parse_args()
	return args


	def _RunFuzzilli(pg_handler, fuzzilli_dir, build_out_dir, minutes, profraw_dir):
	"""Builds and runs Fuzzilli.

	Bypasses the 'swift run' wrapper to ensure signals are handled correctly.
	Launches a process group in order to kill all associated processes
	after running for the desired duration.
	"""

	os.environ["LLVM_PROFILE_FILE"] = \
	os.path.join(profraw_dir, "fuzzilli.%4m%c.profraw")

	build_command = ['swift', 'build', '-c', 'release']
	try:
	subprocess.run(build_command, check=True, cwd=fuzzilli_dir)
	except (subprocess.CalledProcessError, FileNotFoundError) as e:
	logging.fatal(e)
	return False

	# Fuzzilli strips executed programs of any environment variables, setting
	# only the environment variables specified in the optionally provided
	# profile. To ensure that js_in_process_fuzzer can see the environment
	# variable, create a temporary script for Fuzzilli to execute, in which the
	# environment variable is set.
	temp_wrapper = tempfile.NamedTemporaryFile(mode='w+',
	delete=False,
	suffix='.sh')
	try:
	temp_wrapper.write(
	'#!/bin/bash\n'
	f'export LLVM_PROFILE_FILE="{os.environ["LLVM_PROFILE_FILE"]}"\n'
	f'exec {os.path.join(build_out_dir, "js_in_process_fuzzer")} "$@"')
	temp_wrapper.flush()
	temp_wrapper.close()

	# mark wrapper as executable
	os.chmod(temp_wrapper.name, 0o755)
	print(f"The temporary file is located at: {temp_wrapper.name}")

	fuzzilli_executable = os.path.join(fuzzilli_dir,
	'.build/release/FuzzilliCli')
	run_command = [
	fuzzilli_executable, *DEFAULT_FUZZILLI_FLAGS, temp_wrapper.name
	]

	# Use a process group, as this command will create many child processes that
	# continue to live if only the parent is killed
	p = subprocess.Popen(run_command, cwd=fuzzilli_dir, start_new_session=True)
	pg_handler.SetProcessGroup(p)
	try:
	timeout = minutes * 60
	p.wait(timeout=timeout)
	if p.returncode != 0:
	return False
	except subprocess.TimeoutExpired:
	# Kill the entire process group to ensure no orphan child processes
	pg_handler.KillProcessGroup()
	finally:
	if os.path.exists(temp_wrapper.name):
	os.remove(temp_wrapper.name)

	return True


	def _GenerateCoverageReport(build_out_dir, report_out_dir,
	ignore_filename_regex, filters, profraw_dir):
	if filters is None:
	filters = []

	try:
	# Create the indexed profile to pass into `coverage.py`
	#
	# glob.glob expands the wildcard
	gen_profile_command = [
	'llvm-profdata', 'merge', '-o',
	os.path.join(profraw_dir, 'coverage.profdata')
	] + glob.glob(os.path.join(profraw_dir, '*.profraw'))
	subprocess.run(gen_profile_command, check=True)

	coverage_command = [
	f'{SRC_DIR}/tools/code_coverage/coverage.py', 'js_in_process_fuzzer',
	'-b', build_out_dir, '-o', report_out_dir, '-p',
	os.path.join(profraw_dir, 'coverage.profdata'), '--no-component-view'
	]
	for f in filters:
	coverage_command.extend(['-f', f])
	if ignore_filename_regex:
	coverage_command.extend(['-i', ignore_filename_regex])
	subprocess.run(coverage_command, check=True)
	except FileNotFoundError as e:
	logging.fatal(e)
	return False
	except subprocess.CalledProcessError:
	# Don't log here to avoid duplicate logs (as both llvm-profdata and
	# coverage.py already log).
	return False

	return True


	def Main():
	"""Execute script."""

	pg_handler = ProcessGroupHandler()
	signal.signal(signal.SIGINT, pg_handler.interrupt_handler)

	args = _ParseCommandArguments()

	# Create a unique directory for storing the generated *.profraw files
	with tempfile.TemporaryDirectory() as profraw_dir:
	profraw_path = Path(profraw_dir)
	print(f'Using {profraw_path} for profraw files.')

	if not _RunFuzzilli(pg_handler, args.fuzzilli_dir, args.build_out_dir,
	args.minutes, profraw_path):
	sys.exit('Error: Fuzzilli failed to build or run.')

	if not _GenerateCoverageReport(args.build_out_dir, args.report_out_dir,
	args.ignore_filename_regex, args.filters,
	profraw_path):
	sys.exit('Error: Failed to generate coverage report.')


	if __name__ == '__main__':
	sys.exit(Main())