testing/libfuzzer/confirm_fuzztests.py

#!/usr/bin/env python3
#
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""List the fuzzers within a fuzztest and confirm they match what we expect.

Invoked by GN from fuzzer_test.gni.
"""

import argparse
import os
import re
import subprocess
import sys

# //build imports.
sys.path.append(
  os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
    os.pardir,
    'build',
  )
)
import action_helpers


def CreateArgumentParser():
  """Creates an argparse.ArgumentParser instance."""
  parser = argparse.ArgumentParser(description='Generate fuzztest fuzzers.')
  parser.add_argument(
    '--executable', help='Executable to interrogate for present fuzztests.'
  )
  parser.add_argument(
    '--output',
    help='Path to the output file (which will be intentionally blank).',
  )
  parser.add_argument(
    '--fuzztests', nargs='+', help='List of fuzztests we expect to find.'
  )
  return parser


def main():
  parser = CreateArgumentParser()
  args = parser.parse_args()

  expected_tests = set(args.fuzztests)

  env = os.environ
  env['ASAN_OPTIONS'] = 'detect_odr_violation=0'
  try:
    process_result = subprocess.run(
      [args.executable, '--list_fuzz_tests=1'],
      env=env,
      stdout=subprocess.PIPE,
      check=False,
    )
  except OSError:
    process_result = None

  if process_result and process_result.returncode == 0:
    test_list = process_result.stdout.decode('utf-8')

    actual_tests = re.findall('Fuzz test: (.*)', test_list)
    # Gardeners may disable fuzztests by changing the names of the fuzztest
    # in code to start with DISABLED_. We don't want gardeners to need to
    # change gn files correspondingly, so strip such prefixes when checking
    # that the test lists match.
    actual_tests = {
      test.strip().replace('DISABLED_', '').replace('FLAKY_', '')
      for test in actual_tests
    }

    if expected_tests != actual_tests:
      print(
        'Unexpected fuzztests found in this binary.\nFuzztest binary: '
        + args.executable
        + '\n'
        + 'Expected fuzztests (as declared by the gn "fuzztests" variable on'
        ' this target): '
        + str(expected_tests)
        + '\nActual tests (as found in the binary): '
        + str(actual_tests)
        + '\nYou probably need to update the gn variable to match the tests'
        ' actually in the binary.'
      )
      sys.exit(-1)

  # If we couldn't run the fuzztest binary itself, we'll
  # regard that as fine. This is a best-efforts check that the
  # gn 'fuzztests' variable is correct, and sometimes fuzzers don't
  # run on LUCI e.g. due to lack of dependencies.

  with action_helpers.atomic_output(args.output) as output:
    output.write(''.encode('utf-8'))


if __name__ == '__main__':
  main()