tools/codeql/index_target.py

# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script for building a Chromium CodeQL database."""

import argparse
import functools
import json
import multiprocessing
import subprocess
import logging
import time
import os
import traceback
import gn_sources_tools
import targets_to_index

from collections import namedtuple

CommandOutput = namedtuple("CommandOutput", "output traceback")


def log_subprocess_output(output, logger=None):
  """Reads from a subprocess's stdout and writes it to `logger`,
  or, if none given, to stdout."""
  if not logger:
    print(output)
  else:
    logger.info(output)


class CodeQLDatabase:
  def __init__(self, src_path, db_path, codeql_binary_path):
    """Construct a new `CodeQLDatabase` object.
    :param src_path: The path to the chromium/src tree.
    :param db_path: The path where the CodeQL database will be created.
    :return: returns nothing
    """
    self.db_path = db_path
    try:
      process_stdout = subprocess.check_output(
        [
          codeql_binary_path,
          'database',
          'init',
          f'--source-root={src_path}',
          '--language=cpp',
          db_path,
          '--overwrite',
        ]
      )
      log_subprocess_output(process_stdout)
    except subprocess.CalledProcessError:
      # Presumably failed due to an invalid value for db_path.
      raise ValueError


def index_one_target(
  target_name,
  src_path,
  db_path,
  codeql_binary_path,
  out_path,
  logger,
  ninja_path='ninja',
  gn_path='gn',
  logfile=None,
  reduce_cores_used=False,
):
  try:
    process_stdout = subprocess.check_output([gn_path, 'clean', out_path])
    log_subprocess_output(process_stdout)
  except subprocess.CalledProcessError as e:
    print("Failed to clean build directory between targets")
    print("stdout: %s" % e.stdout)
    print("stderr: %s" % e.stderr)
    exit(1)
  db_path = os.path.join(db_path, target_name)
  os.mkdir(os.path.join(db_path))

  start_time = time.time()

  print("Initializing codeql.")
  codeql_db = ""
  try:
    codeql_db = CodeQLDatabase(src_path, db_path, codeql_binary_path)
  except ValueError:
    print("Could not initialize CodeQL database at %s" % db_path)
    exit(1)

  print("Tracing compilation.")
  trace_command = [
    codeql_binary_path,
    'database',
    'trace-command',
    db_path,
    f'--working-dir={src_path}',
    '--',
    ninja_path,
    '-C',
    out_path,
    target_name,
  ]
  if reduce_cores_used:
    usable_cpu_count = int(multiprocessing.cpu_count() / 2)
    trace_command.extend(['-j', str(usable_cpu_count)])
  try:
    process_stdout = subprocess.check_output(trace_command)
    log_subprocess_output(process_stdout)
  except subprocess.CalledProcessError as e:
    print("CodeQL trace-process failed with return code %s" % e.returncode)
    print("stdout: %s" % e.stdout)
    print("stderr: %s" % e.stderr)
    exit(1)

  print("Finalizing codeql db.")
  try:
    process_stdout = subprocess.check_output(
      [codeql_binary_path, 'database', 'finalize', '-j=-1', db_path]
    )
    log_subprocess_output(process_stdout)
  except subprocess.CalledProcessError as e:
    print("CodeQL DB finalization failed with return code %s" % e.returncode)
    print("stdout: %s" % e.stdout)
    print("stderr: %s" % e.stderr)
  print("Database creation complete.")
  total_time = time.time() - start_time
  print("Time elapsed:")
  print(str(total_time))


def main():
  logger = logging.getLogger('log')
  logger.setLevel(logging.INFO)
  actual_cwd = os.getcwd()
  script_directory = os.path.dirname(os.path.realpath(__file__))
  src_path = os.path.join(script_directory, '..', '..')
  if actual_cwd != os.path.normpath(src_path):
    print("Failure: Script must be executed from `chromium/src`. Exiting.")
    print(actual_cwd)
    print(src_path)
    exit(1)

  print("Parsing command line arguments.")
  parser = argparse.ArgumentParser(
    description='Build CodeQL database for Chromium browser process'
  )
  parser.add_argument(
    '--out_path',
    '-o',
    type=str,
    default='out/release',
    help='Relative path inside chromium checkout to build directory',
  )
  parser.add_argument(
    '--db_path', '-d', type=str, required=True, help='Path to output database'
  )
  parser.add_argument(
    '--logfile',
    '-l',
    type=str,
    help="absolute path to logfile for `trace` calls, if desired",
  )
  parser.add_argument(
    '--gn_targets',
    '-g',
    action='append',
    type=str,
    help=(
      'name for the specific GN target you want a CodeQL database for '
      '(e.g. `//components:components_unittests`); if left blank, indexes '
      'everything'
    ),
  )
  parser.add_argument(
    '--codeql_binary_path',
    '-c',
    type=str,
    default='codeql',
    help=(
      'Path to the codeql binary. If this is not set, the script assumes '
      'it is located at `codeql` somewhere in the user\'s PATH.'
    ),
  )
  parser.add_argument(
    '--gn_path',
    type=str,
    default='gn',
    help=(
      'Path to the gn executable. If this is not set, the script assumes '
      'it is located at `gn` somehwere in the user\'s PATH.'
    ),
  )
  parser.add_argument(
    '--ninja_path',
    type=str,
    default='ninja',
    help=(
      'Path to the ninja executable. If this is not set, the script '
      'assumes it is located at `ninja` somehwere in the user\'s PATH.'
    ),
  )
  parser.add_argument(
    '--reduce_cores_used',
    default=False,
    action='store_true',
    help=('If set, reduces the number of cores used when building a target.'),
  )
  args = parser.parse_args()

  if args.logfile:
    ch = logging.FileHandler(args.logfile)
    ch.setFormatter(logging.Formatter('%(message)s'))
    logger.addHandler(ch)
  src_path = os.path.abspath(os.path.expanduser(src_path))
  args.db_path = os.path.abspath(os.path.expanduser(args.db_path))

  # If an args.gn_target is given, index those targets.
  # Otherwise, index the targets in targets_to_index.
  actual_targets_to_index = []
  if not args.gn_targets:
    actual_targets_to_index = targets_to_index.full_targets
  else:
    actual_targets_to_index = args.gn_targets
  for target in actual_targets_to_index:
    index_one_target(
      target,
      src_path,
      args.db_path,
      args.codeql_binary_path,
      args.out_path,
      logger,
      args.ninja_path,
      args.gn_path,
      args.logfile,
      args.reduce_cores_used,
    )


if __name__ == '__main__':
  main()