build/modules/modularize/modularize.py

#!/usr/bin/env python

# Copyright 2025 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Modularize modularizes a platform."""

import argparse
import concurrent.futures
import logging
import pathlib
import re
import subprocess
import sys
import traceback

from compiler import Compiler
from config import fix_graph
from graph import IncludeDir
from graph import run_build
from platforms import Cpu
from platforms import Os
import render

SOURCE_ROOT = pathlib.Path(__file__).parents[3].resolve()
_OS_OPTS = '|'.join(os.value for os in Os)
_CPU_OPTS = '|'.join(cpu.value for cpu in Cpu)
_PLATFORM = re.compile('^' + ''.join([
    f'(?P<os>{_OS_OPTS})',
    f'(?:-(?P<cpu>{_CPU_OPTS}))?',
    r'(?P<xcode_suffix>_xcode\d+)?',
]) + '$')


def main(args):
  logging.basicConfig(level=logging.getLevelNamesMapping()[args.verbosity])
  existing_platforms = [
      f.name for f in (SOURCE_ROOT / 'build/modules').iterdir()
  ]

  filter = {
      'os': args.os,
      'cpu': args.cpu,
  }

  calls = {}
  for platform in existing_platforms:
    match = _PLATFORM.match(platform)
    if match is None:
      continue

    props = match.groupdict()
    os = Os(props['os'])
    # If no CPU is provided, we support all CPUs with a single BUILD.gn.
    # But we do need to be consistent about the CPU we target, so we arbitrarily
    # pick x64 since it's probably the most supported CPU.
    cpu = Cpu(props['cpu'] or 'x64')

    # We want to reconstruct the platform to dedupe xcode versions, and ensure
    # that we always have the correct cpu arch.
    platform = f"{os.value}-{cpu.value}"

    skip = False
    for k, v in filter.items():
      if v is not None and props[k] != v:
        skip = True
    if skip:
      continue
    out_dir = SOURCE_ROOT / 'out' / platform
    args_gn = out_dir / 'args.gn'
    if not args_gn.exists():
      out_dir.mkdir(exist_ok=True, parents=True)
      args_gn.write_text('\n'.join([
          # These are required
          f'target_os = "{os.value}"',
          f'target_cpu = "{cpu.value}"',
          # This is required (see README.md)
          'use_clang_modules = true',
          # This is strongly recommended, otherwise you may accidentally
          # test manual modules.
          'use_autogenerated_modules = true',
          # Some useful defaults. User is free to change these.
          'use_remoteexec = true',
          'symbol_level = 0',
          'is_debug = false',
          'running_modularize = true',
      ]))

    error_log = None if args.error_log is None else args.error_log / platform
    calls[platform] = dict(
        out_dir=out_dir,
        error_log=error_log,
        use_cache=args.cache,
        compile=args.compile,
        os=os,
        cpu=cpu,
    )

  if not calls:
    print('No matching platforms. Try copying an existing one', file=sys.stderr)
  elif len(calls) == 1:
    _modularize(**next(iter(calls.values())))
    return

  # Use a ProcessPoolExecutor rather than a ThreadPoolExecutor because:
  # * No shared state between instances
  # * GIL prevents a performance benefit from a thread pool executor.
  with concurrent.futures.ProcessPoolExecutor() as executor:
    futures = {
        k: executor.submit(_modularize, **kwargs)
        for k, kwargs in calls.items()
    }

  success = True
  for platform, future in sorted(futures.items()):
    exc = future.exception()
    if exc is not None:
      success = False
      print(f'{platform} raised an exception:', file=sys.stderr)
      traceback.print_exception(exc)
  if not success:
    exit(1)


def _modularize(out_dir: pathlib.Path, error_log: pathlib.Path | None,
                use_cache: bool, compile: bool, cpu: Cpu, os: Os):
  # Modularize requires gn gen to have been run at least once.
  if not (out_dir / 'build.ninja').is_file():
    subprocess.run(['gn', 'gen', out_dir], check=True)
  compiler = Compiler(
      source_root=SOURCE_ROOT,
      gn_out=out_dir,
      error_dir=error_log,
      use_cache=use_cache,
      cpu=cpu,
      os=os,
  )

  if compile:
    ps, files = compiler.compile_one(compile)
    print('stderr:', ps.stderr.decode('utf-8'), file=sys.stderr)
    print('Files used:')
    print('\n'.join(sorted(map(str, files))))
    print('Setting breakpoint to allow further debugging')
    breakpoint()
    return

  graph = compiler.compile_all()
  replacements = fix_graph(graph, compiler)
  targets = run_build(graph)

  platform = (out_dir / 'gen/module_platform.txt').read_text()
  logging.info('Detected platform %s', platform)

  out_dir = SOURCE_ROOT / 'build/modules' / platform
  out_dir.mkdir(exist_ok=True, parents=False)
  if compiler.sysroot_dir == IncludeDir.Sysroot:
    render.render_modulemap(out_dir=out_dir,
                            replacements=replacements,
                            targets=targets)
  render.render_build_gn(
      out_dir=out_dir,
      targets=targets,
      compiler=compiler,
  )


def _optional_path(s: str) -> pathlib.Path | None:
  if s:
    return pathlib.Path(s).resolve()


if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--cpu',
      help='Only update platforms matching this cpu',
      type=lambda x: None if x is None else Cpu(x),
      default=None,
      choices=[cpu.value for cpu in Cpu],
  )
  parser.add_argument(
      '--os',
      help='Only update platforms matching this cpu',
      type=lambda x: None if x is None else Os(x),
      default=None,
      choices=[os.value for os in Os],
  )

  # Make it required so the user understands how compilation works.
  cache = parser.add_mutually_exclusive_group(required=True)
  cache.add_argument(
      '--cache',
      action='store_true',
      help='Enable caching. Will attempt to reuse the compilation results.')
  cache.add_argument(
      '--no-cache',
      action='store_false',
      dest='cache',
      help='Disable caching. Will attempt to recompile the whole libcxx, ' +
      'builtins, and sysroot on every invocation')

  parser.add_argument(
      '--compile',
      help='Compile a single header file (eg. --compile=sys/types.h) instead ' +
      'of the whole sysroot. Useful for debugging.',
  )

  parser.add_argument('--error-log', type=_optional_path)

  parser.add_argument(
      '--verbosity',
      help='Verbosity of logging',
      default='INFO',
      choices=logging.getLevelNamesMapping().keys(),
      type=lambda x: x.upper(),
  )

  main(parser.parse_args())