build/fuchsia/gen_build_defs.py

#!/usr/bin/env vpython3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Generates a single BUILD.gn file with build targets generated using the
# manifest files in the SDK.

# TODO(b/40935282): Likely this file should belong to the
# //third_party/fuchsia-gn-sdk/ instead of //build/fuchsia/.

import json
import logging
import os
import subprocess
import sys

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             'test')))

from common import DIR_SRC_ROOT, SDK_ROOT, GN_SDK_ROOT, get_host_os

assert GN_SDK_ROOT.startswith(DIR_SRC_ROOT)
assert GN_SDK_ROOT[-1] != '/'
GN_SDK_GN_ROOT = GN_SDK_ROOT[len(DIR_SRC_ROOT):]
assert GN_SDK_GN_ROOT.startswith('/')

# Inserted at the top of the generated BUILD.gn file.
_GENERATED_PREAMBLE = f"""# DO NOT EDIT! This file was generated by
# //build/fuchsia/gen_build_def.py.
# Any changes made to this file will be discarded.

import("/{GN_SDK_GN_ROOT}/fidl_library.gni")
import("/{GN_SDK_GN_ROOT}/fuchsia_sdk_package.gni")
import("/{GN_SDK_GN_ROOT}/fuchsia_sdk_pkg.gni")

"""


def ReformatTargetName(dep_name):
  """"Substitutes characters in |dep_name| which are not valid in GN target
  names (e.g. dots become hyphens)."""
  return dep_name


def FormatGNTarget(fields):
  """Returns a GN target definition as a string.

  |fields|: The GN fields to include in the target body.
            'target_name' and 'type' are mandatory."""

  output = '%s("%s") {\n' % (fields['type'], fields['target_name'])
  del fields['target_name']
  del fields['type']

  # Ensure that fields with no ordering requirement are sorted.
  for field in ['sources', 'public_deps']:
    if field in fields:
      fields[field].sort()

  for key, val in fields.items():
    if isinstance(val, str):
      val_serialized = '\"%s\"' % val
    elif isinstance(val, list):
      # Serialize a list of strings in the prettiest possible manner.
      if len(val) == 0:
        val_serialized = '[]'
      elif len(val) == 1:
        val_serialized = '[ \"%s\" ]' % val[0]
      else:
        val_serialized = '[\n    ' + ',\n    '.join(['\"%s\"' % x
                                                     for x in val]) + '\n  ]'
    else:
      raise Exception('Could not serialize %r' % val)

    output += '  %s = %s\n' % (key, val_serialized)
  output += '}'

  return output


def MetaRootRelativePaths(sdk_relative_paths, meta_root):
  return [os.path.relpath(path, meta_root) for path in sdk_relative_paths]


def _FindReadelfPath() -> str:
  """Define the path of the readelf tool."""
  if os.environ.get('FUCHSIA_READELF'):
    return os.environ['FUCHSIA_READELF']
  return os.path.join(DIR_SRC_ROOT, 'third_party', 'llvm-build',
                      'Release+Asserts', 'bin', 'llvm-readelf')


def GetGnuBuildId(elf_file: str, readelf_path: str) -> str:
  """Extracts the GNU build ID from an ELF64 file.

    Args:
        elf_file: Path to input file.
    Returns:
        The build-id value has an hexadecimal string, or
        an empty string on failure (e.g. not an ELF file,
        or no .note.gnu.build-id section in it).
    """
  ret = subprocess.run([readelf_path, "-n", elf_file],
                       text=True,
                       capture_output=True)
  if ret.returncode == 0:
    for line in ret.stdout.splitlines():
      _, prefix, build_id = line.partition("Build ID:")
      if prefix:
        return build_id.strip()

  return ""


def ConvertCommonFields(json):
  """Extracts fields from JSON manifest data which are used across all
  target types. Note that FIDL packages do their own processing."""

  meta_root = json['root']

  converted = {'target_name': ReformatTargetName(json['name'])}

  if 'deps' in json:
    converted['public_deps'] = MetaRootRelativePaths(json['deps'],
                                                     os.path.dirname(meta_root))

  # FIDL bindings dependencies are relative to the "fidl" sub-directory.
  if 'fidl_binding_deps' in json:
    for entry in json['fidl_binding_deps']:
      converted['public_deps'] += MetaRootRelativePaths([
          'fidl/' + dep + ':' + os.path.basename(dep) + '_' +
          entry['binding_type'] for dep in entry['deps']
      ], meta_root)

  return converted


def ConvertFidlLibrary(json):
  """Converts a fidl_library manifest entry to a GN target.

  Arguments:
    json: The parsed manifest JSON.
  Returns:
    The GN target definition, represented as a string."""

  meta_root = json['root']

  converted = ConvertCommonFields(json)
  converted['type'] = 'fidl_library'
  converted['sources'] = MetaRootRelativePaths(json['sources'], meta_root)
  converted['library_name'] = json['name']

  return converted


def ConvertCcPrebuiltLibrary(json):
  """Converts a cc_prebuilt_library manifest entry to a GN target.

  Arguments:
    json: The parsed manifest JSON.
  Returns:
    The GN target definition, represented as a string."""

  meta_root = json['root']

  converted = ConvertCommonFields(json)
  converted['type'] = 'fuchsia_sdk_pkg'

  converted['sources'] = MetaRootRelativePaths(json['headers'], meta_root)

  converted['include_dirs'] = MetaRootRelativePaths([json['include_dir']],
                                                    meta_root)

  if json['format'] == 'shared':
    converted['shared_libs'] = [json['name']]
  else:
    converted['static_libs'] = [json['name']]

  return converted


def ConvertCcSourceLibrary(json):
  """Converts a cc_source_library manifest entry to a GN target.

  Arguments:
    json: The parsed manifest JSON.
  Returns:
    The GN target definition, represented as a string."""

  meta_root = json['root']

  converted = ConvertCommonFields(json)
  converted['type'] = 'fuchsia_sdk_pkg'

  # Headers and source file paths can be scattered across "sources", "headers",
  # and "files". Merge them together into one source list.
  converted['sources'] = MetaRootRelativePaths(json['sources'], meta_root)
  if 'headers' in json:
    converted['sources'] += MetaRootRelativePaths(json['headers'], meta_root)
  if 'files' in json:
    converted['sources'] += MetaRootRelativePaths(json['files'], meta_root)
  converted['sources'] = list(set(converted['sources']))

  converted['include_dirs'] = MetaRootRelativePaths([json['include_dir']],
                                                    meta_root)

  return converted


def ConvertLoadableModule(json):
  """Converts a loadable module manifest entry to GN targets.

  Arguments:
    json: The parsed manifest JSON.
  Returns:
    A list of GN target definitions."""

  name = json['name']
  if name != 'vulkan_layers':
    raise RuntimeError('Unsupported loadable_module: %s' % name)

  # Copy resources and binaries
  resources = json['resources']

  binaries = json['binaries']

  def _filename_no_ext(name):
    return os.path.splitext(os.path.basename(name))[0]

  # Pair each json resource with its corresponding binary. Each such pair
  # is a "layer". We only need to check one arch because each arch has the
  # same list of binaries.
  arch = next(iter(binaries))
  binary_names = binaries[arch]
  local_pkg = json['root']
  vulkan_targets = []

  for res in resources:
    layer_name = _filename_no_ext(res)

    # Filter binaries for a matching name.
    filtered = [n for n in binary_names if _filename_no_ext(n) == layer_name]

    if not filtered:
      # If the binary could not be found then do not generate a
      # target for this layer. The missing targets will cause a
      # mismatch with the "golden" outputs.
      continue

    # Replace hardcoded arch in the found binary filename.
    binary = filtered[0].replace('/' + arch + '/', '/${target_cpu}/')

    target = {}
    target['name'] = layer_name
    target['config'] = os.path.relpath(res, start=local_pkg)
    target['binary'] = os.path.relpath(binary, start=local_pkg)

    vulkan_targets.append(target)

  converted = []
  all_target = {}
  all_target['target_name'] = 'all'
  all_target['type'] = 'group'
  all_target['data_deps'] = []
  for target in vulkan_targets:
    config_target = {}
    config_target['target_name'] = target['name'] + '_config'
    config_target['type'] = 'copy'
    config_target['sources'] = [target['config']]
    config_target['outputs'] = ['${root_gen_dir}/' + target['config']]
    converted.append(config_target)
    lib_target = {}
    lib_target['target_name'] = target['name'] + '_lib'
    lib_target['type'] = 'copy'
    lib_target['sources'] = [target['binary']]
    lib_target['outputs'] = ['${root_out_dir}/lib/{{source_file_part}}']
    converted.append(lib_target)
    group_target = {}
    group_target['target_name'] = target['name']
    group_target['type'] = 'group'
    group_target['data_deps'] = [
        ':' + target['name'] + '_config', ':' + target['name'] + '_lib'
    ]
    converted.append(group_target)
    all_target['data_deps'].append(':' + target['name'])
  converted.append(all_target)
  return converted


def ConvertPackage(json):
  """Converts a package manifest entry to a GN target.

  Arguments:
    json: The parsed manifest JSON.
  Returns:
    The GN target definition."""

  converted = {
      'target_name': ReformatTargetName(json['name']),
      'type': 'fuchsia_sdk_package',
  }

  # Extrapolate the manifest_file's path from the first variant, assuming that
  # they all follow the same format.
  variant = json['variants'][0]
  replace_pattern = '/%s-api-%s/' % (variant['arch'], variant['api_level'])
  segments = variant['manifest_file'].split(replace_pattern)
  if len(segments) != 2:
    raise RuntimeError('Unsupported pattern: %s' % variant['manifest_file'])
  converted['manifest_file'] = \
      '/${target_cpu}-api-${fuchsia_target_api_level}/'.join(segments)

  return converted


def ConvertNoOp(*_):
  """Null implementation of a conversion function. No output is generated."""

  return None


# Maps manifest types to conversion functions.
_CONVERSION_FUNCTION_MAP = {
    'fidl_library': ConvertFidlLibrary,
    'cc_source_library': ConvertCcSourceLibrary,
    'cc_prebuilt_library': ConvertCcPrebuiltLibrary,
    'loadable_module': ConvertLoadableModule,
    'package': ConvertPackage,

    # No need to build targets for these types yet.
    'bind_library': ConvertNoOp,
    'companion_host_tool': ConvertNoOp,
    'component_manifest': ConvertNoOp,
    'config': ConvertNoOp,
    'dart_library': ConvertNoOp,
    'data': ConvertNoOp,
    'device_profile': ConvertNoOp,
    'documentation': ConvertNoOp,
    'experimental_python_e2e_test': ConvertNoOp,
    'ffx_tool': ConvertNoOp,
    'host_tool': ConvertNoOp,
    'image': ConvertNoOp,
    'sysroot': ConvertNoOp,
}


def ConvertMeta(meta_path):
  parsed = json.load(open(meta_path))
  if 'type' not in parsed:
    return

  convert_function = _CONVERSION_FUNCTION_MAP.get(parsed['type'])
  if convert_function is None:
    logging.warning('Unexpected SDK artifact type %s in %s.' %
                    (parsed['type'], meta_path))
    return

  converted = convert_function(parsed)
  if not converted:
    return
  output_path = os.path.join(os.path.dirname(meta_path), 'BUILD.gn')
  if os.path.exists(output_path):
    os.unlink(output_path)
  with open(output_path, 'w') as buildfile:
    buildfile.write(_GENERATED_PREAMBLE)

    # Loadable modules have multiple targets
    if convert_function != ConvertLoadableModule:
      buildfile.write(FormatGNTarget(converted) + '\n\n')
    else:
      for target in converted:
        buildfile.write(FormatGNTarget(target) + '\n\n')


def PopulateBuildIdDirectory(toplevel_meta):
  """Populate the SDK_ROOT/.build-id directory with symlinks.

  Future versions of the IDK will no longer place debug symbols
  in the top-level .build-id/ directory directly. Instead their
  location is available by parsing the meta.json files of various
  prebuilt atom types.

  This function supports both the existing and future layout,
  by only creating entries under SDK_ROOT/.build_id for
  GNU build ID values that are not already listed here.

  Note that this script is run as a DEPS hook and has no knowledge
  of the current target_cpu value or Fuchsia API level, so symlinks
  for all possible debug symbols will be created.
  """
  readelf_path = _FindReadelfPath()

  # First, collect all debug symbols location from the manifest
  # that are not already in the top-level .build-id directory.

  # Map a Build ID hex value to the corresponding debug symbol file
  # path, relative to the SDK root.
  build_ids_map = {}

  def probe_path(debug):
    if debug and not debug.startswith(".build-id/"):
      debug_path = os.path.join(SDK_ROOT, debug)
      build_id = GetGnuBuildId(debug_path, readelf_path)
      assert build_id, (
          f"Could not extract GNU Build ID from debug symbol path: {debug_path}"
      )
      build_ids_map[build_id] = debug

  def parse_sysroot(meta_json):
    # Debug symbols for the HEAD API level, if available, are in
    # binaries.$ARCH.debug_libs
    for arch, values in meta_json.get("binaries", {}).items():
      for debug in values.get("debug_libs", []):
        probe_path(debug)

    # Debug symbols for specific (ARCH, API_LEVEL) are in
    # variants[$INDEX].values.debug_libs, where
    # variants[$INDEX].constraints.{arch, api_level} match
    # (ARCH, API_LEVEL).
    for variant in meta_json.get("variants", []):
      for debug in variant["values"].get("debug_libs", []):
        probe_path(debug)

  def parse_cc_prebuilt_library(meta_json):
    # Only prebuilt shared libraries have Build ID values.
    if meta_json["format"] != "shared":
      return

    # Debug symbols for HEAD API level  are in binaries.$ARCH.debug
    for arch, values in meta_json.get("binaries", {}).items():
      probe_path(values.get("debug"))

    # Debug symbols for specific (ARCH, API_LEVEL) are in
    # variants[$INDEX].values.debug, where
    # variants[$INDEX].constraints.{arch, api_leve} match
    # (ARCH, API_LEVEL)
    for variant in meta_json.get("variants", []):
      debug = variant["values"].get("debug")
      probe_path(debug)

  def parse_loadable_module(meta_json):
    # https://issues.fuchsia.dev/407488427: There are currently
    # no debug symbols for loadable_module atoms. Implement
    # the function properly once this bug is fixed.
    pass

  type_to_parser = {
      "sysroot": parse_sysroot,
      "cc_prebuilt_library": parse_cc_prebuilt_library,
      "loadable_module": parse_loadable_module,
  }

  for part in toplevel_meta["parts"]:
    meta_path = os.path.join(SDK_ROOT, part["meta"])
    parser = type_to_parser.get(part["type"])
    if parser:
      with open(meta_path, "rb") as f:
        meta_json = json.load(f)
      parser(meta_json)

  # Populate the FUCHSIA_SDK_ROOT/.build-id/ directory with hard-links to the
  # corresponding debug symbols
  build_id_dir = os.path.join(SDK_ROOT, ".build-id")
  for build_id, debug_file in build_ids_map.items():
    link_path = os.path.join(build_id_dir, build_id[0:2],
                             f"{build_id[2:]}.debug")

    # https://issues.chromium.org/issues/407890258
    # Remove symlinks in favor of hard-links to avoid incremental flakiness.
    if os.path.islink(link_path):
      os.remove(link_path)

    if os.path.exists(link_path):
      continue  # File already exists, ignore.

    link_target = os.path.join(SDK_ROOT, debug_file)
    link_dir = os.path.dirname(link_path)
    os.makedirs(link_dir, exist_ok=True)
    os.link(link_target, link_path)


def ProcessSdkManifest():
  toplevel_meta = json.load(
      open(os.path.join(SDK_ROOT, 'meta', 'manifest.json')))

  for part in toplevel_meta['parts']:
    meta_path = os.path.join(SDK_ROOT, part['meta'])
    ConvertMeta(meta_path)

  PopulateBuildIdDirectory(toplevel_meta)


def main():
  # Exit if there's no Fuchsia support for this platform.
  try:
    get_host_os()
  except:
    logging.warning('Fuchsia SDK is not supported on this platform.')
    return 0

  # TODO(crbug.com/42050591): Remove this when links to these files inside the
  # sdk directory have been redirected.
  build_path = os.path.join(SDK_ROOT, 'build')
  os.makedirs(build_path, exist_ok=True)
  for gn_file in ['component.gni', 'package.gni']:
    open(os.path.join(build_path, gn_file),
         'w').write("""# DO NOT EDIT! This file was generated by
# //build/fuchsia/gen_build_def.py.
# Any changes made to this file will be discarded.

import("/%s/%s")
      """ % (GN_SDK_GN_ROOT, gn_file))

  ProcessSdkManifest()


if __name__ == '__main__':
  sys.exit(main())