| # Copyright 2024 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import os |
| import sys |
| import glob |
| import constants |
| import tempfile |
| from typing import Dict, Callable, List, Set |
| from pathlib import Path |
| |
| from license_type import LicenseType |
| import license_utils as license_utils |
| |
| REPOSITORY_ROOT = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) |
| |
| sys.path.insert(0, REPOSITORY_ROOT) |
| import components.cronet.gn2bp.targets as gn2bp_targets # pylint: disable=wrong-import-position |
| import components.cronet.tools.utils as cronet_utils # pylint: disable=wrong-import-position |
| |
| _OUT_DIR = os.path.join(REPOSITORY_ROOT, "out") |
| |
| # The license content of all transitive dependencies of the targets listed here will be added |
| # to the top-level LICENSE. |
| _TARGETS_TO_AGGREGATE_LICENSE_FOR_TRANSITIVELY = ["//components/cronet/android:cronet_non_test_package"] |
| # The license generator will generate license file for all the targets found as a |
| # transitive dependency of this target. |
| # For each transitive dependency with a README.chromium, its license file will be |
| # processed. |
| _TARGETS_TO_PROCESS_LICENSE_FOR_TRANSITIVELY = ( |
| ["//components/cronet/android:cronet_package_android"] + |
| _TARGETS_TO_AGGREGATE_LICENSE_FOR_TRANSITIVELY + |
| gn2bp_targets.DEFAULT_TARGETS + |
| gn2bp_targets.DEFAULT_TESTS) |
| |
| METADATA_HEADER = """# This was automatically generated by {} |
| # This directory was imported from Chromium.""".format( |
| os.path.basename(__file__)) |
| |
| _ROOT_CRONET = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, |
| os.path.pardir)) |
| |
| |
| def _create_metadata_file(repo_path: str, directory_path: str, content: str, |
| verify_only: bool): |
| """Creates a METADATA file with a header to ensure that this was generated |
| through the script. If the header is not found then it is assumed that the |
| METADATA file is created manually and will not be touched.""" |
| metadata = Path(os.path.join(directory_path, "METADATA")) |
| if metadata.is_file() and METADATA_HEADER not in metadata.read_text(): |
| # This is a manually created file! Don't overwrite. |
| return |
| |
| metadata_content = "\n".join([ |
| METADATA_HEADER, |
| content |
| ]) |
| if verify_only: |
| if not metadata.exists(): |
| raise Exception( |
| f"Failed to find metadata file {metadata.relative_to(repo_path)}") |
| if not metadata.read_text() == metadata_content: |
| raise Exception( |
| f"Metadata content of {metadata.relative_to(repo_path)} does not match the expected." |
| f"Please re-run create_android_metadata_license.py") |
| else: |
| metadata.write_text(metadata_content) |
| |
| |
| def _create_module_license_file(repo_path: str, directory_path: str, |
| licenses: List[str], |
| verify_only: bool): |
| """Creates a MODULE_LICENSE_XYZ files.""" |
| for license in licenses: |
| license_file = Path(os.path.join(directory_path, |
| f"MODULE_LICENSE_{license_utils.get_license_file_format(license)}")) |
| if verify_only: |
| if not license_file.exists(): |
| raise Exception( |
| f"Failed to find module file {license_file.relative_to(repo_path)}") |
| else: |
| license_file.touch() |
| |
| |
| def _maybe_create_license_file_symlink(directory_path: str, |
| original_license_file: str, |
| verify_only: bool): |
| """Creates a LICENSE symbolic link only if it doesn't exist.""" |
| license_symlink_path = Path(os.path.join(directory_path, "LICENSE")) |
| if license_symlink_path.exists(): |
| # The symlink is already there, skip. |
| return |
| |
| if verify_only: |
| if not license_symlink_path.exists(): |
| raise Exception( |
| f"License symlink does not exist for {license_symlink_path}") |
| else: |
| # license_symlink_path.relative_to(.., walk_up=True) does not exist in |
| # Python 3.10, this is the reason why os.path.relpath is used. |
| os.symlink( |
| os.path.relpath(original_license_file, license_symlink_path.parent), |
| license_symlink_path) |
| |
| |
| def _map_rust_license_path_to_directory(license_file_path: str) -> str: |
| """ Returns the canonical path of the parent directory that includes |
| the LICENSE file for rust crates. |
| |
| :param license_file_path: This is the filepath found in the README.chromium |
| and the expected format is //some/path/license_file |
| """ |
| if not license_file_path.startswith("//"): |
| raise ValueError( |
| f"Rust third-party crate's `License File` is expected to be absolute path " |
| f"(Absolute GN labels are expected to start with //), " |
| f"but found {license_file_path}") |
| return license_file_path[2:license_file_path.rfind("/")] |
| |
| |
| def get_all_readme(repo_path: str): |
| """Fetches all README.chromium files under |repo_path|.""" |
| return glob.glob("**/README.chromium", root_dir=repo_path, recursive=True) |
| |
| def get_all_readme_through_gn(repo_path: str, targets: Set[str]): |
| processed_build_files_path = set() |
| for build_file_path in _get_all_build_files_path(repo_path, targets): |
| processed_build_files_path.add(os.path.dirname(os.path.relpath(build_file_path, repo_path))) |
| |
| readme_files = set() |
| for directory_path in processed_build_files_path: |
| # The python version on the trybots does not support `root_dir` so we have |
| # to work around this by appending it to the glob path then do a relpath. |
| readme_files.update((os.path.relpath( |
| readme_file_path, repo_path) for readme_file_path in glob.glob( |
| f"{os.path.join(repo_path, gn2bp_targets.README_MAPPING.get(directory_path, directory_path))}/**README.chromium", |
| recursive=True))) |
| |
| return readme_files | constants.INCLUDED_README |
| |
| def _get_all_build_files_path(repo_path: str, targets: Set[str]) -> Set[str]: |
| all_build_files_path = set() |
| for arch in cronet_utils.ARCHS: |
| with tempfile.TemporaryDirectory(dir=_OUT_DIR) as gn_out_dir: |
| cronet_utils.gn(gn_out_dir, ' '.join(cronet_utils.get_gn_args_for_aosp(arch))) |
| all_build_files_path.update( |
| cronet_utils.get_transitive_deps_build_files(repo_path, gn_out_dir, targets)) |
| |
| return all_build_files_path |
| |
| def should_skip_readme_file(readme_path: str) -> bool: |
| return readme_path in constants.IGNORED_README |
| |
| def update_license(repo_path: str = _ROOT_CRONET, |
| post_process_dict: Dict[str, Callable] = constants.POST_PROCESS_OPERATION, |
| verify_only: bool = False, reachable_through_dependencies: bool = True): |
| """ |
| Updates the licensing files for the entire repository of external/cronet. |
| |
| Running this will generate the following files for each README.chromium |
| |
| * LICENSE, this is a symbolic link and only created if there is no LICENSE |
| file already. |
| * METADATA |
| * MODULE_LICENSE_XYZ, XYZ represents the license found in README.chromium. |
| |
| Running in verify-only mode will ensure that everything is up to date, an |
| exception will be thrown if there needs to be any changes. |
| :param repo_path: Absolute path to Cronet's AOSP repository |
| :param post_process_dict: A dictionary that includes post-processing, this |
| post processing is not done on the README.chromium file but on the Metadata |
| structure that is extracted from them. |
| :param verify_only: Ensures that everything is up to date or throws. |
| :param reachable_through_dependencies: Ensures that the license generator |
| will only process README.chromium files that are reachable through cronet |
| as a transitive dependency. |
| """ |
| |
| readme_files_to_process = None |
| if reachable_through_dependencies: |
| readme_files_to_process = get_all_readme_through_gn(repo_path, _TARGETS_TO_PROCESS_LICENSE_FOR_TRANSITIVELY) |
| else: |
| readme_files_to_process = get_all_readme(repo_path) |
| |
| if readme_files_to_process == 0: |
| raise Exception( |
| f"Failed to find any README.chromium files under {repo_path}") |
| |
| readme_to_license_content = {} |
| for readme_file in readme_files_to_process: |
| if should_skip_readme_file(readme_file): |
| continue |
| readme_directory = os.path.dirname( |
| os.path.abspath(os.path.join(repo_path, readme_file))) |
| |
| metadata = license_utils.parse_chromium_readme_file( |
| os.path.abspath(os.path.join(repo_path, readme_file)), |
| post_process_dict.get( |
| readme_file, |
| lambda |
| _metadata: _metadata)) |
| |
| license_file_path = metadata.get_license_file_path() |
| if license_file_path is None: |
| # If there is no license file, create one with just the license name. This |
| # is necessary for e.g. |
| # //third_party/boringssl/src/pki/testdata/nist-pkits/README.chromium |
| license_file_path = "LICENSE" |
| with open(os.path.join(readme_directory, license_file_path), "w") as license_file: |
| license_file.write("\n".join(metadata.get_licenses())) |
| |
| license_directory = readme_directory |
| if (os.path.relpath(readme_directory, repo_path) |
| .startswith("third_party/rust/")): |
| # We must do a mapping as Chromium stores the README.chromium |
| # in a different directory than where the src/LICENSE is stored. |
| license_directory = os.path.join(repo_path, |
| _map_rust_license_path_to_directory( |
| license_file_path)) |
| |
| resolved_license_file_path = os.path.join(repo_path, license_utils.resolve_license_path( |
| readme_directory, |
| license_file_path)) |
| if not os.path.exists(resolved_license_file_path): |
| raise Exception(f"License file {resolved_license_file_path} does not exist for README.chromium: {readme_file}") |
| _maybe_create_license_file_symlink(license_directory, resolved_license_file_path, verify_only) |
| _create_module_license_file(repo_path, license_directory, |
| metadata.get_licenses(), verify_only) |
| _create_metadata_file(repo_path, license_directory, |
| metadata.to_android_metadata(), verify_only) |
| readme_to_license_content[readme_file] = cronet_utils.read_file(resolved_license_file_path) |
| |
| readme_for_top_level = None |
| if reachable_through_dependencies: |
| readme_for_top_level = get_all_readme_through_gn(repo_path, _TARGETS_TO_AGGREGATE_LICENSE_FOR_TRANSITIVELY) |
| else: |
| readme_for_top_level = get_all_readme(repo_path) |
| |
| top_level_license_file = Path(os.path.join(repo_path, "LICENSE")) |
| license_aggregated = [] |
| for readme in sorted(readme_for_top_level): |
| if should_skip_readme_file(readme): |
| continue |
| |
| license_aggregated += [ |
| "============================\n", |
| f"Path = {readme}\n", |
| "============================\n", |
| readme_to_license_content[readme] |
| ] |
| top_level_license_file.write_text("\n".join(license_aggregated)) |
| |
| if __name__ == '__main__': |
| sys.exit(update_license(post_process_dict=constants.POST_PROCESS_OPERATION, reachable_through_dependencies=True)) |