| #!/usr/bin/env python3 |
| # Copyright 2012 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Utility for checking and processing licensing information in third_party |
| directories. |
| |
| Usage: licenses.py <command> |
| |
| Commands: |
| scan scan third_party directories, verifying that we have licensing info |
| credits generate about:credits on stdout |
| |
| (You can also import this as a module.) |
| """ |
| from __future__ import print_function |
| |
| import argparse |
| import codecs |
| import csv |
| import io |
| import json |
| import logging |
| import os |
| import pathlib |
| import shutil |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| from typing import Any, Dict, List, Optional |
| |
| if sys.version_info.major == 2: |
| import cgi as html |
| else: |
| import html |
| |
| from spdx_writer import SpdxWriter |
| |
| _REPOSITORY_ROOT = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '..', '..')) |
| sys.path.insert(0, os.path.join(_REPOSITORY_ROOT, 'build')) |
| import action_helpers |
| |
| METADATA_FILE_NAMES = frozenset({ |
| "README.chromium", "README.crashpad", "README.v8", "README.pdfium", |
| "README.angle" |
| }) |
| |
| # Paths from the root of the tree to directories to skip. |
| PRUNE_PATHS = set([ |
| # Placeholder directory only, not third-party code. |
| os.path.join('third_party', 'adobe'), |
| |
| # Will remove it once converted private sdk using cipd. |
| os.path.join('third_party', 'android_tools_internal'), |
| |
| # Build files only, not third-party code. |
| os.path.join('third_party', 'widevine'), |
| |
| # Only binaries, used during development. |
| os.path.join('third_party', 'valgrind'), |
| |
| # Not actually a third party dependency. Supplies configuration for |
| # enabling or disabling field trials and features in Chromium projects. |
| os.path.join('third_party', 'chromium-variations'), |
| |
| # Used for development and test, not in the shipping product. |
| os.path.join('build', 'secondary'), |
| os.path.join('third_party', 'bison'), |
| os.path.join('third_party', 'chromite'), |
| os.path.join('third_party', 'clang-format'), |
| os.path.join('third_party', 'cygwin'), |
| os.path.join('third_party', 'gnu_binutils'), |
| os.path.join('third_party', 'gold'), |
| os.path.join('third_party', 'gperf'), |
| os.path.join('third_party', 'lighttpd'), |
| os.path.join('third_party', 'llvm'), |
| os.path.join('third_party', 'llvm-build'), |
| os.path.join('third_party', 'mingw-w64'), |
| os.path.join('third_party', 'nacl_sdk_binaries'), |
| os.path.join('third_party', 'pefile'), |
| os.path.join('third_party', 'perl'), |
| os.path.join('third_party', 'psyco_win32'), |
| os.path.join('third_party', 'pyelftools'), |
| os.path.join('third_party', 'pylib'), |
| os.path.join('third_party', 'pywebsocket'), |
| os.path.join('third_party', 'syzygy'), |
| |
| # Stuff pulled in from chrome-internal for official builds/tools. |
| os.path.join('third_party', 'amd'), |
| os.path.join('third_party', 'clear_cache'), |
| os.path.join('third_party', 'gnu'), |
| os.path.join('third_party', 'googlemac'), |
| os.path.join('third_party', 'pcre'), |
| os.path.join('third_party', 'psutils'), |
| os.path.join('third_party', 'sawbuck'), |
| os.path.join('third_party', 'wix'), |
| # See crbug.com/350472 |
| os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'), |
| # Chrome for Android proprietary code. |
| os.path.join('clank'), |
| |
| # Proprietary barcode detection library. |
| os.path.join('third_party', 'barhopper'), |
| |
| # Internal Chrome Build only for proprietary webref library. |
| os.path.join('components', 'optimization_guide', 'internal', 'third_party'), |
| |
| # Proprietary DevTools code. |
| os.path.join('third_party', 'devtools-frontend-internal'), |
| |
| # Redistribution does not require attribution in documentation. |
| os.path.join('third_party', 'directxsdk'), |
| |
| # For testing only, presents on some bots. |
| os.path.join('isolate_deps_dir'), |
| |
| # Mock test data. |
| os.path.join('tools', 'binary_size', 'libsupersize', 'testdata'), |
| |
| # Overrides some WebRTC files, same license. Skip this one. |
| os.path.join('third_party', 'webrtc_overrides'), |
| ]) |
| |
| # Directories we don't scan through. |
| VCS_METADATA_DIRS = ('.svn', '.git') |
| PRUNE_DIRS = VCS_METADATA_DIRS + ('layout_tests', ) # lots of subdirs |
| |
| # A third_party directory can define this file, containing a list of |
| # subdirectories to process in addition to itself. Intended for directories |
| # that contain multiple others as transitive dependencies. |
| ADDITIONAL_PATHS_FILENAME = 'additional_readme_paths.json' |
| |
| # A list of paths that contain license information but that would otherwise |
| # not be included. Possible reasons include: |
| # - Third party directories in //clank which are considered to be Google-owned |
| # - Directories that are directly checked out from upstream, and thus |
| # don't have a README.chromium |
| # - Directories that contain example code, or build tooling. |
| # - Nested third_party code inside other third_party libraries. |
| ADDITIONAL_PATHS = ( |
| os.path.join('chrome', 'test', 'chromeos', 'autotest'), |
| os.path.join('chrome', 'test', 'data'), |
| os.path.join('native_client'), |
| os.path.join('third_party', 'boringssl', 'src', 'third_party', 'fiat'), |
| os.path.join('third_party', 'devtools-frontend', 'src', 'front_end', |
| 'third_party'), |
| os.path.join('third_party', 'devtools-frontend-internal', 'front_end', |
| 'third_party'), |
| os.path.join('tools', 'page_cycler', 'acid3'), |
| os.path.join('url', 'third_party', 'mozilla'), |
| os.path.join('v8'), |
| # Fake directories to include the strongtalk and fdlibm licenses. |
| os.path.join('v8', 'strongtalk'), |
| os.path.join('v8', 'fdlibm'), |
| ) |
| |
| # SPECIAL_CASES are used for historical directories where we checked out |
| # directly from upstream into the same directory as we would put metadata. |
| # In new cases, you should check out upstream source into //root/foo/src |
| # and keep the corresponding README.chromium file at //root/foo/README.chromium |
| # instead of adding a SPECIAL_CASE. |
| # These SPECIAL_CASES should not be used to suppress errors. Please fix |
| # any metadata files with errors and if you encounter a parsing issue, |
| # please file a bug. |
| SPECIAL_CASES = { |
| os.path.join('native_client'): { |
| "Name": "native client", |
| "URL": "https://code.google.com/p/nativeclient", |
| "Shipped": "yes", |
| "License": "BSD", |
| "License File": ["//native_client/LICENSE"], |
| }, |
| os.path.join('third_party', 'angle'): { |
| "Name": "Almost Native Graphics Layer Engine", |
| "URL": "https://chromium.googlesource.com/angle/angle/", |
| "Shipped": "yes", |
| "License": "BSD", |
| }, |
| os.path.join('third_party', 'cros_system_api'): { |
| "Name": "Chromium OS system API", |
| "URL": "https://www.chromium.org/chromium-os", |
| "Shipped": "yes", |
| "License": "BSD", |
| # Absolute path here is resolved as relative to the source root. |
| "License File": ["//LICENSE.chromium_os"], |
| }, |
| os.path.join('third_party', 'ipcz'): { |
| "Name": "ipcz", |
| "URL": ( |
| "https://chromium.googlesource.com/chromium/src/third_party/ipcz"), |
| "Shipped": "yes", |
| "License": "BSD", |
| "License File": ["//third_party/ipcz/LICENSE"], |
| }, |
| os.path.join('third_party', 'lss'): { |
| "Name": "linux-syscall-support", |
| "URL": "https://chromium.googlesource.com/linux-syscall-support/", |
| "Shipped": "yes", |
| "License": "BSD", |
| "License File": ["//third_party/lss/LICENSE"], |
| }, |
| os.path.join('third_party', 'openscreen', 'src', 'third_party', 'abseil'): { |
| "Name": "abseil", |
| "URL": "https://github.com/abseil/abseil-cpp/", |
| "Shipped": "yes", |
| "License": "Apache 2.0", |
| "License File": ["//third_party/abseil-cpp/LICENSE"], |
| }, |
| os.path.join('third_party', 'openscreen', 'src', 'third_party', |
| 'boringssl'): { |
| "Name": "BoringSSL", |
| "URL": "https://boringssl.googlesource.com/boringssl/", |
| "Shipped": "yes", |
| "License": "BSDish", |
| "License File": ["//third_party/boringssl/src/LICENSE"], |
| }, |
| os.path.join('third_party', 'openscreen', 'src', 'third_party', |
| 'jsoncpp'): { |
| "Name": "jsoncpp", |
| "URL": "https://github.com/open-source-parsers/jsoncpp", |
| "Shipped": "yes", |
| "License": "MIT", |
| "License File": ["//third_party/jsoncpp/LICENSE"], |
| }, |
| os.path.join('third_party', 'openscreen', 'src', 'third_party', |
| 'mozilla'): { |
| "Name": "mozilla", |
| "URL": "https://github.com/mozilla", |
| "Shipped": "yes", |
| "License": "MPL 1.1/GPL 2.0/LGPL 2.1", |
| "License File": ["LICENSE.txt"], |
| }, |
| os.path.join('third_party', 'pdfium'): { |
| "Name": "PDFium", |
| "URL": "https://pdfium.googlesource.com/pdfium/", |
| "Shipped": "yes", |
| "License": "BSD", |
| }, |
| os.path.join('third_party', 'ppapi'): { |
| "Name": "ppapi", |
| "URL": "https://code.google.com/p/ppapi/", |
| "Shipped": "yes", |
| }, |
| os.path.join('third_party', 'crashpad', 'crashpad', 'third_party', |
| 'getopt'): { |
| "Name": "getopt", |
| "URL": "https://sourceware.org/ml/newlib/2005/msg00758.html", |
| "Shipped": "yes", |
| "License": "Public domain", |
| "License File": [ |
| "//third_party/crashpad/crashpad/third_party/getopt/LICENSE", |
| ], |
| }, |
| os.path.join('third_party', 'crashpad', 'crashpad', 'third_party', 'xnu'): { |
| "Name": "xnu", |
| "URL": "https://opensource.apple.com/source/xnu/", |
| "Shipped": "yes", |
| "License": "Apple Public Source License 2.0", |
| "License File": ["APPLE_LICENSE"], |
| }, |
| os.path.join('third_party', 'v8-i18n'): { |
| "Name": "Internationalization Library for v8", |
| "URL": "https://code.google.com/p/v8-i18n/", |
| "Shipped": "yes", |
| "License": "Apache 2.0", |
| }, |
| os.path.join('third_party', 'blink'): { |
| # about:credits doesn't show "Blink" but "WebKit". |
| # Blink is a fork of WebKit, and Chromium project has maintained it |
| # since the fork. about:credits needs to mention the code before |
| # the fork. |
| "Name": "WebKit", |
| "URL": "https://webkit.org/", |
| "Shipped": "yes", |
| "License": "BSD and LGPL v2 and LGPL v2.1", |
| # Absolute path here is resolved as relative to the source root. |
| "License File": ["//third_party/blink/LICENSE_FOR_ABOUT_CREDITS"], |
| }, |
| os.path.join('v8'): { |
| "Name": "V8 JavaScript Engine", |
| "URL": "https://v8.dev/", |
| "Shipped": "yes", |
| "License": "BSD", |
| }, |
| os.path.join('v8', 'strongtalk'): { |
| "Name": "Strongtalk", |
| "URL": "https://www.strongtalk.org/", |
| "Shipped": "yes", |
| "License": "BSD", |
| # Absolute path here is resolved as relative to the source root. |
| "License File": ["//v8/LICENSE.strongtalk"], |
| }, |
| os.path.join('v8', 'fdlibm'): { |
| "Name": "fdlibm", |
| "URL": "https://www.netlib.org/fdlibm/", |
| "Shipped": "yes", |
| "License": "Freely Distributable", |
| # Absolute path here is resolved as relative to the source root. |
| "License File": ["//v8/LICENSE.fdlibm"], |
| "License Android Compatible": "yes", |
| }, |
| os.path.join('third_party', 'swiftshader'): { |
| "Name": "SwiftShader", |
| "URL": "https://swiftshader.googlesource.com/SwiftShader", |
| "Shipped": "yes", |
| "License": "Apache 2.0 and compatible licenses", |
| "License Android Compatible": "yes", |
| "License File": ["//third_party/swiftshader/LICENSE.txt"], |
| }, |
| os.path.join('third_party', 'swiftshader', 'third_party', 'SPIRV-Tools'): { |
| "Name": "SPIRV-Tools", |
| "URL": "https://github.com/KhronosGroup/SPIRV-Tools", |
| "Shipped": "yes", |
| "License": "Apache 2.0", |
| "License File": [ |
| "//third_party/swiftshader/third_party/SPIRV-Tools/LICENSE", |
| ], |
| }, |
| os.path.join('third_party', 'swiftshader', 'third_party', |
| 'SPIRV-Headers'): { |
| "Name": "SPIRV-Headers", |
| "URL": "https://github.com/KhronosGroup/SPIRV-Headers", |
| "Shipped": "yes", |
| "License": "Apache 2.0", |
| "License File": [ |
| "//third_party/swiftshader/third_party/SPIRV-Headers/LICENSE", |
| ], |
| }, |
| os.path.join('third_party', 'dawn', 'third_party', 'khronos'): { |
| "Name": "khronos_platform", |
| "URL": "https://registry.khronos.org/EGL/", |
| "Shipped": "yes", |
| "License": "Apache 2.0", |
| "License File": ["//third_party/dawn/third_party/khronos/LICENSE"], |
| }, |
| } |
| |
| # These buildtools/third_party directories only contain |
| # chromium build files. The actual third_party source files and their |
| # README.chromium files are under third_party/libc*/. |
| # So we do not include licensing metadata for these directories. |
| # See crbug.com/1458042 for more details. |
| PRUNE_PATHS.update([ |
| os.path.join('buildtools', 'third_party', 'libc++'), |
| os.path.join('buildtools', 'third_party', 'libc++abi'), |
| os.path.join('buildtools', 'third_party', 'libunwind'), |
| ]) |
| |
| # The mandatory metadata fields for a single dependency. |
| MANDATORY_FIELDS = { |
| "Name", # Short name (for header on about:credits). |
| "URL", # Project home page. |
| "License", # Software license. |
| "License File", # Relative paths to license texts. |
| "Shipped", # Whether the package is in the shipped product. |
| } |
| |
| # Field aliases (key is the alias, value is the field to map to). |
| # Note: if both fields are provided, the alias field value will be used. |
| ALIAS_FIELDS = { |
| "Shipped in Chromium": "Shipped", |
| } |
| |
| # The metadata fields that can have multiple values. |
| MULTIVALUE_FIELDS = { |
| "License File", |
| } |
| |
| # Line used to separate dependencies within the same metadata file. |
| PATTERN_DEPENDENCY_DIVIDER = re.compile(r"^-{20} DEPENDENCY DIVIDER -{20}$") |
| |
| # The delimiter used to separate multiple values for one metadata field. |
| VALUE_DELIMITER = "," |
| |
| # Soon-to-be-deprecated special value for 'License File' field used to indicate |
| # that the library is not shipped so the license file should not be used in |
| # about:credits. |
| # This value is still supported, but the preferred method is to set the |
| # 'Shipped' field to 'no' in the library's README.chromium. |
| NOT_SHIPPED = "NOT_SHIPPED" |
| |
| # Valid values for the 'Shipped' field used to indicate whether the library is |
| # shipped and consequently whether the license file should be used in |
| # about:credits. |
| YES = "yes" |
| NO = "no" |
| |
| # Paths for libraries that we have checked are not shipped on iOS. These are |
| # left out of the licenses file primarily because we don't want to cause a |
| # firedrill due to someone thinking that Chrome for iOS is using LGPL code |
| # when it isn't. |
| # This is a temporary hack; the real solution is crbug.com/178215 |
| KNOWN_NON_IOS_LIBRARIES = set([ |
| os.path.join('base', 'third_party', 'symbolize'), |
| os.path.join('base', 'third_party', 'xdg_mime'), |
| os.path.join('base', 'third_party', 'xdg_user_dirs'), |
| os.path.join('chrome', 'installer', 'mac', 'third_party', 'bsdiff'), |
| os.path.join('chrome', 'installer', 'mac', 'third_party', 'xz'), |
| os.path.join('chrome', 'test', 'data', 'third_party', 'kraken'), |
| os.path.join('chrome', 'test', 'data', 'third_party', 'spaceport'), |
| os.path.join('chrome', 'third_party', 'mozilla_security_manager'), |
| os.path.join('third_party', 'angle'), |
| os.path.join('third_party', 'apple_apsl'), |
| os.path.join('third_party', 'apple_sample_code'), |
| os.path.join('third_party', 'ashmem'), |
| os.path.join('third_party', 'blink'), |
| os.path.join('third_party', 'bspatch'), |
| os.path.join('third_party', 'cld'), |
| os.path.join('third_party', 'flot'), |
| os.path.join('third_party', 'gtk+'), |
| os.path.join('third_party', 'iaccessible2'), |
| os.path.join('third_party', 'iccjpeg'), |
| os.path.join('third_party', 'isimpledom'), |
| os.path.join('third_party', 'jsoncpp'), |
| os.path.join('third_party', 'khronos'), |
| os.path.join('third_party', 'libcxx', 'libc++'), |
| os.path.join('third_party', 'libcxx', 'libc++abi'), |
| os.path.join('third_party', 'libevent'), |
| os.path.join('third_party', 'libjpeg'), |
| os.path.join('third_party', 'libusb'), |
| os.path.join('third_party', 'libxslt'), |
| os.path.join('third_party', 'lss'), |
| os.path.join('third_party', 'lzma_sdk'), |
| os.path.join('third_party', 'mesa'), |
| os.path.join('third_party', 'mozc'), |
| os.path.join('third_party', 'mozilla'), |
| os.path.join('third_party', 'npapi'), |
| os.path.join('third_party', 'ots'), |
| os.path.join('third_party', 'perfetto'), |
| os.path.join('third_party', 'ppapi'), |
| os.path.join('third_party', 'qcms'), |
| os.path.join('third_party', 're2'), |
| os.path.join('third_party', 'safe_browsing'), |
| os.path.join('third_party', 'smhasher'), |
| os.path.join('third_party', 'swiftshader'), |
| os.path.join('third_party', 'swig'), |
| os.path.join('third_party', 'talloc'), |
| os.path.join('third_party', 'usb_ids'), |
| os.path.join('third_party', 'v8-i18n'), |
| os.path.join('third_party', 'wtl'), |
| os.path.join('third_party', 'yasm'), |
| os.path.join('v8', 'strongtalk'), |
| ]) |
| |
| |
| class InvalidMetadata(Exception): |
| """This exception is raised when metadata is invalid.""" |
| pass |
| |
| |
| class LicenseError(Exception): |
| """We raise this exception when a dependency's licensing info isn't |
| fully filled out. |
| """ |
| pass |
| |
| |
| def AbsolutePath(path, filename, root): |
| """Convert a path in README.chromium to be absolute based on the source |
| root.""" |
| if filename.startswith('/'): |
| # Absolute-looking paths are relative to the source root |
| # (which is the directory we're run from). |
| absolute_path = os.path.join(root, os.path.normpath(filename.lstrip('/'))) |
| else: |
| absolute_path = os.path.join(root, path, os.path.normpath(filename)) |
| if os.path.exists(absolute_path): |
| return absolute_path |
| return None |
| |
| |
| def ParseMetadataFile(filepath: str, |
| optional_fields: List[str] = []) -> List[Dict[str, Any]]: |
| """Parses the metadata from the file. |
| |
| Args: |
| filepath: the path to a file from which to parse metadata. |
| optional_fields: list of optional metadata fields. |
| |
| Returns: the metadata for all dependencies described in the file. |
| |
| Raises: |
| InvalidMetadata - if the metadata in the file has duplicate fields |
| for a dependency. |
| """ |
| known_fields = (list(MANDATORY_FIELDS) + list(ALIAS_FIELDS.keys()) + |
| optional_fields) |
| field_lookup = {name.lower(): name for name in known_fields} |
| |
| dependencies = [] |
| metadata = {} |
| with codecs.open(filepath, encoding="utf-8") as readme: |
| for line in readme: |
| line = line.strip() |
| # Skip empty lines. |
| if not line: |
| continue |
| |
| # Check if a new dependency will be described. |
| if re.match(PATTERN_DEPENDENCY_DIVIDER, line): |
| # Save the metadata for the previous dependency. |
| if metadata: |
| dependencies.append(metadata) |
| metadata = {} |
| continue |
| |
| # Otherwise, try to parse the field name and field value. |
| parts = line.split(": ", 1) |
| if len(parts) == 2: |
| raw_field, value = parts |
| field = field_lookup.get(raw_field.lower()) |
| if field: |
| if field in metadata: |
| # Duplicate field for this dependency. |
| raise InvalidMetadata(f"duplicate '{field}' in {filepath}") |
| if field in MULTIVALUE_FIELDS: |
| metadata[field] = [ |
| entry.strip() for entry in value.split(VALUE_DELIMITER) |
| ] |
| else: |
| metadata[field] = value |
| |
| # The end of the file has been reached. Save the metadata for the |
| # last dependency, if available. |
| if metadata: |
| dependencies.append(metadata) |
| |
| return dependencies |
| |
| |
| def ProcessMetadata(metadata: Dict[str, Any], |
| readme_path: str, |
| path: str, |
| root: str, |
| require_license_file: bool = True, |
| enable_warnings: bool = False) -> List[str]: |
| """Processes a single dependency's metadata and returns the updated |
| data if it passes validation. This function updates the given metadata |
| to use fallback fields and change any relative paths to absolute. |
| |
| Args: |
| metadata: a single dependency's metadata. |
| readme_path: the source of the metadata (either a metadata file |
| or a SPECIAL_CASES entry). |
| path: the source file for the metadata. |
| root: the root directory of the repo. |
| require_license_file: whether a license file is required. |
| enable_warnings: whether warnings should be displayed. |
| |
| Returns: error messages, if there were any issues processing the |
| metadata for license information. |
| """ |
| errors = [] |
| |
| # The dependency reference, for more precise error messages. |
| dep_ref = os.path.relpath(readme_path, root) |
| dep_name = metadata.get("Name") |
| if dep_name: |
| dep_ref = f"{dep_ref}>>{dep_name}" |
| |
| # Set field values for fields with aliases. |
| for alias, field in ALIAS_FIELDS.items(): |
| if alias in metadata: |
| metadata[field] = metadata[alias] |
| metadata.pop(alias) |
| |
| # Set the default "License File" value. |
| if metadata.get("License File") is None: |
| metadata["License File"] = ["LICENSE"] |
| |
| # If the "Shipped" field isn't present (or is empty), set it based on |
| # the value of the "License File" field. |
| if not metadata.get("Shipped"): |
| shipped = YES |
| if NOT_SHIPPED in metadata.get("License File"): |
| shipped = NO |
| metadata["Shipped"] = shipped |
| |
| # Check all mandatory fields have a non-empty value. |
| for field in MANDATORY_FIELDS: |
| if not metadata.get(field): |
| errors.append(f"couldn't find '{field}' line in README.chromium or " |
| "licenses.py SPECIAL_CASES") |
| |
| license_file_value = metadata.get("License File") |
| shipped_value = metadata.get("Shipped") |
| if enable_warnings: |
| # Check for the deprecated special value used in the "License File" |
| # field. |
| if NOT_SHIPPED in license_file_value: |
| logging.warning( |
| f"{dep_ref} is using deprecated {NOT_SHIPPED} value " |
| "in 'License File' field - remove this and instead specify " |
| f"'Shipped: {NO}'.") |
| |
| # Check the "Shipped" field does not contradict the "License File" |
| # field. |
| if shipped_value == YES: |
| logging.warning( |
| f"Contradictory metadata for {dep_ref} - 'Shipped: {YES}' " |
| f"but 'License File' includes '{NOT_SHIPPED}'") |
| |
| # For the modules that are in the shipping product, we need their |
| # license in about:credits, so update the license files to be the |
| # full paths. |
| license_paths = process_license_files(root, path, license_file_value) |
| if shipped_value == YES and require_license_file and not license_paths: |
| errors.append( |
| f"License file not found for {dep_ref}. Either add a file named " |
| "LICENSE, import upstream's COPYING if available, or add a " |
| "'License File:' line to README.chromium with the appropriate paths.") |
| metadata["License File"] = license_paths |
| |
| if errors: |
| # if there were any errors during parsing, clear all values from |
| # the dependenct metadata so no further processing occurs |
| metadata = {} |
| |
| return errors |
| |
| |
| def ParseDir(path, |
| root, |
| require_license_file=True, |
| optional_keys=[], |
| enable_warnings=False, |
| metadata_file_names=METADATA_FILE_NAMES): |
| """Examine a third_party path and extract that directory's metadata. |
| |
| Note: directory metadata can contain metadata for multiple |
| dependencies. |
| |
| Returns: A tuple with a list of directory metadata, and accrued parsing errors |
| |
| """ |
| # Get the metadata values, from |
| # (a) looking up the path in SPECIAL_CASES; or |
| # (b) parsing the metadata from a README.chromium file. |
| if path in SPECIAL_CASES: |
| readme_path = f"licenses.py SPECIAL_CASES entry for {path}" |
| directory_metadata = dict(SPECIAL_CASES[path]) |
| errors = ProcessMetadata(directory_metadata, |
| readme_path, |
| path, |
| root, |
| require_license_file=require_license_file, |
| enable_warnings=enable_warnings) |
| |
| return [directory_metadata], errors |
| |
| errors = [] |
| readmes_in_dir = False |
| valid_metadata = [] |
| directory_metadata = [] |
| |
| for name in metadata_file_names: |
| for readme_path in (pathlib.Path(root) / path).glob(name): |
| readmes_in_dir = True |
| |
| try: |
| file_metadata = ParseMetadataFile(str(readme_path), |
| optional_fields=optional_keys) |
| for dependency_metadata in file_metadata: |
| meta_errors = ProcessMetadata( |
| dependency_metadata, |
| readme_path, |
| path, |
| root, |
| require_license_file=require_license_file, |
| enable_warnings=enable_warnings) |
| |
| if meta_errors: |
| errors.append( |
| "Errors in %s:\n %s\n" % |
| (os.path.relpath(readme_path, root), ";\n ".join(meta_errors))) |
| continue |
| |
| if dependency_metadata: |
| valid_metadata.append(dependency_metadata) |
| |
| except InvalidMetadata as e: |
| errors.append(f"Invalid metadata file: {e}") |
| continue |
| |
| if not readmes_in_dir: |
| raise LicenseError(f"missing third party metadata file " |
| f"or licenses.py SPECIAL_CASES entry in {path}\n") |
| |
| return valid_metadata, errors |
| |
| |
| def process_license_files( |
| root: str, |
| path: str, |
| license_files: List[str], |
| ) -> List[str]: |
| """ |
| Convert a list of license file paths which were specified in a |
| README.chromium to be absolute paths based on the source root. |
| |
| Args: |
| root: the repository source root. |
| path: the relative path from root. |
| license_files: list of values specified in the 'License File' field. |
| |
| Returns: absolute paths to license files that exist. |
| """ |
| license_paths = [] |
| for file_path in license_files: |
| if file_path == NOT_SHIPPED: |
| continue |
| |
| license_path = AbsolutePath(path, file_path, root) |
| # Check that the license file exists. |
| if license_path is not None: |
| license_paths.append(license_path) |
| |
| # If there are no license files at all, check for the COPYING license file. |
| if not license_paths: |
| license_path = AbsolutePath(path, "COPYING", root) |
| # Check that the license file exists. |
| if license_path is not None: |
| license_paths.append(license_path) |
| |
| return license_paths |
| |
| |
| def _ContainsFiles(path): |
| """Determines whether any files exist in a directory or in any of its |
| subdirectories.""" |
| for _, dirs, files in os.walk(path): |
| if files: |
| return True |
| for vcs_metadata in VCS_METADATA_DIRS: |
| if vcs_metadata in dirs: |
| dirs.remove(vcs_metadata) |
| return False |
| |
| |
| def _MaybeAddThirdPartyPath(root, dirpath, third_party_dirs): |
| """Adds |dirpath| to |third_party_dirs| and processes |
| additional_readme_paths.json.""" |
| # Prune paths and guard against cycles. |
| if dirpath in PRUNE_PATHS or dirpath in third_party_dirs: |
| return |
| # Guard against missing / empty dirs (gclient creates empty directories for |
| # conditionally downloaded submodules). |
| resolved_path = os.path.join(root, dirpath) |
| if not os.path.exists(resolved_path) or not _ContainsFiles(resolved_path): |
| return |
| third_party_dirs.add(dirpath) |
| |
| additional_paths_file = os.path.join(root, dirpath, ADDITIONAL_PATHS_FILENAME) |
| if not os.path.exists(additional_paths_file): |
| return |
| with codecs.open(additional_paths_file, encoding='utf-8') as paths_file: |
| additional_paths = json.load(paths_file) |
| for p in additional_paths: |
| subpath = os.path.normpath(os.path.join(dirpath, p)) |
| _MaybeAddThirdPartyPath(root, subpath, third_party_dirs) |
| |
| |
| def FindThirdPartyDirs(root, extra_third_party_dirs=None): |
| """Find all third_party directories underneath the source root.""" |
| third_party_dirs = set() |
| for path, dirs, files in os.walk(root): |
| path = path[len(root) + 1:] # Pretty up the path. |
| |
| # .gitignore ignores /out*/, so do the same here. |
| if path in PRUNE_PATHS or path.startswith('out'): |
| dirs[:] = [] |
| continue |
| |
| # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular |
| # third_party/foo paths. |
| if path in ADDITIONAL_PATHS: |
| dirs[:] = [] |
| |
| # Prune out directories we want to skip. |
| # (Note that we loop over PRUNE_DIRS so we're not iterating over a |
| # list that we're simultaneously mutating.) |
| for skip in PRUNE_DIRS: |
| if skip in dirs: |
| dirs.remove(skip) |
| |
| if os.path.basename(path) == 'third_party': |
| # Add all subdirectories that are not marked for skipping. |
| for d in dirs: |
| dirpath = os.path.join(path, d) |
| _MaybeAddThirdPartyPath(root, dirpath, third_party_dirs) |
| |
| |
| extra_paths = set(ADDITIONAL_PATHS) |
| if extra_third_party_dirs: |
| extra_paths.update(extra_third_party_dirs) |
| |
| for path in extra_paths: |
| _MaybeAddThirdPartyPath(root, path, third_party_dirs) |
| |
| return sorted(third_party_dirs) |
| |
| |
| # Many builders do not contain 'gn' in their PATH, so use the GN binary from |
| # //buildtools. |
| def _GnBinary(): |
| exe = 'gn' |
| if sys.platform.startswith('linux'): |
| subdir = 'linux64' |
| elif sys.platform == 'darwin': |
| subdir = 'mac' |
| elif sys.platform == 'win32': |
| subdir, exe = 'win', 'gn.exe' |
| else: |
| raise RuntimeError("Unsupported platform '%s'." % sys.platform) |
| |
| return os.path.join(_REPOSITORY_ROOT, 'buildtools', subdir, exe) |
| |
| |
| def LogParseDirErrors(errors): |
| """Provides a convenience method for printing out the errors resulting |
| from running ParseDir() over a directory.""" |
| |
| for error in sorted(errors): |
| print(error) |
| |
| |
| def GetThirdPartyDepsFromGNDepsOutput( |
| gn_deps: str, |
| target_os: str, |
| extra_allowed_dirs: Optional[List[str]] = None): |
| """Returns third_party/foo directories given the output of "gn desc deps". |
| |
| Note that it always returns the direct sub-directory of third_party |
| where README.chromium and LICENSE files are, so that it can be passed to |
| ParseDir(). e.g.: |
| third_party/cld_3/src/src/BUILD.gn -> third_party/cld_3/ |
| Rust dependencies are a special case, with a deeper structure: |
| third_party/rust/foo/v1/crate/BUILD.gn -> third_party/rust/foo/v1/ |
| |
| It returns relative paths from _REPOSITORY_ROOT, not absolute paths. |
| """ |
| allowed_paths_list = ['third_party'] |
| if extra_allowed_dirs: |
| allowed_paths_list.extend(extra_allowed_dirs) |
| |
| # Use non-capturing group with or's for all possible options. |
| allowed_paths = '|'.join([re.escape(x) for x in allowed_paths_list]) |
| sep = re.escape(os.path.sep) |
| path_regex = re.compile( |
| r'''^ |
| ( # capture |
| (.+{sep})? # any prefix |
| (?:{allowed_paths}) # any of the allowed paths |
| {sep} |
| (?: # either.. |
| rust{sep}{nonsep}+{sep}v{nonsep}+ # rust/<crate>/v<version> |
| |{nonsep}+) # or any single path element |
| {sep} |
| ) |
| (.+{sep})?BUILD\.gn$ # with filename BUILD.gn |
| '''.format(allowed_paths=allowed_paths, sep=sep, nonsep=f'[^{sep}]'), |
| re.VERBOSE) |
| |
| third_party_deps = set() |
| for absolute_build_dep in gn_deps.split(): |
| relative_build_dep = os.path.relpath(absolute_build_dep, _REPOSITORY_ROOT) |
| m = path_regex.search(relative_build_dep) |
| if not m: |
| continue |
| third_party_path = m.group(1) |
| if any(third_party_path.startswith(p + os.sep) for p in PRUNE_PATHS): |
| continue |
| if (target_os == 'ios' and any( |
| third_party_path.startswith(p + os.sep) |
| for p in KNOWN_NON_IOS_LIBRARIES)): |
| # Skip over files that are known not to be used on iOS. |
| continue |
| third_party_deps.add(third_party_path[:-1]) |
| return third_party_deps |
| |
| |
| def FindThirdPartyDeps(gn_out_dir: str, |
| gn_target: str, |
| target_os: str, |
| extra_third_party_dirs: Optional[List[str]] = None, |
| extra_allowed_dirs: Optional[List[str]] = None): |
| if not gn_out_dir: |
| raise RuntimeError("--gn-out-dir is required if --gn-target is used.") |
| |
| # Generate gn project in temp directory and use it to find dependencies. |
| # Current gn directory cannot be used when we run this script in a gn action |
| # rule, because gn doesn't allow recursive invocations due to potential side |
| # effects. |
| try: |
| with tempfile.TemporaryDirectory(dir=gn_out_dir) as tmp_dir: |
| shutil.copy(os.path.join(gn_out_dir, "args.gn"), tmp_dir) |
| subprocess.check_output( |
| [_GnBinary(), "gen", |
| "--root=%s" % _REPOSITORY_ROOT, tmp_dir]) |
| gn_deps = subprocess.check_output([ |
| _GnBinary(), "desc", |
| "--root=%s" % _REPOSITORY_ROOT, tmp_dir, gn_target, "deps", |
| "--as=buildfile", "--all" |
| ]) |
| if isinstance(gn_deps, bytes): |
| gn_deps = gn_deps.decode("utf-8") |
| except: |
| if sys.platform == 'win32': |
| print(""" |
| ########################################################################## |
| |
| This is a known issue; please report the failure to |
| https://crbug.com/1208393. |
| |
| ########################################################################## |
| """) |
| subprocess.check_call(['tasklist.exe']) |
| raise |
| |
| third_party_deps = GetThirdPartyDepsFromGNDepsOutput(gn_deps, target_os, |
| extra_allowed_dirs) |
| if extra_third_party_dirs: |
| third_party_deps.update(extra_third_party_dirs) |
| return sorted(third_party_deps) |
| |
| |
| def ScanThirdPartyDirs(root=None): |
| """Scan a list of directories and report on any problems we find.""" |
| if root is None: |
| root = os.getcwd() |
| third_party_dirs = FindThirdPartyDirs(root) |
| |
| errors = [] |
| for path in sorted(third_party_dirs): |
| try: |
| _, errors = ParseDir(path, root, enable_warnings=True) |
| except LicenseError as e: |
| errors.append(f"{path}: {e}") |
| continue |
| |
| LogParseDirErrors(errors) |
| |
| return len(errors) == 0 |
| |
| |
| def GenerateCredits(file_template_file, |
| entry_template_file, |
| reciprocal_template_file, |
| output_file, |
| target_os, |
| gn_out_dir, |
| gn_target, |
| extra_third_party_dirs=None, |
| depfile=None, |
| enable_warnings=False): |
| """Generate about:credits.""" |
| def EvaluateTemplate(template, env, escape=True): |
| """Expand a template with variables like {{foo}} using a |
| dictionary of expansions.""" |
| for key, val in env.items(): |
| if escape: |
| val = html.escape(val) |
| template = template.replace('{{%s}}' % key, val) |
| return template |
| |
| def MetadataToTemplateEntry(metadata, entry_template): |
| licenses = [] |
| for filepath in metadata['License File']: |
| licenses.append( |
| codecs.open(filepath, errors="replace", encoding='utf-8').read()) |
| license_content = '\n\n'.join(licenses) |
| |
| env = { |
| 'name': metadata['Name'], |
| 'url': metadata['URL'], |
| 'license': license_content, |
| } |
| |
| return { |
| 'name': metadata['Name'], |
| 'content': EvaluateTemplate(entry_template, env), |
| 'license_file': metadata['License File'], |
| } |
| |
| if gn_target: |
| third_party_dirs = FindThirdPartyDeps(gn_out_dir, gn_target, target_os, |
| extra_third_party_dirs) |
| |
| # Sanity-check to raise a build error if invalid gn_... settings are |
| # somehow passed to this script. |
| if not third_party_dirs: |
| raise RuntimeError("No deps found.") |
| else: |
| third_party_dirs = FindThirdPartyDirs(_REPOSITORY_ROOT, |
| extra_third_party_dirs) |
| |
| if not file_template_file: |
| file_template_file = os.path.join(_REPOSITORY_ROOT, 'components', |
| 'about_ui', 'resources', |
| 'about_credits.tmpl') |
| if not entry_template_file: |
| entry_template_file = os.path.join(_REPOSITORY_ROOT, 'components', |
| 'about_ui', 'resources', |
| 'about_credits_entry.tmpl') |
| |
| # Used to add a link at the top of credits for Chromium code to |
| # satisfy the requirements for reciprocal license types. |
| if not reciprocal_template_file: |
| reciprocal_template_file = os.path.join(_REPOSITORY_ROOT, 'components', |
| 'about_ui', 'resources', |
| 'about_credits_reciprocal.tmpl') |
| |
| entry_template = codecs.open(entry_template_file, encoding='utf-8').read() |
| entries = [] |
| # Start from Chromium's LICENSE file |
| chromium_license_metadata = { |
| 'Name': 'The Chromium Project', |
| 'URL': 'https://www.chromium.org', |
| 'Shipped': 'yes', |
| 'License File': [os.path.join(_REPOSITORY_ROOT, 'LICENSE')], |
| } |
| entries.append( |
| MetadataToTemplateEntry(chromium_license_metadata, entry_template)) |
| |
| entries_by_name = {} |
| for path in third_party_dirs: |
| try: |
| # Directory metadata can be for multiple dependencies. |
| directory_metadata, _ = ParseDir(path, |
| _REPOSITORY_ROOT, |
| enable_warnings=enable_warnings) |
| if not directory_metadata: |
| continue |
| except LicenseError: |
| # TODO(phajdan.jr): Convert to fatal error (https://crbug.com/39240). |
| continue |
| |
| for dep_metadata in directory_metadata: |
| if dep_metadata['Shipped'] == NO: |
| continue |
| if target_os == 'ios' and not gn_target: |
| # Skip over files that are known not to be used on iOS. But |
| # skipping is unnecessary if GN was used to query the actual |
| # dependencies. |
| # TODO(lambroslambrou): Remove this step once the iOS build is |
| # updated to provide --gn-target to this script. |
| if path in KNOWN_NON_IOS_LIBRARIES: |
| continue |
| |
| new_entry = MetadataToTemplateEntry(dep_metadata, entry_template) |
| # Skip entries that we've already seen (it exists in multiple |
| # directories). |
| prev_entry = entries_by_name.setdefault(new_entry['name'], new_entry) |
| if prev_entry is not new_entry and (prev_entry['content'] |
| == new_entry['content']): |
| continue |
| |
| entries.append(new_entry) |
| |
| entries.sort(key=lambda entry: (entry['name'].lower(), entry['content'])) |
| for entry_id, entry in enumerate(entries): |
| entry['content'] = entry['content'].replace('{{id}}', str(entry_id)) |
| |
| entries_contents = '\n'.join([entry['content'] for entry in entries]) |
| |
| reciprocal_template = codecs.open(reciprocal_template_file, |
| encoding='utf-8').read() |
| reciprocal_contents = EvaluateTemplate(reciprocal_template, { |
| 'opensource_project': 'Chromium', |
| 'opensource_link': 'https://source.chromium.org/chromium' |
| }, |
| escape=False) |
| |
| file_template = codecs.open(file_template_file, encoding='utf-8').read() |
| template_contents = "<!-- Generated by licenses.py; do not edit. -->" |
| template_contents += EvaluateTemplate(file_template, { |
| 'entries': entries_contents, |
| 'reciprocal-license-statement': reciprocal_contents |
| }, |
| escape=False) |
| |
| if output_file: |
| changed = True |
| try: |
| old_output = codecs.open(output_file, 'r', encoding='utf-8').read() |
| if old_output == template_contents: |
| changed = False |
| except: |
| pass |
| if changed: |
| with codecs.open(output_file, 'w', encoding='utf-8') as output: |
| output.write(template_contents) |
| else: |
| print(template_contents) |
| |
| if depfile: |
| assert output_file |
| # Add in build.ninja so that the target will be considered dirty whenever |
| # gn gen is run. Otherwise, it will fail to notice new files being added. |
| # This is still no perfect, as it will fail if no build files are changed, |
| # but a new README.chromium / LICENSE is added. This shouldn't happen in |
| # practice however. |
| license_file_list = [] |
| for entry in entries: |
| license_file_list.extend(entry['license_file']) |
| license_file_list = (os.path.relpath(p) for p in license_file_list) |
| license_file_list = sorted(set(license_file_list)) |
| action_helpers.write_depfile(depfile, output_file, |
| license_file_list + ['build.ninja']) |
| |
| return True |
| |
| |
| def GenerateLicenseFile(args: argparse.Namespace, root_dir=_REPOSITORY_ROOT): |
| """Top level function for the license file generation. |
| |
| Either saves the text to a file or prints it to stdout depending on if |
| args.output_file is set. |
| |
| Args: |
| args: parsed arguments passed from the cli |
| root_dir: The root directory to start the third party search |
| """ |
| # Convert the path separators to the OS specific ones |
| extra_third_party_dirs = args.extra_third_party_dirs |
| if extra_third_party_dirs is not None: |
| extra_third_party_dirs = [ |
| os.path.normpath(path) for path in extra_third_party_dirs |
| ] |
| |
| if args.gn_target is not None: |
| third_party_dirs = FindThirdPartyDeps(args.gn_out_dir, args.gn_target, |
| args.target_os, |
| extra_third_party_dirs, |
| args.extra_allowed_dirs) |
| |
| # Sanity-check to raise a build error if invalid gn_... settings are |
| # somehow passed to this script. |
| if not third_party_dirs: |
| raise RuntimeError("No deps found.") |
| |
| else: |
| third_party_dirs = FindThirdPartyDirs(root_dir, extra_third_party_dirs) |
| |
| metadatas = {} |
| for d in third_party_dirs: |
| try: |
| directory_metadata, errors = ParseDir( |
| d, |
| root_dir, |
| require_license_file=True, |
| enable_warnings=args.enable_warnings) |
| if directory_metadata: |
| metadatas[d] = directory_metadata |
| except LicenseError as lic_exp: |
| # TODO(phajdan.jr): Convert to fatal error (https://crbug.com/39240). |
| if args.enable_warnings: |
| print(f"Error: {lic_exp}") |
| continue |
| |
| if args.enable_warnings: |
| LogParseDirErrors(errors) |
| |
| if args.format == 'spdx': |
| license_txt = GenerateLicenseFileSpdx(metadatas, args.spdx_link, |
| args.spdx_root, args.spdx_doc_name, |
| args.spdx_doc_namespace) |
| elif args.format == 'txt': |
| license_txt = GenerateLicenseFilePlainText(metadatas) |
| |
| elif args.format == 'csv': |
| license_txt = GenerateLicenseFileCsv(metadatas) |
| |
| elif args.format == 'notice': |
| license_txt = GenerateNoticeFilePlainText(metadatas) |
| |
| else: |
| raise ValueError(f'Unknown license format: {args.format}') |
| |
| if args.output_file: |
| with open(args.output_file, 'w', encoding='utf-8') as f: |
| f.write(license_txt) |
| else: |
| print(license_txt) |
| |
| |
| def GenerateLicenseFileCsv( |
| metadata: Dict[str, List[Dict[str, Any]]], |
| repo_root: str = _REPOSITORY_ROOT, |
| ) -> str: |
| """Generates a CSV formatted file which contains license data to be used as |
| part of the submission to the Open Source Licensing Review process. |
| """ |
| third_party_data = [] |
| |
| # Collect all the metadata we want to write out and sort it so that the |
| # resulting CSV is ordered by dependency name |
| for data in metadata.values(): |
| third_party_data.extend(data) |
| third_party_data.sort(key=lambda item: item['Name']) |
| |
| csv_content = io.StringIO() |
| csv_writer = csv.writer(csv_content, quoting=csv.QUOTE_NONNUMERIC) |
| |
| # These values are applied statically to all dependencies which are included |
| # in the exported CSV. |
| # Static fields: |
| # * Name of binary which uses dependency, |
| # * License text for library included in product, |
| # * Mirrored source for reciprocal licences. |
| # * Signoff date. |
| static_data = ["Chromium", "Yes", "Yes", "N/A"] |
| |
| # Add informative CSV header row to make it clear which columns represent |
| # which data in the review spreadsheet. |
| csv_writer.writerow([ |
| "Library Name", "Link to LICENSE file", "License Name", |
| "Binary which uses library", "License text for library included?", |
| "Source code for library includes the mirrored source?", |
| "Authorization date" |
| ]) |
| |
| # Include Chromium at the top of the file as it's the main artifact. |
| csv_writer.writerow([ |
| "Chromium", |
| "https://source.chromium.org/chromium/chromium/src/+/main:LICENSE", |
| "BSD 3-Clause" |
| ] + static_data) |
| |
| for dep_metadata in third_party_data: |
| # Only third party libraries which are shipped as part of a final |
| # product are in scope for license review. |
| if dep_metadata['Shipped'] == NO: |
| continue |
| |
| data_row = [dep_metadata['Name'] or "UNKNOWN"] |
| |
| urls = [] |
| for lic in dep_metadata['License File']: |
| # The review process requires that a link is provided to each license |
| # which is included. We can achieve this by combining a static |
| # Chromium googlesource URL with the relative path to the license |
| # file from the top level Chromium src directory. |
| lic_url = ("https://source.chromium.org/chromium/chromium/src/+/main:" + |
| os.path.relpath(lic, repo_root)) |
| |
| # Since these are URLs and not paths, replace any Windows path `\` |
| # separators with a `/` |
| urls.append(lic_url.replace("\\", "/")) |
| |
| data_row.append(", ".join(urls) or "UNKNOWN") |
| data_row.append(dep_metadata["License"] or "UNKNOWN") |
| |
| # Join the default data which applies to each row |
| csv_writer.writerow(data_row + static_data) |
| |
| return csv_content.getvalue() |
| |
| |
| def GenerateNoticeFilePlainText( |
| metadata: Dict[str, List[Dict[str, Any]]], |
| repo_root: str = _REPOSITORY_ROOT, |
| read_file=lambda x: pathlib.Path(x).read_text(encoding='utf-8') |
| ) -> str: |
| """Generate a plain-text THIRD_PARTY_NOTICE file which can be used when you |
| ship a part of Chromium code (specified by gn_target) as a stand-alone |
| library (e.g., //ios/web_view). |
| |
| The THIRD_PARTY_NOTICE file contains licenses of third-party libraries |
| which gn_target depends on. Each notice contains the following information: |
| * The name of the component. |
| * Identification of the component's license. |
| * The complete text of every unique license (at least once) |
| """ |
| # Start with Chromium's THIRD_PARTY_NOTICE file. |
| content = [] |
| # Add necessary third_party. |
| for directory in sorted(metadata): |
| dir_metadata = metadata[directory] |
| for dep_metadata in dir_metadata: |
| shipped = dep_metadata['Shipped'] |
| license_files = dep_metadata['License File'] |
| if shipped == YES and license_files: |
| for license_file in license_files: |
| content.append('-' * 20) |
| content.append('License notice for %s' % dep_metadata["Name"]) |
| content.append('-' * 20) |
| content.append(read_file(os.path.join(repo_root, license_file))) |
| |
| return '\n'.join(content) |
| |
| |
| def GenerateLicenseFilePlainText( |
| metadata: Dict[str, List[Dict[str, Any]]], |
| repo_root: str = _REPOSITORY_ROOT, |
| read_file=lambda x: pathlib.Path(x).read_text(encoding='utf-8') |
| ) -> str: |
| """Generate a plain-text LICENSE file which can be used when you ship a part |
| of Chromium code (specified by gn_target) as a stand-alone library |
| (e.g., //ios/web_view). |
| |
| The LICENSE file contains licenses of both Chromium and third-party |
| libraries which gn_target depends on. """ |
| # Start with Chromium's LICENSE file. |
| content = [read_file(os.path.join(repo_root, 'LICENSE'))] |
| |
| # Add necessary third_party. |
| for directory in sorted(metadata): |
| dir_metadata = metadata[directory] |
| for dep_metadata in dir_metadata: |
| shipped = dep_metadata['Shipped'] |
| license_files = dep_metadata['License File'] |
| if shipped == YES and license_files: |
| content.append('-' * 20) |
| content.append(dep_metadata["Name"]) |
| content.append('-' * 20) |
| for license_file in license_files: |
| content.append(read_file(os.path.join(repo_root, license_file))) |
| |
| return '\n'.join(content) |
| |
| |
| def GenerateLicenseFileSpdx( |
| metadata: Dict[str, List[Dict[str, Any]]], |
| spdx_link_prefix: str, |
| spdx_root: str, |
| doc_name: Optional[str], |
| doc_namespace: Optional[str], |
| repo_root: str = _REPOSITORY_ROOT, |
| read_file=lambda x: pathlib.Path(x).read_text(encoding='utf-8') |
| ) -> str: |
| """Generates a LICENSE file in SPDX format. |
| |
| The SPDX output contains the following elements: |
| |
| 1. SPDX Document. |
| 2. SPDX root Package. |
| 3. An SPDX Package for each package that has license files. |
| 4. An SPDX License element for each license file. |
| |
| The output is based on the following specification: |
| https://spdx.github.io/spdx-spec/v2-draft/ |
| """ |
| root_license = os.path.join(repo_root, 'LICENSE') |
| spdx_writer = SpdxWriter(spdx_root, |
| 'Chromium', |
| root_license, |
| spdx_link_prefix, |
| doc_name=doc_name, |
| doc_namespace=doc_namespace, |
| read_file=read_file) |
| |
| # Add all third party libraries |
| for directory in sorted(metadata): |
| dir_metadata = metadata[directory] |
| for dep_metadata in dir_metadata: |
| shipped = dep_metadata['Shipped'] |
| license_files = dep_metadata['License File'] |
| if shipped == YES and license_files: |
| for license_file in license_files: |
| license_path = os.path.join(repo_root, license_file) |
| spdx_writer.add_package(dep_metadata['Name'], license_path) |
| |
| return spdx_writer.write() |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--file-template', |
| help='Template HTML to use for the license page.') |
| parser.add_argument('--entry-template', |
| help='Template HTML to use for each license.') |
| parser.add_argument('--reciprocal-template', |
| help=('Template HTML to use for adding a link to an open ' |
| 'source code repository to satisfy reciprocal ' |
| 'license requirements. eg Chromium.')) |
| parser.add_argument( |
| '--extra-third-party-dirs', |
| help='Gn list of additional third_party dirs to look through.') |
| parser.add_argument( |
| '--extra-allowed-dirs', |
| help=('List of extra allowed paths to look for ' |
| '(deps that will be picked up automatically) besides third_party'), |
| ) |
| parser.add_argument('--target-os', help='OS that this build is targeting.') |
| parser.add_argument('--gn-out-dir', |
| help='GN output directory for scanning dependencies.') |
| parser.add_argument('--gn-target', help='GN target to scan for dependencies.') |
| parser.add_argument('--format', |
| default='txt', |
| choices=['txt', 'spdx', 'csv', 'notice'], |
| help='What format to output in') |
| parser.add_argument('--spdx-root', |
| default=_REPOSITORY_ROOT, |
| help=('Use a different root for license refs than ' + |
| _REPOSITORY_ROOT)) |
| parser.add_argument( |
| '--spdx-link', |
| default='https://source.chromium.org/chromium/chromium/src/+/main:', |
| help='Link prefix for license cross ref') |
| parser.add_argument('--spdx-doc-name', |
| help='Specify a document name for the SPDX doc') |
| parser.add_argument( |
| '--spdx-doc-namespace', |
| default='https://chromium.googlesource.com/chromium/src/tools/', |
| help='Specify the document namespace for the SPDX doc') |
| parser.add_argument( |
| '--enable-warnings', |
| action='store_true', |
| help='Enables warning logs when processing directory metadata for ' |
| 'credits or license file generation.') |
| parser.add_argument('command', |
| choices=['help', 'scan', 'credits', 'license_file']) |
| parser.add_argument('output_file', nargs='?') |
| action_helpers.add_depfile_arg(parser) |
| args = parser.parse_args() |
| args.extra_third_party_dirs = action_helpers.parse_gn_list( |
| args.extra_third_party_dirs) |
| args.extra_allowed_dirs = action_helpers.parse_gn_list( |
| args.extra_allowed_dirs) |
| |
| if args.command == 'scan': |
| if not ScanThirdPartyDirs(): |
| return 1 |
| elif args.command == 'credits': |
| if not GenerateCredits( |
| args.file_template, args.entry_template, args.reciprocal_template, |
| args.output_file, args.target_os, args.gn_out_dir, args.gn_target, |
| args.extra_third_party_dirs, args.depfile, args.enable_warnings): |
| return 1 |
| elif args.command == 'license_file': |
| try: |
| GenerateLicenseFile(args) |
| except LicenseError as e: |
| print("Failed to parse README.chromium: {}".format(e)) |
| return 1 |
| else: |
| print(__doc__) |
| return 1 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
