Add script to generate ebuilds from metadata

`cargo metadata` gives us a list of required packages, their versions
and their dependencies. We can use this to generate the ebuilds needed
to support the current Cargo.lock entries.

BUG=b:176846220
TEST=Generated all ebuilds for systembt. See crrev/c/2770279

Change-Id: I6bf9863dc7805a1dc8c83b8b3ebd28b8abfc37aa
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2766249
Reviewed-by: Allen Webb <allenwebb@google.com>
Commit-Queue: Abhishek Pandit-Subedi <abhishekpandit@chromium.org>
Tested-by: Abhishek Pandit-Subedi <abhishekpandit@chromium.org>
diff --git a/contrib/cargo2ebuild.py b/contrib/cargo2ebuild.py
new file mode 100755
index 0000000..6a76cee
--- /dev/null
+++ b/contrib/cargo2ebuild.py
@@ -0,0 +1,497 @@
+#!/usr/bin/env python3
+# pylint: disable=line-too-long, subprocess-run-check, unused-argument
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Convert Cargo metadata to ebuilds using cros-rust.
+
+Used to add new Rust projects and building up ebuilds for the dependency tree.
+"""
+
+import argparse
+from datetime import datetime
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+SCRIPT_NAME = 'cargo2ebuild.py'
+AUTOGEN_NOTICE = '\n# This file was automatically generated by {}'.format(
+    SCRIPT_NAME)
+
+CRATE_DL_URI = 'https://crates.io/api/v1/crates/{PN}/{PV}/download'
+
+# Required parameters
+#   copyright_year: Current year for copyright assignment.
+#   description: Description of the crates.
+#   homepage: Homepage of the crates.
+#   license: Ebuild compatible license string.
+#   dependencies: Ebuild compatible dependency string.
+#   autogen_notice: Autogenerated notification string.
+EBUILD_TEMPLATE = \
+"""# Copyright {copyright_year} The Chromium OS Authors. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="7"
+
+CROS_RUST_REMOVE_DEV_DEPS=1
+
+inherit cros-rust
+
+DESCRIPTION="{description}"
+HOMEPAGE="{homepage}"
+SRC_URI="https://crates.io/api/v1/crates/${{PN}}/${{PV}}/download -> ${{P}}.crate"
+
+LICENSE="{license}"
+SLOT="${{PV}}/${{PR}}"
+KEYWORDS="*"
+
+{dependencies}{autogen_notice}
+"""
+
+# Required parameters:
+#   copyright_year: Current year for copyright assignment.
+#   crate_features: Features to add to this empty crate.
+#   autogen_notice: Autogenerated notification string.
+EMPTY_CRATE = \
+"""# Copyright {copyright_year} The Chromium OS Authors. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="7"
+
+CROS_RUST_EMPTY_CRATE=1
+{crate_features}
+inherit cros-rust
+
+DESCRIPTION="Empty crate"
+HOMEPAGE=""
+
+LICENSE="BSD-Google"
+SLOT="${{PV}}/${{PR}}"
+KEYWORDS="*"
+{autogen_notice}
+"""
+
+LICENSES = {
+    'Apache-2.0': 'Apache-2.0',
+    'MIT': 'MIT',
+    'BSD-3-Clause': 'BSD',
+    '0BSD': '0BSD',
+    'ISC': 'ISC',
+}
+
+VERSION_RE = (
+    '^(?P<dep>[\\^~=])?'  # Dependency type: ^, ~, =
+    '(?P<major>[0-9]+|[*])'  # Major version (can be *)
+    '(.(?P<minor>[0-9]+|[*]))?'  # Minor version
+    '(.(?P<patch>[0-9]+|[*]))?'  # Patch version
+    '([+\\-].*)?$'  # Any semver values beyond patch version
+)
+
+
+def prepare_staging(args):
+    """Prepare staging directory."""
+    sdir = args.staging_dir
+    dirs = [
+        os.path.join(sdir, 'ebuild', 'dev-rust'),
+        os.path.join(sdir, 'crates')
+    ]
+    for d in dirs:
+        os.makedirs(d, exist_ok=True)
+
+
+def load_metadata(manifest_path):
+    """Run cargo metadata and get metadata for build."""
+    cwd = os.path.dirname(manifest_path)
+    cmd = [
+        'cargo', 'metadata', '--format-version', '1', '--manifest-path',
+        manifest_path
+    ]
+    output = subprocess.check_output(cmd, cwd=cwd)
+
+    return json.loads(output)
+
+
+def get_crate_path(package, staging_dir):
+    """Get path to crate in staging directory."""
+    return os.path.join(
+        staging_dir, 'crates', '{}-{}.crate'.format(package['name'],
+                                                    package['version']))
+
+
+def get_clean_crate_name(package):
+    """Clean up crate name to {name}-{major}.{minor}.{patch}."""
+    return '{}-{}.crate'.format(package['name'],
+                                get_clean_package_version(package))
+
+
+def version_to_tuple(name, version, missing=-1):
+    """Extract dependency type and semver from a given version string."""
+    def version_to_int(num):
+        if not num or num == '*':
+            return missing
+        return int(num)
+
+    m = re.match(VERSION_RE, version)
+    if not m:
+        print('{} failed to parse dep version: {}'.format(name, version))
+
+    dep = m.group('dep')
+    major = m.group('major')
+    minor = m.group('minor')
+    patch = m.group('patch')
+
+    has_star = any([x == '*' for x in [major, minor, patch]])
+
+    major = version_to_int(major)
+    minor = version_to_int(minor)
+    patch = version_to_int(patch)
+
+    if has_star:
+        dep = '~'
+    elif not dep:
+        dep = '^'
+
+    return (dep, major, minor, patch)
+
+
+def get_clean_package_version(package):
+    """Get package version in the format {major}.{minor}.{patch}."""
+    (_, major, minor, patch) = version_to_tuple(package['name'],
+                                                package['version'],
+                                                missing=0)
+    return '{}.{}.{}'.format(major, minor, patch)
+
+
+def get_ebuild_path(package, staging_dir, make_dir=False):
+    """Get path to ebuild in given directory."""
+    ebuild_path = os.path.join(
+        staging_dir, 'dev-rust', package['name'],
+        '{}-{}.ebuild'.format(package['name'],
+                              get_clean_package_version(package)))
+
+    if make_dir:
+        os.makedirs(os.path.dirname(ebuild_path), exist_ok=True)
+
+    return ebuild_path
+
+
+def download_package(package, staging_dir):
+    """Download the crate from crates.io."""
+    dl_uri = CRATE_DL_URI.format(PN=package['name'], PV=package['version'])
+    crate_path = get_crate_path(package, staging_dir)
+
+    # Already downloaded previously
+    if os.path.isfile(crate_path):
+        return
+
+    ret = subprocess.run(['curl', '-L', dl_uri, '-o', crate_path],
+                   stdout=subprocess.DEVNULL,
+                   stderr=subprocess.DEVNULL).returncode
+
+    if ret:
+        print('{} failed to download: {}'.format(dl_uri, ret))
+
+
+def get_description(package):
+    """Get a description of the crate from metadata."""
+    # pylint: disable=invalid-string-quote
+    if package.get('description', None):
+        desc = package['description'].replace('`', '\'').replace('"', '\'')
+        return desc.rstrip('\n')
+
+    return ''
+
+
+def get_homepage(package):
+    """Get the homepage of the crate from metadata or use crates.io."""
+    if package.get('homepage', None):
+        return package['homepage']
+
+    return 'https://crates.io/crates/{}'.format(package['name'])
+
+
+def convert_license(cargo_license, package):
+    """Convert licenses from cargo to a format usable in ebuilds."""
+    cargo_license = '' if not cargo_license else cargo_license
+    has_or = ' OR ' in cargo_license
+    delim = ' OR ' if has_or else '/'
+
+    found = cargo_license.split(delim)
+    licenses_or = []
+    for f in found:
+        if f in LICENSES:
+            licenses_or.append(LICENSES[f])
+
+    if not licenses_or:
+        print('{} is missing an appropriate license: {}'.format(
+            package['name'], license))
+        return "$(die 'Please replace with appropriate license')"
+
+    if len(licenses_or) > 1:
+        lstr = '|| ( {} )'.format(' '.join(licenses_or))
+    else:
+        lstr = licenses_or[0]
+
+    return lstr
+
+
+def convert_dependencies(dependencies, package, optional_packages):
+    """Convert dependencies from metadata into the ebuild format."""
+    def caret_to_ebuild(info):
+        prefix = '>=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+        suffix = '<dev-rust/{name}-{major_1}'.format(**info)
+
+        if info['minor'] == -1:
+            prefix = '>=dev-rust/{name}-{major}:='.format(**info)
+        elif info['patch'] == -1:
+            prefix = '>=dev-rust/{name}-{major}.{minor}:='.format(**info)
+
+        if info['major'] == 0:
+            suffix = '<dev-rust/{name}-{major}.{minor_1}'.format(**info)
+
+        return '{} {}'.format(prefix, suffix)
+
+    def tilde_to_ebuild(info):
+        prefix = '>=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+        suffix = '<dev-rust/{name}-{major}.{minor_1}'.format(**info)
+        ebuild = '{} {}'.format(prefix, suffix)
+
+        if info['minor'] == -1:
+            ebuild = '=dev-rust/{name}-{major}*:='.format(**info)
+        elif info['patch'] == -1:
+            ebuild = '=dev-rust/{name}-{major}.{minor}*:='.format(**info)
+
+        return ebuild
+
+    def dep_to_ebuild(name, dep_type, major, minor, patch):
+        info = {
+            'name': name,
+            'major': major,
+            'minor': minor,
+            'patch': patch,
+            'major_1': major + 1,
+            'minor_1': minor + 1,
+            'patch_1': patch + 1
+        }
+
+        if dep_type == '^':
+            return caret_to_ebuild(info)
+        if dep_type == '~':
+            return tilde_to_ebuild(info)
+
+        # Remaining dep type is =
+        return '=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+
+    deps = []
+    for dep in dependencies:
+        # Skip all dev dependencies. We will still handle normal and build deps.
+        if dep.get('kind', None) == 'dev':
+            continue
+
+        # Optional dependencies get empty packages created
+        if dep.get('optional', None):
+            optional_packages[dep['name']] = {
+                    'name': dep['name'],
+                    'version': dep['req'],
+                    'features': dep['features'],
+            }
+
+        # Convert requirement to version tuple
+        (deptype, major, minor, patch) = version_to_tuple(dep['name'], dep['req'])
+        ebuild = dep_to_ebuild(dep['name'], deptype, major, minor, patch)
+        deps.append('\t{}'.format(ebuild))
+
+    if not deps:
+        return ''
+
+    # Add DEPEND= into template with all dependencies
+    # RDEPEND="${DEPEND}" required for race in cros-rust
+    fmtstring = 'DEPEND="\n{}\n"\nRDEPEND="${{DEPEND}}"\n'
+    return fmtstring.format('\n'.join(deps))
+
+
+def package_ebuild(package, ebuild_dir, optional_packages):
+    """Create ebuild from metadata and write to ebuild directory."""
+    ebuild_path = get_ebuild_path(package, ebuild_dir, make_dir=True)
+
+    autogen_notice = AUTOGEN_NOTICE
+
+    # Check if version matches clean version or modify the autogen notice
+    if package['version'] != get_clean_package_version(package):
+        autogen_notice = '\n'.join([
+            autogen_notice,
+            '# ${{PV}} was changed from the original {}'.format(
+                package['version'])
+        ])
+
+    dependencies = convert_dependencies(package['dependencies'], package,
+                                        optional_packages)
+    template_info = {
+        'copyright_year': datetime.now().year,
+        'description': get_description(package),
+        'homepage': get_homepage(package),
+        'license': convert_license(package['license'], package),
+        'dependencies': dependencies,
+        'autogen_notice': autogen_notice,
+    }
+
+    with open(ebuild_path, 'w') as ebuild:
+        ebuild.write(EBUILD_TEMPLATE.format(**template_info))
+
+
+def upload_gsutil(package, staging_dir, no_upload=False):
+    """Upload crate to distfiles."""
+    if no_upload:
+        return
+
+    crate_path = get_crate_path(package, staging_dir)
+    crate_name = get_clean_crate_name(package)
+    ret = subprocess.run([
+        'gsutil', 'cp', '-a', 'public-read', crate_path,
+        'gs://chromeos-localmirror/distfiles/{}'.format(crate_name)
+    ]).returncode
+
+    if ret:
+        print('{} failed to upload to chromeos-localmirror: {}'.format(
+            crate_name, ret))
+
+
+def update_ebuild(package, ebuild_dir, target_dir):
+    """Update ebuild with generated one and generate MANIFEST."""
+    ebuild_src = get_ebuild_path(package, ebuild_dir)
+    ebuild_dest = get_ebuild_path(package, target_dir, make_dir=True)
+
+    shutil.copy(ebuild_src, ebuild_dest)
+
+    # Generate manifest w/ ebuild digest
+    ret = subprocess.run(['ebuild', ebuild_dest, 'digest']).returncode
+    if ret:
+        print('ebuild {} digest failed: {}'.format(ebuild_dest, ret))
+
+
+def process_package(package, args, optional_packages):
+    """Process each package listed in the metadata."""
+    staging_dir = args.staging_dir
+    ebuild_dir = os.path.join(staging_dir, 'ebuild')
+    target_dir = args.target_dir
+
+    download_package(package, staging_dir)
+    package_ebuild(package, ebuild_dir, optional_packages)
+
+    if not args.dry_run and package['name'] not in args.skip:
+        upload_gsutil(package, staging_dir, no_upload=args.no_upload)
+        update_ebuild(package, ebuild_dir, target_dir)
+
+def process_empty_package(empty_package, args):
+    """Process packages that should generate empty ebuilds."""
+    staging_dir = args.staging_dir
+    ebuild_dir = os.path.join(staging_dir, 'ebuild')
+    target_dir = args.target_dir
+
+    ebuild_src = get_ebuild_path(empty_package, ebuild_dir, make_dir=True)
+    ebuild_dest = get_ebuild_path(empty_package, target_dir, make_dir=True)
+
+    crate_features = ''
+    if empty_package.get('features', None):
+        crate_features = 'CROS_RUST_EMPTY_CRATE_FEATURES=( {} )'.format(
+            ' '.join(['"{}"'.format(x) for x in empty_package['features']]))
+
+    template_info = {
+        'copyright_year': datetime.now().year,
+        'crate_features': crate_features,
+        'autogen_notice': AUTOGEN_NOTICE,
+    }
+
+    with open(ebuild_src, 'w') as ebuild:
+        ebuild.write(EMPTY_CRATE.format(**template_info))
+
+    if not args.dry_run and empty_package['name'] not in args.skip:
+        shutil.copy(ebuild_src, ebuild_dest)
+
+def check_if_package_is_required(empty_package, args):
+    """Check if an empty package already has a non-empty ebuild."""
+    ebuild_dest = get_ebuild_path(empty_package, args.target_dir, make_dir=False)
+    ebuild_target_dir = os.path.dirname(ebuild_dest)
+
+    # Find all ebuilds in ebuild_target dir and confirm they have
+    # CROS_RUST_EMPTY_CRATE in them. If they do not, they need to be manually
+    # handled.
+    for root, _, files in os.walk(ebuild_target_dir):
+        for efile in files:
+            if not efile.endswith('.ebuild'):
+                continue
+
+            with open(os.path.join(root, efile), 'r') as ebuild:
+                if 'CROS_RUST_EMPTY_CRATE' not in ebuild.read():
+                    print('{} was not an empty crate.'.format(efile))
+                    return True
+
+    return False
+
+def main(argv):
+    """Convert dependencies from Cargo.toml into ebuilds."""
+    args = parse_args(argv)
+
+    prepare_staging(args)
+
+    processed_packages = {}
+    optional_packages = {}
+    metadata = load_metadata(args.manifest_path)
+    for p in metadata['packages']:
+        process_package(p, args, optional_packages)
+        processed_packages[p['name']] = True
+
+    for key in optional_packages:
+        if key not in processed_packages and not check_if_package_is_required(
+                optional_packages[key], args):
+            process_empty_package(optional_packages[key], args)
+
+
+def parse_args(argv):
+    """Parse the command-line arguments."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '-s',
+        '--staging-dir',
+        required=True,
+        help='Staging directory for temporary and generated files')
+
+    parser.add_argument(
+        '-d',
+        '--dry-run',
+        action='store_true',
+        help='Generate dependency tree but do not upload anything')
+
+    parser.add_argument('-t',
+                        '--target-dir',
+                        help='Path to chromiumos-overlay')
+
+    parser.add_argument('-k',
+                        '--skip',
+                        action='append',
+                        help='Skip these packages when updating ebuilds')
+    parser.add_argument('-n',
+                        '--no-upload',
+                        action='store_true',
+                        help='Skip uploading crates to distfiles')
+
+    parser.add_argument('manifest_path',
+                        help='Cargo.toml used to generate ebuilds.')
+
+    args = parser.parse_args(argv)
+
+    # Require target directory if not dry run
+    if not args.target_dir and not args.dry_run:
+        raise Exception('Target directory must be set unless dry-run is True.')
+
+    if not args.skip:
+        args.skip = []
+
+    return args
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))