Add script to generate ebuilds from metadata
`cargo metadata` gives us a list of required packages, their versions
and their dependencies. We can use this to generate the ebuilds needed
to support the current Cargo.lock entries.
BUG=b:176846220
TEST=Generated all ebuilds for systembt. See crrev/c/2770279
Change-Id: I6bf9863dc7805a1dc8c83b8b3ebd28b8abfc37aa
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2766249
Reviewed-by: Allen Webb <allenwebb@google.com>
Commit-Queue: Abhishek Pandit-Subedi <abhishekpandit@chromium.org>
Tested-by: Abhishek Pandit-Subedi <abhishekpandit@chromium.org>
diff --git a/contrib/cargo2ebuild.py b/contrib/cargo2ebuild.py
new file mode 100755
index 0000000..6a76cee
--- /dev/null
+++ b/contrib/cargo2ebuild.py
@@ -0,0 +1,497 @@
+#!/usr/bin/env python3
+# pylint: disable=line-too-long, subprocess-run-check, unused-argument
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Convert Cargo metadata to ebuilds using cros-rust.
+
+Used to add new Rust projects and building up ebuilds for the dependency tree.
+"""
+
+import argparse
+from datetime import datetime
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+SCRIPT_NAME = 'cargo2ebuild.py'
+AUTOGEN_NOTICE = '\n# This file was automatically generated by {}'.format(
+ SCRIPT_NAME)
+
+CRATE_DL_URI = 'https://crates.io/api/v1/crates/{PN}/{PV}/download'
+
+# Required parameters
+# copyright_year: Current year for copyright assignment.
+# description: Description of the crates.
+# homepage: Homepage of the crates.
+# license: Ebuild compatible license string.
+# dependencies: Ebuild compatible dependency string.
+# autogen_notice: Autogenerated notification string.
+EBUILD_TEMPLATE = \
+"""# Copyright {copyright_year} The Chromium OS Authors. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="7"
+
+CROS_RUST_REMOVE_DEV_DEPS=1
+
+inherit cros-rust
+
+DESCRIPTION="{description}"
+HOMEPAGE="{homepage}"
+SRC_URI="https://crates.io/api/v1/crates/${{PN}}/${{PV}}/download -> ${{P}}.crate"
+
+LICENSE="{license}"
+SLOT="${{PV}}/${{PR}}"
+KEYWORDS="*"
+
+{dependencies}{autogen_notice}
+"""
+
+# Required parameters:
+# copyright_year: Current year for copyright assignment.
+# crate_features: Features to add to this empty crate.
+# autogen_notice: Autogenerated notification string.
+EMPTY_CRATE = \
+"""# Copyright {copyright_year} The Chromium OS Authors. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="7"
+
+CROS_RUST_EMPTY_CRATE=1
+{crate_features}
+inherit cros-rust
+
+DESCRIPTION="Empty crate"
+HOMEPAGE=""
+
+LICENSE="BSD-Google"
+SLOT="${{PV}}/${{PR}}"
+KEYWORDS="*"
+{autogen_notice}
+"""
+
+LICENSES = {
+ 'Apache-2.0': 'Apache-2.0',
+ 'MIT': 'MIT',
+ 'BSD-3-Clause': 'BSD',
+ '0BSD': '0BSD',
+ 'ISC': 'ISC',
+}
+
+VERSION_RE = (
+ '^(?P<dep>[\\^~=])?' # Dependency type: ^, ~, =
+ '(?P<major>[0-9]+|[*])' # Major version (can be *)
+ '(.(?P<minor>[0-9]+|[*]))?' # Minor version
+ '(.(?P<patch>[0-9]+|[*]))?' # Patch version
+ '([+\\-].*)?$' # Any semver values beyond patch version
+)
+
+
+def prepare_staging(args):
+ """Prepare staging directory."""
+ sdir = args.staging_dir
+ dirs = [
+ os.path.join(sdir, 'ebuild', 'dev-rust'),
+ os.path.join(sdir, 'crates')
+ ]
+ for d in dirs:
+ os.makedirs(d, exist_ok=True)
+
+
+def load_metadata(manifest_path):
+ """Run cargo metadata and get metadata for build."""
+ cwd = os.path.dirname(manifest_path)
+ cmd = [
+ 'cargo', 'metadata', '--format-version', '1', '--manifest-path',
+ manifest_path
+ ]
+ output = subprocess.check_output(cmd, cwd=cwd)
+
+ return json.loads(output)
+
+
+def get_crate_path(package, staging_dir):
+ """Get path to crate in staging directory."""
+ return os.path.join(
+ staging_dir, 'crates', '{}-{}.crate'.format(package['name'],
+ package['version']))
+
+
+def get_clean_crate_name(package):
+ """Clean up crate name to {name}-{major}.{minor}.{patch}."""
+ return '{}-{}.crate'.format(package['name'],
+ get_clean_package_version(package))
+
+
+def version_to_tuple(name, version, missing=-1):
+ """Extract dependency type and semver from a given version string."""
+ def version_to_int(num):
+ if not num or num == '*':
+ return missing
+ return int(num)
+
+ m = re.match(VERSION_RE, version)
+ if not m:
+ print('{} failed to parse dep version: {}'.format(name, version))
+
+ dep = m.group('dep')
+ major = m.group('major')
+ minor = m.group('minor')
+ patch = m.group('patch')
+
+ has_star = any([x == '*' for x in [major, minor, patch]])
+
+ major = version_to_int(major)
+ minor = version_to_int(minor)
+ patch = version_to_int(patch)
+
+ if has_star:
+ dep = '~'
+ elif not dep:
+ dep = '^'
+
+ return (dep, major, minor, patch)
+
+
+def get_clean_package_version(package):
+ """Get package version in the format {major}.{minor}.{patch}."""
+ (_, major, minor, patch) = version_to_tuple(package['name'],
+ package['version'],
+ missing=0)
+ return '{}.{}.{}'.format(major, minor, patch)
+
+
+def get_ebuild_path(package, staging_dir, make_dir=False):
+ """Get path to ebuild in given directory."""
+ ebuild_path = os.path.join(
+ staging_dir, 'dev-rust', package['name'],
+ '{}-{}.ebuild'.format(package['name'],
+ get_clean_package_version(package)))
+
+ if make_dir:
+ os.makedirs(os.path.dirname(ebuild_path), exist_ok=True)
+
+ return ebuild_path
+
+
+def download_package(package, staging_dir):
+ """Download the crate from crates.io."""
+ dl_uri = CRATE_DL_URI.format(PN=package['name'], PV=package['version'])
+ crate_path = get_crate_path(package, staging_dir)
+
+ # Already downloaded previously
+ if os.path.isfile(crate_path):
+ return
+
+ ret = subprocess.run(['curl', '-L', dl_uri, '-o', crate_path],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL).returncode
+
+ if ret:
+ print('{} failed to download: {}'.format(dl_uri, ret))
+
+
+def get_description(package):
+ """Get a description of the crate from metadata."""
+ # pylint: disable=invalid-string-quote
+ if package.get('description', None):
+ desc = package['description'].replace('`', '\'').replace('"', '\'')
+ return desc.rstrip('\n')
+
+ return ''
+
+
+def get_homepage(package):
+ """Get the homepage of the crate from metadata or use crates.io."""
+ if package.get('homepage', None):
+ return package['homepage']
+
+ return 'https://crates.io/crates/{}'.format(package['name'])
+
+
+def convert_license(cargo_license, package):
+ """Convert licenses from cargo to a format usable in ebuilds."""
+ cargo_license = '' if not cargo_license else cargo_license
+ has_or = ' OR ' in cargo_license
+ delim = ' OR ' if has_or else '/'
+
+ found = cargo_license.split(delim)
+ licenses_or = []
+ for f in found:
+ if f in LICENSES:
+ licenses_or.append(LICENSES[f])
+
+ if not licenses_or:
+ print('{} is missing an appropriate license: {}'.format(
+ package['name'], license))
+ return "$(die 'Please replace with appropriate license')"
+
+ if len(licenses_or) > 1:
+ lstr = '|| ( {} )'.format(' '.join(licenses_or))
+ else:
+ lstr = licenses_or[0]
+
+ return lstr
+
+
+def convert_dependencies(dependencies, package, optional_packages):
+ """Convert dependencies from metadata into the ebuild format."""
+ def caret_to_ebuild(info):
+ prefix = '>=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+ suffix = '<dev-rust/{name}-{major_1}'.format(**info)
+
+ if info['minor'] == -1:
+ prefix = '>=dev-rust/{name}-{major}:='.format(**info)
+ elif info['patch'] == -1:
+ prefix = '>=dev-rust/{name}-{major}.{minor}:='.format(**info)
+
+ if info['major'] == 0:
+ suffix = '<dev-rust/{name}-{major}.{minor_1}'.format(**info)
+
+ return '{} {}'.format(prefix, suffix)
+
+ def tilde_to_ebuild(info):
+ prefix = '>=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+ suffix = '<dev-rust/{name}-{major}.{minor_1}'.format(**info)
+ ebuild = '{} {}'.format(prefix, suffix)
+
+ if info['minor'] == -1:
+ ebuild = '=dev-rust/{name}-{major}*:='.format(**info)
+ elif info['patch'] == -1:
+ ebuild = '=dev-rust/{name}-{major}.{minor}*:='.format(**info)
+
+ return ebuild
+
+ def dep_to_ebuild(name, dep_type, major, minor, patch):
+ info = {
+ 'name': name,
+ 'major': major,
+ 'minor': minor,
+ 'patch': patch,
+ 'major_1': major + 1,
+ 'minor_1': minor + 1,
+ 'patch_1': patch + 1
+ }
+
+ if dep_type == '^':
+ return caret_to_ebuild(info)
+ if dep_type == '~':
+ return tilde_to_ebuild(info)
+
+ # Remaining dep type is =
+ return '=dev-rust/{name}-{major}.{minor}.{patch}:='.format(**info)
+
+ deps = []
+ for dep in dependencies:
+ # Skip all dev dependencies. We will still handle normal and build deps.
+ if dep.get('kind', None) == 'dev':
+ continue
+
+ # Optional dependencies get empty packages created
+ if dep.get('optional', None):
+ optional_packages[dep['name']] = {
+ 'name': dep['name'],
+ 'version': dep['req'],
+ 'features': dep['features'],
+ }
+
+ # Convert requirement to version tuple
+ (deptype, major, minor, patch) = version_to_tuple(dep['name'], dep['req'])
+ ebuild = dep_to_ebuild(dep['name'], deptype, major, minor, patch)
+ deps.append('\t{}'.format(ebuild))
+
+ if not deps:
+ return ''
+
+ # Add DEPEND= into template with all dependencies
+ # RDEPEND="${DEPEND}" required for race in cros-rust
+ fmtstring = 'DEPEND="\n{}\n"\nRDEPEND="${{DEPEND}}"\n'
+ return fmtstring.format('\n'.join(deps))
+
+
+def package_ebuild(package, ebuild_dir, optional_packages):
+ """Create ebuild from metadata and write to ebuild directory."""
+ ebuild_path = get_ebuild_path(package, ebuild_dir, make_dir=True)
+
+ autogen_notice = AUTOGEN_NOTICE
+
+ # Check if version matches clean version or modify the autogen notice
+ if package['version'] != get_clean_package_version(package):
+ autogen_notice = '\n'.join([
+ autogen_notice,
+ '# ${{PV}} was changed from the original {}'.format(
+ package['version'])
+ ])
+
+ dependencies = convert_dependencies(package['dependencies'], package,
+ optional_packages)
+ template_info = {
+ 'copyright_year': datetime.now().year,
+ 'description': get_description(package),
+ 'homepage': get_homepage(package),
+ 'license': convert_license(package['license'], package),
+ 'dependencies': dependencies,
+ 'autogen_notice': autogen_notice,
+ }
+
+ with open(ebuild_path, 'w') as ebuild:
+ ebuild.write(EBUILD_TEMPLATE.format(**template_info))
+
+
+def upload_gsutil(package, staging_dir, no_upload=False):
+ """Upload crate to distfiles."""
+ if no_upload:
+ return
+
+ crate_path = get_crate_path(package, staging_dir)
+ crate_name = get_clean_crate_name(package)
+ ret = subprocess.run([
+ 'gsutil', 'cp', '-a', 'public-read', crate_path,
+ 'gs://chromeos-localmirror/distfiles/{}'.format(crate_name)
+ ]).returncode
+
+ if ret:
+ print('{} failed to upload to chromeos-localmirror: {}'.format(
+ crate_name, ret))
+
+
+def update_ebuild(package, ebuild_dir, target_dir):
+ """Update ebuild with generated one and generate MANIFEST."""
+ ebuild_src = get_ebuild_path(package, ebuild_dir)
+ ebuild_dest = get_ebuild_path(package, target_dir, make_dir=True)
+
+ shutil.copy(ebuild_src, ebuild_dest)
+
+ # Generate manifest w/ ebuild digest
+ ret = subprocess.run(['ebuild', ebuild_dest, 'digest']).returncode
+ if ret:
+ print('ebuild {} digest failed: {}'.format(ebuild_dest, ret))
+
+
+def process_package(package, args, optional_packages):
+ """Process each package listed in the metadata."""
+ staging_dir = args.staging_dir
+ ebuild_dir = os.path.join(staging_dir, 'ebuild')
+ target_dir = args.target_dir
+
+ download_package(package, staging_dir)
+ package_ebuild(package, ebuild_dir, optional_packages)
+
+ if not args.dry_run and package['name'] not in args.skip:
+ upload_gsutil(package, staging_dir, no_upload=args.no_upload)
+ update_ebuild(package, ebuild_dir, target_dir)
+
+def process_empty_package(empty_package, args):
+ """Process packages that should generate empty ebuilds."""
+ staging_dir = args.staging_dir
+ ebuild_dir = os.path.join(staging_dir, 'ebuild')
+ target_dir = args.target_dir
+
+ ebuild_src = get_ebuild_path(empty_package, ebuild_dir, make_dir=True)
+ ebuild_dest = get_ebuild_path(empty_package, target_dir, make_dir=True)
+
+ crate_features = ''
+ if empty_package.get('features', None):
+ crate_features = 'CROS_RUST_EMPTY_CRATE_FEATURES=( {} )'.format(
+ ' '.join(['"{}"'.format(x) for x in empty_package['features']]))
+
+ template_info = {
+ 'copyright_year': datetime.now().year,
+ 'crate_features': crate_features,
+ 'autogen_notice': AUTOGEN_NOTICE,
+ }
+
+ with open(ebuild_src, 'w') as ebuild:
+ ebuild.write(EMPTY_CRATE.format(**template_info))
+
+ if not args.dry_run and empty_package['name'] not in args.skip:
+ shutil.copy(ebuild_src, ebuild_dest)
+
+def check_if_package_is_required(empty_package, args):
+ """Check if an empty package already has a non-empty ebuild."""
+ ebuild_dest = get_ebuild_path(empty_package, args.target_dir, make_dir=False)
+ ebuild_target_dir = os.path.dirname(ebuild_dest)
+
+ # Find all ebuilds in ebuild_target dir and confirm they have
+ # CROS_RUST_EMPTY_CRATE in them. If they do not, they need to be manually
+ # handled.
+ for root, _, files in os.walk(ebuild_target_dir):
+ for efile in files:
+ if not efile.endswith('.ebuild'):
+ continue
+
+ with open(os.path.join(root, efile), 'r') as ebuild:
+ if 'CROS_RUST_EMPTY_CRATE' not in ebuild.read():
+ print('{} was not an empty crate.'.format(efile))
+ return True
+
+ return False
+
+def main(argv):
+ """Convert dependencies from Cargo.toml into ebuilds."""
+ args = parse_args(argv)
+
+ prepare_staging(args)
+
+ processed_packages = {}
+ optional_packages = {}
+ metadata = load_metadata(args.manifest_path)
+ for p in metadata['packages']:
+ process_package(p, args, optional_packages)
+ processed_packages[p['name']] = True
+
+ for key in optional_packages:
+ if key not in processed_packages and not check_if_package_is_required(
+ optional_packages[key], args):
+ process_empty_package(optional_packages[key], args)
+
+
+def parse_args(argv):
+ """Parse the command-line arguments."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ '-s',
+ '--staging-dir',
+ required=True,
+ help='Staging directory for temporary and generated files')
+
+ parser.add_argument(
+ '-d',
+ '--dry-run',
+ action='store_true',
+ help='Generate dependency tree but do not upload anything')
+
+ parser.add_argument('-t',
+ '--target-dir',
+ help='Path to chromiumos-overlay')
+
+ parser.add_argument('-k',
+ '--skip',
+ action='append',
+ help='Skip these packages when updating ebuilds')
+ parser.add_argument('-n',
+ '--no-upload',
+ action='store_true',
+ help='Skip uploading crates to distfiles')
+
+ parser.add_argument('manifest_path',
+ help='Cargo.toml used to generate ebuilds.')
+
+ args = parser.parse_args(argv)
+
+ # Require target directory if not dry run
+ if not args.target_dir and not args.dry_run:
+ raise Exception('Target directory must be set unless dry-run is True.')
+
+ if not args.skip:
+ args.skip = []
+
+ return args
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))