nassh/bin/mkdeps

#!/usr/bin/env python3
# Copyright 2019 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Build the project's dependencies."""

import html
import json
import logging
import re
import sys

import nassh  # pylint: disable=wrong-import-order
import libdot


LICENSE_FILE = nassh.DIR / "html" / "licenses.html"
LICENSE_TEMPLATE = nassh.DIR / "html" / "licenses.html.in"

# Make sure we keep a handle on what licenses we ship.  This is not the list of
# licenses that are actually used, just all the ones that we're OK with using.
ALLOWED_LICENSES = {
    "Apache-2.0",
    "BSD",
    "BSD-2-Clause",
    "BSD-3-Clause",
    "CC0",
    "GPL-2",
    "GPL-2.0",
    "GPL-3",
    "GPL-3.0",
    "ISC",
    "LGPL",
    "LGPL-2.1",
    "LGPL-3",
    "MIT",
    "Unlicense",
    "zlib",
}

# Licenses that we explicitly never want to use.  Do *not* remove any of these.
BANNED_LICENSES = {
    "AGPL",
    "AGPL-3",
    "AGPL-3+",
    "CC-BY-NC",
    "CC-BY-NC-ND",
    "CC-BY-NC-SA",
    "CPAL",
    "EUPL",
    "SISSL",
    "SSPL",
    "WTFPL",  # nocheck
}
assert not BANNED_LICENSES & ALLOWED_LICENSES


def mkdeps(local_only: bool = False):
    """Build the dependencies."""
    # NB: We assume hterm mkdist runs libdot mkdist.
    libdot.run([libdot.LIBAPPS_DIR / "hterm" / "bin" / "mkdist"])
    if not local_only:
        libdot.node.run(["rollup", "-c"], cwd=nassh.DIR)

    # Hack up the local tree for wassh integration.
    # NB: We can't create loops or deep directory trees as loading unpacked
    # extensions will make Chrome walk the entire tree and be very slow.

    for project in ("hterm", "libdot", "wasi-js-bindings", "wassh"):
        src = nassh.LIBAPPS_DIR / project
        dst = nassh.DIR / project
        dst.mkdir(exist_ok=True)
        for path in ("dist", "index.js", "js", "third_party"):
            if (src / path).exists():
                libdot.symlink(src / path, dst / path)

    wasm_src = nassh.LIBAPPS_DIR / "ssh_client" / "output"
    wasm_dst = nassh.DIR / "ssh_client" / "output"
    wasm_dst.mkdir(parents=True, exist_ok=True)
    libdot.symlink(wasm_src / "plugin", wasm_dst / "plugin")


def concat_third_party_dir(third_party_dir):
    """Concatenate all licenses of |third_party_dir| bundles."""
    ret = {}
    for package_dir in third_party_dir.glob("*"):
        entry = {}
        version = None

        package = package_dir.name

        metadata_file = package_dir / "METADATA"
        with metadata_file.open(encoding="utf-8") as fp:
            lines = fp.readlines()
            for i, line in enumerate(lines):
                if "HOMEPAGE" in line:
                    m = re.match(r'.*"(.*)"', lines[i + 1])
                    entry["repository"] = m.group(1)
                elif "version" in line:
                    m = re.match(r'.*"(.*)"', line)
                    version = m.group(1)

        for path in ("LICENSE", "LICENSE.md"):
            license_file = package_dir / path
            if not license_file.exists():
                continue
            logging.debug("%s: loading %s", package, license_file)
            entry["data"] = license_file.read_text(encoding="utf-8")
            break
        else:
            raise ValueError(f"Unable to locate LICENSE for {package}")

        ret[f"{package}@{version}"] = entry

    return ret


def concat_local_deps():
    """Concatenate all licenses of third_party/ bundles."""
    ret = {}
    ret.update(concat_third_party_dir(nassh.DIR / "third_party"))
    ret.update(
        concat_third_party_dir(
            libdot.LIBAPPS_DIR / "ssh_client" / "third_party"
        )
    )
    return ret


def concat_licenses():
    """Concatenate all licenses of npm dependencies."""
    ret = libdot.node.run(
        [
            "license-checker",
            "--search",
            nassh.DIR,
            "--onlyunknown",
            "--production",
            "--csv",
        ],
        capture_output=True,
        cwd=nassh.DIR,
    )
    # 'Found error' in stderr indicates that no packages with unspecified
    # licenses were found.
    if b"Found error" in ret.stderr:
        ret = libdot.node.run(
            [
                "license-checker",
                "--search",
                nassh.DIR,
                "--unknown",
                "--production",
                "--json",
                "--onlyAllow",
                ";".join(sorted(ALLOWED_LICENSES)),
            ],
            capture_output=True,
            cwd=nassh.DIR,
        )
        res = json.loads(ret.stdout.decode("utf8"))
        for entry in res.values():
            with open(entry["licenseFile"], "r", encoding="utf8") as fp:
                entry["data"] = fp.read().strip()

        res.update(concat_local_deps())

        # Dedupe by license to save on space.
        lic_to_pkgs = {}
        for package, entry in res.items():
            # Collapse whitespace to make sure people reformatting doesn't
            # result in a "new" license.
            lic = re.sub(r"\s+", "", entry["data"].lower())
            packages = lic_to_pkgs.setdefault(lic, [])
            packages.append(package)
            packages.sort()

        generate_html(sorted(lic_to_pkgs.values()), res)
    else:
        logging.error("The following packages did not specify their licenses:")
        logging.error(ret.stdout.decode("utf8"))


# Template for every package/license entry.
LICENSE_ENTRY_TEMPLATE_HOMEPAGE = "<a href='%(repository)s'>%(package)s</a>"
LICENSE_ENTRY_TEMPLATE = """
<h2 class='package' id='package-%(idx)s'>
  %(homepages)s
  <a class='license'
     i18n='{"aria-label": "$id", "_": "LICENSES_EXPAND_LINK"}'
     href='#'></a>
</h2>
<pre class='license' id='license-%(idx)s'>
%(data)s
</pre>
"""


def generate_html(packages_order, licenses):
    """Write the collected |licenses| to an HTML file."""
    with open(LICENSE_TEMPLATE, "r", encoding="utf8") as fp:
        template = fp.read()
    output = ""
    for idx, packages in enumerate(packages_order):
        homepages = []
        for package in packages:
            entry = licenses[package]
            homepages.append(
                LICENSE_ENTRY_TEMPLATE_HOMEPAGE
                % {
                    "repository": entry["repository"],
                    "package": html.escape(package),
                }
            )
        output += LICENSE_ENTRY_TEMPLATE % {
            "homepages": "<br>".join(homepages),
            "idx": idx,
            "data": html.escape(entry["data"], quote=False),
        }
    output = template.replace("%%LICENSES%%", output)
    with open(LICENSE_FILE, "w", encoding="utf8") as fp:
        fp.write(output)


def get_parser():
    """Get a command line parser."""
    parser = libdot.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--local-only",
        action="store_true",
        default=False,
        help="Only rollup local (libapps) dependencies.",
    )
    return parser


def main(argv):
    """The main func!"""
    parser = get_parser()
    opts = parser.parse_args(argv)
    libdot.node_and_npm_setup()
    nassh.fonts.fonts_update()
    nassh.plugin.plugin_update()
    mkdeps(local_only=opts.local_only)
    nassh.generate_changelog.generate_changelog()

    if not opts.local_only:
        logging.info("Concatenating licenses...")
        # We use nassh's package.json, but reuse libapps' node_modules.
        libdot.symlink(
            libdot.LIBAPPS_DIR / "node_modules", nassh.DIR / "node_modules"
        )
        try:
            concat_licenses()
        finally:
            libdot.unlink(nassh.DIR / "node_modules")


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))