libdot/bin/closure-compiler

#!/usr/bin/env python3
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Download & run the Closure Compiler.

https://github.com/google/closure-compiler/wiki/Binary-Downloads
"""

import json
import os
import sys

import libdot


# The version we currently run.  Pinned to make sure everyone gets consistent
# behavior all the time.
VERSION = '20200927'

# Full path to Google's closure compiler.
URI = ('https://repo1.maven.org/maven2/com/google/javascript/closure-compiler/'
       f'v{VERSION}/closure-compiler-v{VERSION}.jar')


# Where we cache our copy of closure.
CLOSURE = libdot.BIN_DIR / f'.closure-compiler.{VERSION}.jar'


# Where we store externs files.
EXTERNS_DIR = libdot.DIR / 'externs'

# The version of externs that we pull in from closure.  We normally pin to the
# closure version for stability, but this can be updated independently.
EXTERNS_VERSION = VERSION

# The version of Chrome externs that we pull from Chromium directly.
CHROME_EXTERNS_VERSION = '18029c0278c7da713fb6b629e84992ceabda7ae2'


# We pick 50 as the mid point as it allows us to easily add files before and
# after the closure modules.
CLOSURE_ORDER_BASE = 50

# Load the Chrome externs after the closure ones.  We'll prob never have 10
# closure modules, but if we do, we can increase that here.
CHROME_ORDER_BASE = CLOSURE_ORDER_BASE + 10


def update_closure_externs():
    """Update our local cache of externs."""
    URI_TEMPLATE = ('https://github.com/google/closure-compiler/raw/'
                    f'v{EXTERNS_VERSION}/contrib/externs/%(name)s.js')

    # NB: Order here matters when modules are subsets of others.  For example,
    # 'chrome.js' defines 'var chrome = {}' which 'chrome_extensions.js' needs.
    # If we reorder them, closure-compiler might accept it, but then silently
    # not check some things.  It's kind of buggy that way :(.
    # https://github.com/google/closure-compiler/issues/3586
    EXTERNS = (
        'chrome',
        'chrome_extensions',
    )

    for i, name in enumerate(EXTERNS, start=CLOSURE_ORDER_BASE):
        uri = URI_TEMPLATE % {'name': name}
        path = EXTERNS_DIR / f'{i}-closure-{name}-v{EXTERNS_VERSION}.js'
        if path.exists():
            continue

        for oldpath in EXTERNS_DIR.glob(f'*closure-{name}-v*.js'):
            oldpath.unlink()
        libdot.fetch(uri, path)


def update_chrome_externs():
    """Update our local cache of externs from Chrome."""
    URI_TEMPLATE = ('https://chromium.googlesource.com/chromium/src/+/'
                    f'{CHROME_EXTERNS_VERSION}/third_party/closure_compiler/'
                    'externs/%(name)s.js?format=TEXT')

    # Order here shouldn't matter as each file shouldn't have overlapping APIs.
    EXTERNS = (
        'terminal_private',
    )

    for i, name in enumerate(EXTERNS, start=CHROME_ORDER_BASE):
        uri = URI_TEMPLATE % {'name': name}
        path = EXTERNS_DIR / f'{i}-chrome-{name}-v{CHROME_EXTERNS_VERSION}.js'
        if path.exists():
            continue

        for oldpath in EXTERNS_DIR.glob(f'*-chrome-{name}-v*.js'):
            oldpath.unlink()
        libdot.fetch(uri, path, b64=True)


def update_closure():
    """Update our local cache of Google's closure compiler."""
    if CLOSURE.exists():
        return

    for path in libdot.BIN_DIR.glob('.closure-compiler.*.jar'):
        path.unlink()

    libdot.fetch(URI, CLOSURE)


def convert_to_kokoro(data):
    """Take closure JSON output and convert it to kokoro comment format.

    The |data| input will look like:
    [
      {
        "column": 21,
        "context": " * @return {!Promise<FileSystemFileEntry>}\n...",
        "description": "FileSystemFileEntry is a reference type with ...",
        "key": "JSC_MISSING_NULLABILITY_MODIFIER_JSDOC",
        "level": "error",
        "line": 174,
        "source": "js/lib_fs.js"
      },
      {
        "description": "1 error(s), 0 warning(s)",
        "level": "info"
      }
    ]

    See convert_eslint_to_kokoro for example return value.
    """
    for result in data:
        # Ignore info lines.
        if result['level'] == 'info':
            continue

        # Ignore generated files not in git.
        if libdot.lint.is_generated_path(result['source']):
            continue

        # Add leading space prefix to results to get code text.
        msg = '[closure] %s: %s\n%s\n\n %s' % (
            result['level'], result['key'], result['description'],
            '\n '.join(result['context'].splitlines()),
        )

        yield {
            'path': os.path.relpath(result['source'], libdot.LIBAPPS_DIR),
            'message': msg,
            'startLine': result['line'],
            'endLine': result.get('endline', result['line']),
            'startCharacter': result['column'],
            'endCharacter': result['column'],
        }


def setup():
    """Initialize the tool settings."""
    update_closure()
    update_closure_externs()
    update_chrome_externs()


def run(argv=(), **kwargs):
    """Run the tool directly."""
    setup()
    return libdot.run(
        list(argv),
        cmd_prefix=['java', '-jar', CLOSURE],
        log_prefix=['closure-compiler'],
        **kwargs)


def perform(argv=(), paths=(), fix=False, gerrit_comments_file=None):
    """Run high level tool logic."""
    ret = True
    argv = list(argv)
    paths = list(paths)

    comments_path = libdot.lint.kokoro_comments_path(
        gerrit_comments_file, 'closure')

    # Closure doesn't have any automatic fixing logic.
    if fix:
        return ret

    setup()
    externs = []
    externs_paths = libdot.DIR / 'externs'
    for extern in externs_paths.glob('*.js'):
        externs += ['--externs', os.path.relpath(extern)]

    argv += [
        '--checks-only',
        '--language_in=ECMASCRIPT_2018',
    ] + externs
    result = run(argv + paths, check=False)
    if result.returncode:
        ret = False

        # Rerun for Gerrit.
        if comments_path:
            # Handle relative paths like "foo.json".
            dirname = os.path.dirname(comments_path)
            if dirname:
                os.makedirs(dirname, exist_ok=True)

            argv += ['--error_format=JSON']
            result = run(argv + paths, check=False, capture_output=True)

            # Save a copy for debugging later.
            with open(comments_path + '.in', 'wb') as fp:
                fp.write(result.stderr)

            data = json.loads(result.stderr.decode('utf-8'))
            comments = list(convert_to_kokoro(data))
            with open(comments_path, 'w', encoding='utf-8') as fp:
                json.dump(comments, fp, sort_keys=True)
    elif comments_path:
        # If there were no failures, clear the files to avoid leaving previous
        # results laying around & confuse devs.
        libdot.unlink(comments_path)
        libdot.unlink(comments_path + '.in')

    return ret


def main(argv):
    """The main func!"""
    libdot.setup_logging()
    run(argv)


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))