libdot/bin/pylint

#!/usr/bin/env python3
# Copyright 2018 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run pylint with the right settings."""

import functools
import json
import logging
import os
from pathlib import Path
import shutil
import sys
from typing import List

import libdot


# URI to the official pylint docs.
MAIN_DOCS = (
    "http://pylint.pycqa.org/en/latest/technical_reference/"
    "features.html#pylint-checkers-options-and-switches"
)

# URI base for user managed wiki.  It's sometimes better.
USER_BASE_URI = "http://pylint-messages.wikidot.com/messages:%s"


def convert_to_kokoro(data):
    """Take pylint JSON output and convert it to kokoro comment format.

    The |data| input will look like:
    [
      {
        "type": "<informational|convention|error|fatal|...>",
        "module": "generate-externs",
        "obj": "typename",
        "line": 20,
        "column": 0,
        "path": "bin/generate-externs",
        "symbol": "docstring-first-line-empty",
        "message": "First line empty in function docstring",
        "message-id": "C0199"
      }
    ]

    See eslint.convert_to_kokoro for example return value.
    """
    for result in data:
        msg = (
            f"[pylint] {result['symbol']} ({result['message-id']})\n"
            + result["message"]
            + "\n"
            + MAIN_DOCS
            + "\n"
            + USER_BASE_URI % (result["message-id"],)
        )

        path = os.path.join(os.getcwd(), result["path"])
        yield {
            "path": os.path.relpath(path, libdot.LIBAPPS_DIR),
            "message": msg,
            "startLine": result["line"],
            "endLine": result["line"],
            "startCharacter": result["column"],
            "endCharacter": result["column"],
        }


def filter_known_files(paths: List[Path]) -> List[Path]:
    """Figure out what files this linter supports."""
    ret = []
    for path in paths:
        path = Path(path)

        if path.suffix == ".py":
            # Add all .py files as they should only be Python.
            ret += [path]
        elif path.is_symlink():
            # Ignore symlinks.
            pass
        elif not path.exists():
            # Ignore missing files here (caller will handle it).
            pass
        elif path.is_file() and path.stat().st_mode & 0o111:
            # Add executable programs with python shebangs.
            with path.open("rb") as fp:
                shebang = fp.readline()
            if b"python" in shebang:
                ret += [path]

    return [str(x) for x in ret]


@functools.lru_cache(maxsize=1)
def find_pylint():
    """Figure out the name of the pylint tool.

    It keeps changing with Python 2->3 migrations.  Fun.
    """
    # Prefer our vpython copy if possible.
    if shutil.which("vpython3"):
        return libdot.BIN_DIR / "pylint-vpython"

    # Prefer pylint3 as that's what we want.
    if shutil.which("pylint3"):
        return "pylint3"

    # If there's no pylint, give up.
    if not shutil.which("pylint"):
        logging.error(
            "unable to locate pylint; please install:\n"
            "sudo apt-get install pylint"
        )
        sys.exit(1)

    # Make sure pylint is using Python 3.
    result = libdot.run(
        ["pylint", "--version"], capture_output=True, encoding="utf-8"
    )
    if "Python 3" in result.stdout:
        return "pylint"

    logging.error(
        "pylint does not support Python 3; please upgrade:\n%s",
        result.stdout.strip(),
    )
    sys.exit(1)


def setup():
    """Initialize the tool settings."""
    find_pylint()


def run(argv=(), pythonpaths=(), **kwargs):
    """Run the tool directly."""
    setup()

    # Add libdot to search path so pylint can find it.  Any subproject that
    # uses us will make sure it's in the search path too.
    path = os.environ.get("PYTHONPATH", "")
    paths = [libdot.BIN_DIR] + list(pythonpaths)
    if path is not None:
        paths.append(path)
    extra_env = kwargs.pop("extra_env", {})
    assert "PYTHONPATH" not in extra_env
    kwargs["extra_env"] = {
        "PYTHONPATH": os.pathsep.join(str(x) for x in paths),
        **extra_env,
    }

    pylintrc = os.path.relpath(
        os.path.join(libdot.LIBAPPS_DIR, ".pylintrc"), os.getcwd()
    )
    cmd = [find_pylint(), "--rcfile", pylintrc] + list(argv)
    return libdot.run(cmd, **kwargs)


def perform(
    argv=(), paths=(), fix=False, gerrit_comments_file=None, pythonpaths=()
):
    """Run high level tool logic."""
    ret = True
    argv = list(argv)
    paths = list(paths)

    # Pylint doesn't have any automatic fixing logic.
    if fix:
        return ret

    comments_path = libdot.lint.kokoro_comments_path(
        gerrit_comments_file, "pylint"
    )

    result = run(argv + paths, pythonpaths=pythonpaths, check=False)
    if result.returncode:
        ret = False

        # Rerun for Gerrit.
        if comments_path:
            # Handle relative paths like "foo.json".
            dirname = os.path.dirname(comments_path)
            if dirname:
                os.makedirs(dirname, exist_ok=True)

            argv += ["--output-format=json"]
            result = run(
                argv + paths,
                pythonpaths=pythonpaths,
                check=False,
                capture_output=True,
            )

            # Save a copy for debugging later.
            with open(comments_path + ".in", "wb") as fp:
                fp.write(result.stdout)

            data = json.loads(result.stdout.decode("utf-8"))
            comments = list(convert_to_kokoro(data))
            with open(comments_path, "w", encoding="utf-8") as fp:
                json.dump(comments, fp, sort_keys=True)
    elif comments_path:
        # If there were no failures, clear the files to avoid leaving previous
        # results laying around & confuse devs.
        libdot.unlink(comments_path)
        libdot.unlink(comments_path + ".in")

    return ret


def get_parser():
    """Get a command line parser."""
    parser = libdot.ArgumentParser(description=__doc__, short_options=False)
    parser.add_argument(
        "--gerrit-comments-file",
        help="Save errors for posting files to Gerrit.",
    )
    parser.add_argument("paths", nargs="*", help="Paths to lint.")
    return parser


def main(argv, pythonpaths=()):
    """The main func!"""
    parser = get_parser()
    opts, args = parser.parse_known_args(argv)

    return (
        0
        if perform(
            argv=args,
            paths=opts.paths,
            pythonpaths=pythonpaths,
            gerrit_comments_file=opts.gerrit_comments_file,
        )
        else 1
    )


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))