libdot/bin/pylint

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run pylint with the right settings."""

from __future__ import print_function

import functools
import json
import logging
import os
from pathlib import Path
import shutil
import sys
from typing import List

import libdot


# URI to the official pylint docs.
MAIN_DOCS = ('http://pylint.pycqa.org/en/latest/technical_reference/'
             'features.html#pylint-checkers-options-and-switches')

# URI base for user managed wiki.  It's sometimes better.
USER_BASE_URI = 'http://pylint-messages.wikidot.com/messages:%s'


def convert_to_kokoro(data):
    """Take pylint JSON output and convert it to kokoro comment format.

    The |data| input will look like:
    [
      {
        "type": "<informational|convention|error|fatal|...>",
        "module": "generate-externs",
        "obj": "typename",
        "line": 20,
        "column": 0,
        "path": "bin/generate-externs",
        "symbol": "docstring-first-line-empty",
        "message": "First line empty in function docstring",
        "message-id": "C0199"
      }
    ]

    See eslint.convert_to_kokoro for example return value.
    """
    for result in data:
        msg = '[pylint] %s (%s)\n%s\n%s\n%s' % (
            result['symbol'], result['message-id'], result['message'],
            MAIN_DOCS, USER_BASE_URI % (result['message-id'],),
        )

        path = os.path.join(os.getcwd(), result['path'])
        yield {
            'path': os.path.relpath(path, libdot.LIBAPPS_DIR),
            'message': msg,
            'startLine': result['line'],
            'endLine': result['line'],
            'startCharacter': result['column'],
            'endCharacter': result['column'],
        }


def filter_known_files(paths: List[Path]) -> List[Path]:
    """Figure out what files this linter supports."""
    ret = []
    for path in paths:
        path = Path(path)

        # TODO(vapier): Figure out why pylint hates us.  Running in the
        # container on these modules fails with "internal errors:
        # maximum recursion depth exceeded.".
        spec = (path.parent.parent.name, path.parent.name, path.name)
        if spec == ('libdot', 'bin', 'node'):
            continue
        elif spec == ('ssh_client', 'bin', 'ssh_client.py'):
            continue

        if path.suffix == '.py':
            # Add all .py files as they should only be Python.
            ret += [path]
        elif path.is_symlink():
            # Ignore symlinks.
            pass
        elif not path.exists():
            # Ignore missing files here (caller will handle it).
            pass
        elif path.stat().st_mode & 0o111:
            # Add executable programs with python shebangs.
            with path.open('rb') as fp:
                shebang = fp.readline()
            if b'python' in shebang:
                ret += [path]

    return [str(x) for x in ret]


@functools.lru_cache(maxsize=1)
def find_pylint():
    """Figure out the name of the pylint tool.

    It keeps changing with Python 2->3 migrations.  Fun.
    """
    # Prefer pylint3 as that's what we want.
    if shutil.which('pylint3'):
        return 'pylint3'

    # If there's no pylint, give up.
    if not shutil.which('pylint'):
        logging.error('unable to locate pylint; please install:\n'
                      'sudo apt-get install pylint')
        sys.exit(1)

    # Make sure pylint is using Python 3.
    result = libdot.run(['pylint', '--version'], capture_output=True,
                        encoding='utf-8')
    if 'Python 3' in result.stdout:
        return 'pylint'

    logging.error('pylint does not support Python 3; please upgrade:\n%s',
                  result.stdout.strip())
    sys.exit(1)


def setup():
    """Initialize the tool settings."""
    find_pylint()


def run(argv=(), pythonpaths=(), **kwargs):
    """Run the tool directly."""
    setup()

    # Add libdot to search path so pylint can find it.  Any subproject that
    # uses us will make sure it's in the search path too.
    path = os.environ.get('PYTHONPATH', '')
    paths = [libdot.BIN_DIR] + list(pythonpaths)
    if path is not None:
        paths.append(path)
    extra_env = kwargs.pop('extra_env', {})
    assert 'PYTHONPATH' not in extra_env
    kwargs['extra_env'] = {
        'PYTHONPATH': os.pathsep.join(str(x) for x in paths),
        **extra_env,
    }

    pylintrc = os.path.relpath(os.path.join(libdot.LIBAPPS_DIR, '.pylintrc'),
                               os.getcwd())
    cmd = [find_pylint(), '--rcfile', pylintrc] + list(argv)
    return libdot.run(cmd, **kwargs)


def perform(argv=(), paths=(), fix=False, gerrit_comments_file=None,
            pythonpaths=()):
    """Run high level tool logic."""
    ret = True
    argv = list(argv)
    paths = list(paths)

    # Pylint doesn't have any automatic fixing logic.
    if fix:
        return ret

    comments_path = libdot.lint.kokoro_comments_path(
        gerrit_comments_file, 'pylint')

    result = run(argv + paths, pythonpaths=pythonpaths, check=False)
    if result.returncode:
        ret = False

        # Rerun for Gerrit.
        if comments_path:
            # Handle relative paths like "foo.json".
            dirname = os.path.dirname(comments_path)
            if dirname:
                os.makedirs(dirname, exist_ok=True)

            argv += ['--output-format=json']
            result = run(argv + paths, pythonpaths=pythonpaths, check=False,
                         capture_output=True)

            # Save a copy for debugging later.
            with open(comments_path + '.in', 'wb') as fp:
                fp.write(result.stdout)

            data = json.loads(result.stdout.decode('utf-8'))
            comments = list(convert_to_kokoro(data))
            with open(comments_path, 'w', encoding='utf-8') as fp:
                json.dump(comments, fp, sort_keys=True)
    elif comments_path:
        # If there were no failures, clear the files to avoid leaving previous
        # results laying around & confuse devs.
        libdot.unlink(comments_path)
        libdot.unlink(comments_path + '.in')

    return ret


def get_parser():
    """Get a command line parser."""
    parser = libdot.ArgumentParser(description=__doc__, short_options=False)
    parser.add_argument('--gerrit-comments-file',
                        help='Save errors for posting files to Gerrit.')
    parser.add_argument('paths', nargs='*',
                        help='Paths to lint.')
    return parser


def main(argv, pythonpaths=()):
    """The main func!"""
    parser = get_parser()
    opts, args = parser.parse_known_args(argv)

    return 0 if perform(argv=args, paths=opts.paths,
                        pythonpaths=pythonpaths,
                        gerrit_comments_file=opts.gerrit_comments_file) else 1


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))