| #!/usr/bin/env python3 |
| # Copyright 2019 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Lint our source files.""" |
| |
| import fnmatch |
| import logging |
| import os |
| from pathlib import Path |
| import sys |
| from typing import List |
| |
| import libdot |
| |
| |
| # All checks except strictMissingRequire. |
| DEFAULT_CLOSURE_ARGS = ( |
| '--jscomp_error=*', '--jscomp_off=strictMissingRequire') |
| |
| |
| # Options passed when invoking eslint. |
| ESLINT_ARGS = ( |
| '--cache', |
| '--cache-location', libdot.LIBAPPS_DIR / '.eslintcache', |
| ) |
| |
| |
| def kokoro_comments_path(path, tool): |
| """Expand % markers that might exist in |path|.""" |
| if not path: |
| return None |
| else: |
| return path.replace('%(tool)', tool) |
| |
| |
| def is_generated_path(path): |
| """Return True if |path| is generated. |
| |
| Useful for filtering out comments for files not in the tree. |
| """ |
| return ( |
| fnmatch.fnmatch(path, '*.concat.js') or |
| fnmatch.fnmatch(path, '*.rollup.js') |
| ) |
| |
| |
| def get_known_files(basedir: Path) -> List[Path]: |
| """Return list of files committed to the tree.""" |
| # -z appends \0 to each path, it doesn't delimit them. So strip off the |
| # trailing \0 to avoid a spurious '' entry at the end. |
| all_files = libdot.run( |
| ['git', 'ls-tree', '--name-only', '-r', '-z', 'HEAD'], cwd=basedir, |
| capture_output=True, encoding='utf-8').stdout[:-1].split('\0') |
| return [basedir / x for x in all_files if (basedir / x).exists()] |
| |
| |
| def get_known_sources(basedir: Path) -> List[Path]: |
| """Get list of committed files that we could possibly lint.""" |
| return (x for x in get_known_files(basedir) |
| if x.suffix not in { |
| '.log', '.jpg', '.ogg', '.patch', '.png', '.webp', |
| }) |
| |
| |
| def lint_whitespace_data(path: Path, data: str) -> bool: |
| """Basic whitespace checks on text files.""" |
| ret = True |
| |
| if '\r' in data: |
| ret = False |
| logging.error(r'%s: \r not allowed in text files', path) |
| |
| if data.startswith('\n'): |
| ret = False |
| logging.error('%s: No leading blank lines allowed', path) |
| |
| if not data.endswith('\n'): |
| ret = False |
| logging.error('%s: Files must end with a newline', path) |
| |
| return ret |
| |
| |
| def lint_whitespace_files(paths: List[Path]) -> bool: |
| """Basic whitespace checks on text files.""" |
| ret = True |
| |
| for path in paths: |
| with open(path, mode='r', encoding='utf-8') as fp: |
| data = fp.read() |
| |
| ret &= lint_whitespace_data(path, data) |
| |
| return ret |
| |
| |
| def lint_html_files(paths: List[Path]) -> bool: |
| """Basic lint checks on HTML files.""" |
| logging.info('Linting HTML files %s', libdot.cmdstr(paths)) |
| ret = True |
| |
| META_CHARSET_TAG = "<meta charset='utf-8'/>" |
| |
| for path in paths: |
| with open(path, mode='r', encoding='utf-8') as fp: |
| data = fp.read() |
| |
| if META_CHARSET_TAG not in data: |
| ret = False |
| logging.error('%s: <head>: missing %s tag', path, META_CHARSET_TAG) |
| |
| ret &= lint_whitespace_data(path, data) |
| |
| return ret |
| |
| |
| def lint_text_files(paths: List[Path]) -> bool: |
| """Basic lint checks on HTML files.""" |
| logging.info('Linting text files %s', libdot.cmdstr(paths)) |
| return lint_whitespace_files(paths) |
| |
| |
| def _get_default_paths(basedir: Path) -> List[Path]: |
| """Get list of paths to lint by default.""" |
| most_files = sorted(x for x in get_known_sources(basedir) |
| if x.suffix not in {'.js'}) |
| |
| # All files in js/*.js excluding generated files. |
| # Use relpath for nicer default output. |
| # Sort to ensure lib.js comes before lib_array.js, etc. |
| # Filter out the generated libdot.js/libdot.min.js/etc... |
| js_files = ( |
| list((libdot.DIR / 'html').glob('*.html')) + |
| list((libdot.DIR / 'js').glob('*.js')) + |
| list((libdot.DIR / 'third_party').glob('*/*.js'))) |
| js_files = sorted(x for x in js_files |
| if not x.name.startswith('libdot.')) |
| |
| return [os.path.relpath(x) for x in most_files + js_files] |
| |
| |
| def get_parser(): |
| """Get a command line parser.""" |
| parser = libdot.ArgumentParser(description=__doc__) |
| parser.add_argument('--version', action='store_true', |
| help='Show linter versions.') |
| parser.add_argument('--fix', action='store_true', |
| help='Run linters with --fix setting if possible.') |
| parser.add_argument('--gerrit-comments-file', |
| help='Save errors for posting files to Gerrit.') |
| parser.add_argument('--skip-mkdeps', dest='run_mkdeps', |
| action='store_false', default=True, |
| help='Skip (re)building of dependencies.') |
| |
| group = parser.add_argument_group( |
| 'File selection/filtering', |
| description='All files are linted by default') |
| group.add_argument('--cpp', action='store_true', |
| help='Lint C/C++ files.') |
| group.add_argument('--js', action='store_true', |
| help='Lint JavaScript files.') |
| group.add_argument('--json', action='store_true', |
| help='Lint JSON files.') |
| group.add_argument('--md', action='store_true', |
| help='Lint Markdown files.') |
| group.add_argument('--py', action='store_true', |
| help='Lint Python files.') |
| group.add_argument('--txt', action='store_true', |
| help='Lint text files.') |
| |
| parser.add_argument('paths', nargs='*', |
| help='Paths to lint.') |
| |
| return parser |
| |
| |
| def main(argv, get_default_paths=get_known_sources, basedir=None, mkdeps=None, |
| closure_args=DEFAULT_CLOSURE_ARGS): |
| """The common main func for all linters. |
| |
| Args: |
| argv: The command line to process. |
| paths: The default set of files to lint. If the user doesn't specify |
| any, we'll use these instead. |
| basedir: The base source tree to search for default set of files to lint. |
| mkdeps: Callback to build dependencies after we've initialized. |
| closure_args: Extra arguments to pass to closure-compiler. |
| """ |
| parser = get_parser() |
| opts = parser.parse_args(argv) |
| libdot.node_and_npm_setup() |
| |
| if opts.version: |
| for func in ( |
| libdot.eslint.run, |
| libdot.closure_compiler.run, |
| libdot.pylint.run, |
| ): |
| func(['--version']) |
| return 0 |
| |
| if not opts.paths: |
| opts.paths = [str(x) for x in get_default_paths(basedir)] |
| if not opts.paths: |
| print('No files to lint.') |
| return 0 |
| |
| if mkdeps: |
| if opts.run_mkdeps: |
| mkdeps(opts) |
| else: |
| logging.info('Skipping building dependencies due to --skip-mkdeps') |
| |
| lint_groups = {'cpp', 'js', 'json', 'md', 'py', 'txt'} |
| lint_all = all(not getattr(opts, x) for x in lint_groups) |
| for group in lint_groups: |
| setattr(opts, group, lint_all or getattr(opts, group)) |
| |
| tolint_paths = set(opts.paths) |
| kwargs = { |
| 'fix': opts.fix, |
| 'gerrit_comments_file': opts.gerrit_comments_file, |
| } |
| checks = [] |
| |
| html_files = [x for x in opts.paths |
| if x.endswith('.html') or x.endswith('.html.in')] |
| |
| js_files = [x for x in opts.paths if x.endswith('.js')] |
| tolint_paths -= set(js_files) |
| if js_files and opts.js: |
| js_kwargs = {'paths': js_files, **kwargs} |
| html_js_kwargs = {'paths': html_files + js_files, **kwargs} |
| checks += [ |
| libdot.eslint.perform(argv=ESLINT_ARGS, **html_js_kwargs), |
| libdot.closure_compiler.perform(argv=closure_args, **js_kwargs), |
| ] |
| |
| py_files = libdot.pylint.filter_known_files(opts.paths) |
| tolint_paths -= set(py_files) |
| if py_files and opts.py: |
| py_kwargs = {'paths': py_files, **kwargs} |
| checks += [libdot.pylint.perform(**py_kwargs)] |
| |
| cpp_files = libdot.cpplint.filter_known_files(opts.paths) |
| tolint_paths -= set(cpp_files) |
| if cpp_files and opts.cpp: |
| cpp_kwargs = {'paths': cpp_files, **kwargs} |
| checks += [libdot.cpplint.perform(**cpp_kwargs)] |
| |
| tolint_paths -= set(html_files) |
| if html_files and opts.txt: |
| checks += [lint_html_files(html_files)] |
| |
| md_files = libdot.mdlint.filter_known_files(opts.paths) |
| tolint_paths -= set(md_files) |
| if md_files and opts.md: |
| md_kwargs = {'paths': md_files, **kwargs} |
| checks += [ |
| lint_whitespace_files(md_files), |
| libdot.mdlint.perform(**md_kwargs), |
| ] |
| |
| json_files = libdot.jsonlint.filter_known_files(opts.paths) |
| tolint_paths -= set(json_files) |
| if json_files and opts.json: |
| json_kwargs = {'paths': json_files, **kwargs} |
| checks += [ |
| lint_whitespace_files(json_files), |
| libdot.jsonlint.perform(**json_kwargs), |
| ] |
| |
| TEXT_FILENAMES = { |
| '.clang-format', '.eslintrc.js', '.gitignore', '.markdownlintrc', |
| '.npmrc', '.pylintrc', |
| 'Dockerfile', 'LICENSE', 'Makefile', 'METADATA', 'OWNERS', |
| } |
| TEXT_EXTENSIONS = { |
| '.cfg', '.concat', '.css', '.el', '.sh', '.svg', '.txt', '.vim', '.xml', |
| } |
| text_files = [x for x in opts.paths |
| if (os.path.basename(x) in TEXT_FILENAMES or |
| os.path.splitext(x)[1] in TEXT_EXTENSIONS)] |
| tolint_paths -= set(text_files) |
| if text_files and opts.txt: |
| checks += [lint_text_files(text_files)] |
| |
| if tolint_paths: |
| logging.warning('Linting skipped:\n%s', ' '.join(sorted(tolint_paths))) |
| |
| return 0 if all(checks) else 1 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:], basedir=libdot.DIR, |
| get_default_paths=_get_default_paths)) |
