tools/code_hygiene.py

#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Simple Code Hygiene tool meant to be run as a presubmit test or standalone

Usage:
./code_hygiene.py <file1> <file2> ...

For the styleguides see:
http://code.google.com/p/soc/wiki/PythonStyleGuide
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
"""

import os
import re
import subprocess
import sys

# ======================================================================
VERBOSE = 0
LIMIT = 5

# These programs are likely not available/working on windows.

# http://tidy.sourceforge.net/
HTML_CHECKER = ['tidy', '-errors']

# From depot_tools:
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
CPP_CHECKER = ['cpplint.py',
               '--filter='
               '-build/header_guard'
               ',-build/class'
               ',-build/include_order'
               ',-readability/casting'
               ',-readability/check'
               ',-readability/multiline_comment'
               ',-readability/streams'
               ',-runtime/sizeof'
               ',-runtime/threadsafe_fn'
               ',-whitespace/newline'
               ]

# From depot_tools (currently not used -- too many false positives).
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
C_CHECKER = ['cpplint.py',
             '--filter='
             '-build/header_guard'
             ',-build/class'
             ',-build/include_order'
             ',-readability/casting'
             ',-readability/check'
             ',-readability/multiline_comment'
             ',-runtime/sizeof'
             ',-runtime/threadsafe_fn'
             ',-whitespace/newline'
             ]

# http://pychecker.sourceforge.net/
PYTHON_CHECKER = ['pychecker', '-stdlib']

RE_IGNORE = re.compile(r'@IGNORE_LINES_FOR_CODE_HYGIENE\[([0-9]+)\]')

# ======================================================================
def Debug(s):
  if VERBOSE:
    print s


def RunCommand(cmd):
  """Run a shell command given an argv style vector."""
  Debug(str(cmd))

  p = subprocess.Popen(cmd,
                       bufsize=1000*1000,
                       stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)
  stdout, stderr = p.communicate()
  retcode = p.wait()

  return retcode, stdout, stderr


# ======================================================================
class ExternalChecker(object):
  """Base Class"""
  def __init__(self, commandline, use_stderr=False):
    self._commandline = commandline
    self._use_stderr = use_stderr

  def FindProblems(self, props, unused_data):
    filename = props['name']
    try:
      retcode, stdout, stderr = RunCommand(self._commandline + [filename])
    except Exception, err:
      print 'Error processing %s:' % filename
      print '  Cannot execute command %s. failed reason: %s ' % (
          str(self._commandline), str(err))
      return []
    if retcode == 0:
      return []
    if self._use_stderr:
      return [stderr]
    else:
      return [stdout]

  def FileFilter(self, unused_props):
    raise NotImplementedError()


class TidyChecker(ExternalChecker):
  """Invokes tidy tool on html files."""
  def __init__(self):
    ExternalChecker.__init__(self, HTML_CHECKER, use_stderr=True)

  def FileFilter(self, props):
    return '.html' in props


class PyChecker(ExternalChecker):
  """Invokes pychecker tool on python files."""
  def __init__(self):
    ExternalChecker.__init__(self, PYTHON_CHECKER)

  def FileFilter(self, props):
    return '.py' in props


class CppLintChecker(ExternalChecker):
  """Invokes Google's cpplint tool on c++ files."""
  def __init__(self):
    ExternalChecker.__init__(self, CPP_CHECKER, use_stderr=True)

  def FileFilter(self, props):
    return '.cc' in props or '.h' in props


class NotForCommitChecker(object):
  def FindProblems(self, unused_props, data):
    return ['This file should not be committed!']

  def FileFilter(self, props):
    return 'not_for_commit' in props['name']


# ======================================================================
class GenericRegexChecker(object):
  """Base Class"""
  def __init__(self, content_regex, line_regex=None,
               analyze_match=False, message=''):
    self._content_regex = re.compile(content_regex, re.MULTILINE)
    if line_regex:
      self._line_regex = re.compile(line_regex)
    else:
      self._line_regex = re.compile(content_regex)
    self._analyze_match = analyze_match
    self._message = message

  def FindProblems(self, unused_props, data):
    """Looks for presubmit issues in the data from a file."""
    # Speedy early out.
    if not self._content_regex.search(data):
      return []

    lines = data.split('\n')
    problem = []
    ignore_line_count = 0
    for no, line in enumerate(lines):
      if ignore_line_count > 0:
        ignore_line_count -= 1
        continue
      match = RE_IGNORE.search(line)
      if match:
        ignore_line_count = int(match.group(1))
        continue
      match = self._line_regex.search(line)
      if not match:
        continue
      if self._analyze_match:
        if not self.IsProblemMatch(match):
          continue
      # 'no' is zero-based, use 'no + 1' to get the actual line numbers.
      problem.append(self._RenderItem(no + 1, line))
    return problem

  def _RenderItem(self, no, line):
    """Returns a formatted message."""
    return 'line %d: [%s] %s' % (no, repr(line), self._message)

  def FileFilter(self, unused_props):
    """Returns true if the file should be checked by the checker."""
    # NOTE: All subclasses need to implement this and return True if the
    # input Properties show that this file should be checked.
    raise NotImplementedError()

  def IsProblemMatch(self, unused_match):
    """Returns true if the file contains a presubmit problem."""
    # NOTE: Not all subclasses need to implement this.
    pass


class TrailingWhiteSpaceChecker(GenericRegexChecker):
  """No trailing whitespaces in code we control."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r' $')

  def FileFilter(self, props):
    return ('.patch' not in props and
            '.stdout' not in props and
            '.table' not in props)


class CarriageReturnChecker(GenericRegexChecker):
  """Abolish windows style line terminators."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r'\r')

  def FileFilter(self, props):
    return '.h' in props or '.cc' in props or '.c' in props or '.py' in props


class CppCommentChecker(GenericRegexChecker):
  """No cpp comments in regular c files."""
  def __init__(self):
    # We tolerate http:// etc in comments
    GenericRegexChecker.__init__(self, r'(^|[^:])//')

  def FileFilter(self, props):
    return '.c' in props


class TabsChecker(GenericRegexChecker):
  """No tabs except for makefiles."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r'\t')

  def FileFilter(self, props):
    return ('is_makefile' not in props and
            '.patch' not in props and
            '.valerr' not in props and
            '.dis' not in props)


class FixmeChecker(GenericRegexChecker):
  """No FIXMEs in code we control."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r'\bFIXME\b')

  def FileFilter(self, props):
    return '.patch' not in props


class RewriteChecker(GenericRegexChecker):
  """No rewrite markers allowed (probaly not an issue anymore)."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r'[@][@]REWRITE')

  def FileFilter(self, unused_props):
    return 1


VALID_INCLUDE_PREFIX = [
    'base/',
    'breakpad/',
    'chrome/common/',
    'chrome/renderer/',
    'gen/native_client/',
    'gpu/',
    'gtest/',
    'native_client/',
    'ppapi/',
    'srpcgen/',
    'third_party/',
    'trusted/srpcgen/',
    'untrusted/srpcgen/',
    ]


class IncludeChecker(GenericRegexChecker):
  """Enforce some base 'include' rules. Cpplint does more of these."""
  def __init__(self):
    GenericRegexChecker.__init__(self, r'#include',
                                 line_regex=r'^(#include.*|})',
                                 analyze_match=True)

  def FindProblems(self, props, data):
    self._seen_code = False
    return GenericRegexChecker.FindProblems(self, props, data)

  def IsProblemMatch(self, match):
    if match.group(1) == '}':
      # This test is our hacked indicator signaling that no includes
      # directives should follow after we have seen actual code.
      self._seen_code = True
      return False

    if self._seen_code:
      return True
    token = match.group(1).split()
    if len(token) < 2:
      return False
    path = token[1][1:]
    if '..' in path:
      return True
    if not token[1].startswith('<'):
      for prefix in VALID_INCLUDE_PREFIX:
        if path.startswith(prefix):
          break
      else:
        return True
    return False

  def FileFilter(self, props):
    return '.cc' in props or '.c' in props


class BuildConfigChecker(GenericRegexChecker):
  """Enforce that build_config.h is include when some macros are present.

  build_config.h is required as a header in many sources to provide macro
  definitions for the operating system and architecture. This presubmit check
  exists to ensure that it's never accidentally omitted in source files.

  This is implemented as a presubmit as there is no usable mechanism to warn on
  undefined macro usage on Windows.
  """

  def __init__(self):
    GenericRegexChecker.__init__(
        self,
        'include "native_client/src/include/build_config.h"|' +
        'NACL_ANDROID|NACL_LINUX|NACL_OSX|NACL_WINDOWS|' +
        'NACL_BUILD_ARCH|NACL_BUILD_SUBARCH',
        analyze_match=True,
        message='must include "native_client/src/include/build_config.h"')

  def FindProblems(self, props, data):
    # Don't check build_config.h itself. The path is shortened here to make
    # this work on trybots.
    if 'src/include/build_config.h' in props['name']:
      return False
    self._seen_include = False
    return GenericRegexChecker.FindProblems(self, props, data)

  def IsProblemMatch(self, match):
    if 'include "native_client/src/include/build_config.h"' in match.group():
      self._seen_include = True
    return not self._seen_include

  def FileFilter(self, props):
    return '.c' in props or '.cc' in props or '.h' in props or '.S' in props


# ======================================================================
# 'props' (short for properties) are tags associated with checkable files.
#
# Except for the 'name', props do not carry any associated value.
#
# Props are used to make decisions on whether (and how) a certain check is
# applicable.
#
# Props are usually derived from the pathname but can also be forced using
# the magic string '@PROPS[prop1,prop2,...]' anywhere in the first 2k of
# data.

VALID_PROPS = {
    'name': True,                # Filename, only prop with a value.
    'is_makefile': True,
    'is_trusted': True,          # Is trusted code.
    'is_untrusted': True,        # Is untrusted code.
    'is_shared': True,           # Is shared code.
    'is_scons': True,            # Is scons file (not one generated by gyp).
}


def IsValidProp(prop):
  """Returns true if the property name is valid.

  Valid property names start with '.' or are listed in VALID_PROPS.

  Args:
    prop: a property name.
  Returns:
    True if the property name is valid.
  """
  if prop.startswith('.'):
    return True
  return prop in VALID_PROPS


def FindProperties(data):
  """Finds extra properties for a file.

  Finds strings of the form '@PROPS[name name name]'.

  Args:
    data: The contents of the file being checked.
  Returns:
    A list of properties from the PROPS line in the file.
  """
  match = re.search(r'@PROPS\[([^\]]*)\]', data)
  if match:
    return match.group(1).split(',')
  return []


def FileProperties(filename, data):
  """Finds all properties associated with this file.

  Valid properties are based on the file's name and on the '@PROPS' line in
  the file.

  Args:
    filename: the filename.
    data: The contents of the file being checked.
  Returns:
    A map of property to value for each property associated with this file.
  """
  d = {}
  d['name'] = filename
  _, ext = os.path.splitext(filename)
  if ext:
    d[ext] = True

  if 'akefile' in filename:
    d['is_makefile'] = True

  if 'src/trusted/' in filename:
    d['is_trusted'] = True
  if 'src/untrusted/' in filename:
    d['is_untrusted'] = True
  if 'src/shared/' in filename:
    d['is_shared'] = True

  if (filename.endswith('nacl.scons') or
      filename.endswith('build.scons') or
      filename.endswith('SConstruct')):
      d['is_scons'] = True

  # NOTE: Look in a somewhat arbitrary region at the file beginning for
  #       a special marker advertising addtional properties.
  for p in FindProperties(data[:2048]):
    d[p] = True

  for p in d:
    assert IsValidProp(p)
  return d


# ======================================================================
def IsBinary(data):
  """Checks for a 0x00 byte as a proxy for 'binary-ness'."""
  return '\x00' in data


# ======================================================================
# CHECK entry is: is_fatal, name, check_function.
CHECKS = [# fatal checks
          (True, 'not_for_commit_files', NotForCommitChecker()),
          (True, 'tabs', TabsChecker()),
          (True, 'trailing_space', TrailingWhiteSpaceChecker()),
          (True, 'cpp_comment', CppCommentChecker()),
          (True, 'fixme', FixmeChecker()),
          (True, 'include', IncludeChecker()),
          (True, 'build_config', BuildConfigChecker()),
          # Non fatal checks
          (False, 'carriage_return', CarriageReturnChecker()),
          (False, 'rewrite', RewriteChecker()),
          (False, 'tidy', TidyChecker()),
          (False, 'pychecker', PyChecker()),
          (False, 'cpplint', CppLintChecker()),
          ]


def EmitStatus(filename, status, details=[]):
  """Prints the status and all supporting details for a file."""
  print '%s: %s' % (filename, status)
  for no, line in details[:LIMIT]:
    print '  line %d: [%s]' % (no, repr(line))


def FindExemptions(data):
  """Looks for and returns EXEMPTION items from the data.

  Finds strings of the form '@EXEMPTION[name name name]' where
  'name' is one of the names in the CHECKS table.

  Args:
    data: The contents of the file being checked.
  """
  match = re.search(r'@EXEMPTION\[([^\]]*)\]', data)
  if match:
    return match.group(1).split(',')
  return []


def CheckFile(filename, report):
  """Runs all possible presubmit checks against the filename."""
  Debug('\n' + filename)
  errors = {}
  warnings = {}

  data = open(filename).read()

  if IsBinary(data):
    EmitStatus(filename, 'binary')
    return errors, warnings

  exemptions = FindExemptions(data)
  props = FileProperties(filename, data)
  Debug('PROPS: ' + repr(props.keys()))

  for fatal, name, check in CHECKS:
    Debug('trying %s' % name)

    if not check.FileFilter(props):
      Debug('skipping')
      continue

    if name in exemptions:
      Debug('exempt')
      continue

    problems = check.FindProblems(props, data)
    if problems:
      info = '%s: %s' % (filename, name)
      items = problems[:LIMIT]
      if report:
        print info
        for i in items:
          print i
      if fatal:
        errors[info] = items
        break
      else:
        warnings[info] = items
  return errors, warnings


def main(argv):
  """Runs all required presubmit checks for all listed files."""
  num_error = 0
  for filename in argv:
    if os.path.isdir(filename):
      EmitStatus(filename, 'directory')
      continue

    if '-' in os.path.basename(filename):
      print 'hyphen in:', filename
      num_error += 1
      continue

    errors, warnings = CheckFile(filename, True)
    if errors:
      num_error += 1
  return num_error

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))