tools/git/suggest_owners.py

#!/usr/bin/env python3
# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import argparse
import subprocess
import pickle
import re
import os
from pathlib import PurePath
from os import path
from datetime import date, timedelta
from collections import namedtuple, defaultdict

Commit = namedtuple('Commit', ['hash', 'author', 'commit_date', 'dirs'])

# dict mapping each subdirectory and author to the number of their commits and
# modifications in that directory
DIRECTORY_AUTHORS = defaultdict(dict)

# cache for directory owners for memoisation purposes
OWNERS_CACHE = {}

# filename for pickle cache
CACHE_FILENAME = 'suggest_owners.cache'


def _RunGitCommand(options, cmd_args, pipe_output=False):
  repo_path = path.join(options.repo_path, '.git')
  cmd = ['git', '--git-dir', repo_path] + cmd_args
  print('>', ' '.join(cmd))
  if not pipe_output:
    return subprocess.check_output(cmd, encoding='utf-8')
  else:
    return subprocess.Popen(cmd, encoding='utf-8',
                            stdout=subprocess.PIPE).stdout


def _ValidAuthor(author):
  return author.endswith(
      ('@chromium.org', '@google.com')) and 'roller' not in author


# Returns additions/deletions by a commit to a directory (and its descendants).
def getEditsForDirectory(commit, directory):
  additions = deletions = 0
  for commit_directory, (directory_additions, directory_deletions) \
      in commit.dirs.items():
    # check if commit_directory is same as or a descendant of directory
    if isSubDirectory(directory, commit_directory):
      additions += directory_additions
      deletions += directory_deletions
  return additions, deletions


# This propagates a commit touching a directory to also be touching all
# ancesstor directories.
def _PropagateCommit(options, commit):
  touched_dirs = set()
  # first get all the touched dirs and their ancestors
  for directory in commit.dirs.keys():
    # PurePath.parent returns '.' for non absolute paths in the limit.
    while str(directory) != '.':
      touched_dirs.add(str(directory))
      # get the parent directory
      directory = PurePath(directory).parent
  # loop over them and calculate the edits per directory
  for directory in touched_dirs:
    author_commits, author_additions, author_deletions = \
        DIRECTORY_AUTHORS[directory].get(commit.author, (0,0,0))
    directory_additions, directory_deletions = \
        getEditsForDirectory(commit, directory)
    DIRECTORY_AUTHORS[directory][commit.author] = \
        (author_commits + 1, author_additions + directory_additions,
         author_deletions + directory_deletions)


# Checks if child_directory is same as or below parent_directory. For some
# reason the os.path module does not have this functionality.
def isSubDirectory(parent_directory, child_directory):
  parent_directory = PurePath(parent_directory)
  child_directory = PurePath(child_directory)
  return child_directory.is_relative_to(parent_directory)


def _GetGitLogCmd(options):
  # TODO(mheikal): git-log with --numstat vs --name-only takes 10x the time to
  # complete. It takes >15 mins for git log --numstat to return the 1 year git
  # history of the full repo. Should probably add a script flag to switch off
  # keeping track of number of modifications per commit.
  date_limit = date.today() - timedelta(days=options.days_ago)
  format_string = "%h,%ae,%cI"
  cmd_args = [
    'log',
    '--since', date_limit.isoformat(),
    '--numstat',
    '--pretty=format:%s'%format_string,
  ]
  # has to be last arg
  if options.subdirectory:
    cmd_args += ['--', options.subdirectory]
  return cmd_args


def _ParseCommitLine(line):
  commit_hash, author, commit_date = line.split(",")
  return Commit(hash=commit_hash, author=author, commit_date=commit_date,
                dirs={})


def _ParseFileStatsLine(current_commit, line):
  try:
    additions, deletions, filepath = line.split('\t')
  except ValueError:
    return False
  if additions == '-':
    additions = 0
  else:
    additions = int(additions)
  if deletions == '-':
    deletions = 0
  else:
    deletions = int(deletions)
  if additions == 0 and deletions == 0:
    return True
  dir_path = path.dirname(filepath)
  # For git renames, we count the destination directory
  if '=>' in dir_path:
    dir_path = re.sub(r'\{[^=]* => ([^\}]*)\}', r'\1', dir_path)
    # remove possibly empty path parts.
    dir_path = dir_path.replace('//', '/')
  commit_additions, commit_deletions = \
      current_commit.dirs.get(dir_path, (0,0))
  current_commit.dirs[dir_path] = (
      additions + commit_additions, deletions + commit_deletions)
  return True


def processAllCommits(options):
  if not options.subdirectory and options.days_ago > 100:
    print('git log for your query might take > 5 minutes, limit by a '
          'subdirectory or reduce the number of days of history to low double '
          'digits to make this faster. There is no progress indicator, it is '
          'all waiting for single git log to finish.')
  output_pipe = _RunGitCommand(options,
                               _GetGitLogCmd(options),
                               pipe_output=True)
  current_commit = None
  for line in iter(output_pipe.readline, ''):
    line = line.rstrip('\n')
    if current_commit is None:
      current_commit = _ParseCommitLine(line)
    else:
      if line == '': # all commit details read
        if _ValidAuthor(current_commit.author):
          _PropagateCommit(options, current_commit)
        current_commit = None
      else:
        # Merge commits weird out git-log. If we fail to parse the line, then
        # the last commit was a merge and this line is actually another commit
        # description line.
        if not _ParseFileStatsLine(current_commit, line):
          current_commit = _ParseCommitLine(line)
  # process the final commit
  if _ValidAuthor(current_commit.author):
    _PropagateCommit(options, current_commit)
  print('Done parsing commit log.')


def _CountCommits(directory):
  return sum(
      [count for (count, _a, _d) in DIRECTORY_AUTHORS[directory].values()])


def _GetOwnerLevel(options, author, directory):
  sorted_owners = sorted(_GetOwners(options, directory), key=lambda e: e[1])
  for owner, level in sorted_owners:
    if author == owner:
      return level
  else:
    return -1


# Returns the owners for a repo subdirectory. This does not understand per-file
# directives.
# TODO(mheikal): use depot_tools owners.py for parsing owners files.
def _GetOwners(options, directory_path):
  if directory_path in OWNERS_CACHE:
    return OWNERS_CACHE[directory_path]
  owners_path = path.join(options.repo_path, directory_path, 'OWNERS')
  owners = set()
  parent_dir = directory_path
  owner_level = 0
  while parent_dir != '':
    if path.isfile(owners_path):
      parsed_owners, noparent = _ParseOwnersFile(options, owners_path)
      owners.update([(owner, owner_level) for owner in parsed_owners])
      owner_level += 1
      if noparent:
        break
    parent_dir = path.dirname(parent_dir)
    owners_path = path.join(parent_dir, 'OWNERS')
  OWNERS_CACHE[directory_path] = set(owners)
  return owners


# Parse an OWNERS file, returns set of owners and if the file sets noparent
def _ParseOwnersFile(options, filepath):
  owners = set()
  noparent = False
  with open(filepath) as f:
    for line in f.readlines():
      line = line.strip()
      # The script deals with directories so per-files are ignored.
      if line == '' or line[0] == '#' or line.startswith('per-file'):
        continue
      if line.startswith('file://'):
        relpath = line[7:]
        abspath = path.join(options.repo_path, relpath)
        parsed_owners, _ = _ParseOwnersFile(options, abspath)
        owners.update(parsed_owners)
      if line == 'set noparent':
        noparent = True
      index = line.find('@chromium.org')
      if index > -1:
        owners.add(line[:index + len('@chromium.org')])
  return owners, noparent


# Trivial directories are ones that just contain a single child subdir and
# nothing else.
def _IsTrivialDirectory(options, repo_subdir):
  try:
    return len(os.listdir(path.join(options.repo_path, repo_subdir))) == 1
  except OSError:
    # directory no longer exists
    return False


def computeSuggestions(options):
  directory_suggestions = []
  for directory, authors in sorted(DIRECTORY_AUTHORS.items()):
    if _IsTrivialDirectory(options, directory):
      continue
    if _CountCommits(directory) < options.dir_commit_limit:
      continue
    # skip suggestions for directories outside the passed in directory
    if (options.subdirectory
        and not isSubDirectory(options.subdirectory, directory)):
      continue
    # sort authors by descending number of commits
    sorted_authors = sorted(authors.items(), key=lambda entry: -entry[1][0])
    # keep only authors above the limit
    suggestions = [(a,c) for a,c in sorted_authors if \
                   a not in options.ignore_authors \
                   and c[0] >= options.author_cl_limit]
    directory_suggestions.append((directory, suggestions))
  return directory_suggestions


def _PrintSettings(options):
  print('Showing directories with at least ({}) commits in the last ({}) '
        'days.'.format(options.dir_commit_limit, options.days_ago))
  print('Showing top ({}) committers who have commited at least ({}) commits '
        'to the directory in the last ({}) days.'.format(
            options.max_suggestions, options.author_cl_limit,
            options.days_ago))
  print('(owners+N) represents distance through OWNERS files for said owner\n')


def printSuggestions(options, directory_suggestions):
  print('\nCommit stats:')
  _PrintSettings(options)
  for directory, suggestions in directory_suggestions:
    print('{}: {} commits in the last {} days'.format(
        directory, _CountCommits(directory), options.days_ago))
    non_owner_suggestions = 0
    for author, (commit_count, additions, deletions) in suggestions:
      owner_level = _GetOwnerLevel(options, author, directory)
      if owner_level > -1:
        owner_string = ' (owner+{})'.format(owner_level)
      else:
        non_owner_suggestions +=1
        owner_string = ''
      print('{}{}, commits: {}, additions:{}, deletions: {}'.format(
          author, owner_string, commit_count, additions, deletions))
      if non_owner_suggestions >= options.max_suggestions:
        break
    print()


def _GetHeadCommitHash(options):
  return _RunGitCommand(options, ['rev-parse', 'HEAD']).strip()


def _GetCacheMetadata(options):
  return _GetHeadCommitHash(options), options.days_ago, options.subdirectory


def _IsCacheValid(options, metadata):
  head_hash, days_ago, cached_subdirectory = metadata
  if head_hash != _GetHeadCommitHash(options):
    return False
  if days_ago != options.days_ago:
    return False
  if (cached_subdirectory is not None
      and not isSubDirectory(cached_subdirectory, options.subdirectory)):
    return False
  return True


def cacheProcessedCommits(options):
  metadata = _GetCacheMetadata(options)
  with open(CACHE_FILENAME, 'wb') as f:
    pickle.dump((metadata, DIRECTORY_AUTHORS), f)


def maybeRestoreProcessedCommits(options):
  global DIRECTORY_AUTHORS
  if not path.exists(CACHE_FILENAME):
    return False
  with open(CACHE_FILENAME, 'rb') as f:
    stored_metadata, cached_directory_authors = pickle.load(f)
    if _IsCacheValid(options, stored_metadata):
      print('Loading from cache')
      DIRECTORY_AUTHORS = cached_directory_authors
      return True
    else:
      print('Cache is stale or invalid, must rerun `git log`')
      return False

def do(options):
  if options.skip_cache or not maybeRestoreProcessedCommits(options):
    processAllCommits(options)
    cacheProcessedCommits(options)
  directory_suggestions = computeSuggestions(options)
  printSuggestions(options, directory_suggestions)


def main():
  parser = argparse.ArgumentParser(
      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument('repo_path')
  parser.add_argument('--days-ago', type=int,
                      help='Number of days of history to search through.',
                      default=365, metavar='DAYS_AGO')
  parser.add_argument('--subdirectory',
                      help='Limit suggestions to this subdirectory', default='')
  parser.add_argument('--ignore-authors',
                      help='Ignore this comma separated list of authors')
  parser.add_argument('--max-suggestions', type=int, help='Maximum number of '
                      'suggested authors per directory.', default=5)
  parser.add_argument('--author-cl-limit', type=int, help='Do not suggest '
                      'authors who have commited less than this to the '
                      'directory in the last DAYS_AGO days.', default=10)
  parser.add_argument('--dir-commit-limit', type=int, help='Skip directories '
                      'with less than this number of commits in the last '
                      'DAYS_AGO days.', default=100)
  parser.add_argument('--skip-cache', action='store_true',
                      help='Do not read from cache.', default=False)
  options = parser.parse_args()
  if options.ignore_authors:
    options.ignore_authors = set(
        map(str.strip, options.ignore_authors.split(',')))
  else:
    options.ignore_authors = set()
  do(options)


if __name__ == '__main__':
  main()