base/android/jni_generator/jni_refactorer.py

#!/usr/bin/env python2.7
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tool for doing Java refactors over native methods.

Converts
(a) non-static natives to static natives using @JCaller
e.g.
class A {
  native void nativeFoo(int a);

  void bar() {
    nativeFoo(5);
  }
}
->
import .....JCaller
class A {
  static native void nativeFoo(@JCaller caller, int a);

  void bar() {
    nativeFoo(A.this, 5);
  }
}

(b) static natives to new mockable static natives.
e.g.
class A {
  static native void nativeFoo(@JCaller caller, int a);

  void bar() {
    nativeFoo(5);
  }
}
->
import .....JCaller
class A {
  void bar() {
    AJni.get().foo(A.this, 5);
  }

  @NativeMethods
  interface Natives {
    static native void foo(@JCaller caller, int a);
  }
}

Created for large refactors to @NativeMethods.
Note: This tool does most of the heavy lifting in the conversion but
there are some things that are difficult to implement with regex and
infrequent enough that they can be done by hand.

These exceptions are:
1) public native methods calls used in other classes are not refactored.
2) native methods inherited from a super class are not refactored
3) Non-static methods are always assumed to be called by the class instance
instead of by another class using that object.
"""

from __future__ import print_function

import argparse
import sys
import string
import re
import os
import pickle

import jni_generator

_JNI_INTERFACE_TEMPLATES = string.Template("""
    @NativeMethods
    interface ${INTERFACE_NAME} {${METHODS}
    }
""")

_JNI_METHOD_DECL = string.Template("""
        ${RETURN_TYPE} ${NAME}($PARAMS);""")

_COMMENT_REGEX_STRING = r'(?:(?:(?:\/\*[^\/]*\*\/)+|(?:\/\/[^\n]*?\n))+\s*)*'

_NATIVES_REGEX = re.compile(
    r'(?P<comments>' + _COMMENT_REGEX_STRING + ')'
    r'(?P<annotations>(@NativeClassQualifiedName'
    r'\(\"(?P<native_class_name>.*?)\"\)\s+)?'
    r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)'
    r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s+static\s*native '
    r'(?P<return_type>\S*) '
    r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL)

_NON_STATIC_NATIVES_REGEX = re.compile(
    r'(?P<comments>' + _COMMENT_REGEX_STRING + ')'
    r'(?P<annotations>(@NativeClassQualifiedName'
    r'\(\"(?P<native_class_name>.*?)\"\)\s+)?'
    r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)'
    r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s*native '
    r'(?P<return_type>\S*) '
    r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL)

JNI_IMPORT_STRING = 'import org.chromium.base.annotations.NativeMethods;'
JCALLER_IMPORT_STRING = 'import org.chromium.base.annotations.JCaller;'
IMPORT_REGEX = re.compile(r'^import .*?;', re.MULTILINE)

PICKLE_LOCATION = './jni_ref_pickle'


def build_method_declaration(return_type, name, params, annotations, comments):
  out = _JNI_METHOD_DECL.substitute({
      'RETURN_TYPE': return_type,
      'NAME': name,
      'PARAMS': params
  })
  if annotations:
    out = '\n' + annotations + out
  if comments:
    out = '\n' + comments + out
  if annotations or comments:
    out += '\n'
  return out


def add_chromium_import_to_java_file(contents, import_string):
  # Just in cases there are no imports default to after the package statement.
  import_insert = contents.find(';') + 1

  # Insert in alphabetical order into org.chromium. This assumes
  # that all files will contain some org.chromium import.
  for match in IMPORT_REGEX.finditer(contents):
    import_name = match.group()

    if not 'import org.chromium' in import_name:
      continue
    if import_name > import_insert:
      import_insert = match.start()
      break
    else:
      import_insert = match.end() + 1

  return "%s%s\n%s" % (contents[:import_insert], JCALLER_IMPORT_STRING,
                       contents[import_insert:])


def convert_nonstatic_to_static(java_file_name, dry=False, verbose=True):
  if java_file_name is None:
    return
  if not os.path.isfile(java_file_name):
    if verbose:
      print('%s does not exist', java_file_name)
    return

  with open(java_file_name, 'r') as f:
    contents = f.read()

  no_comment_content = jni_generator.RemoveComments(contents)
  parsed_natives = jni_generator.ExtractNatives(no_comment_content, 'long')
  non_static_natives = [n for n in parsed_natives if not n.static]
  if not non_static_natives:
    if verbose:
      print('no natives found')
    return

  # Import @JCaller import.
  contents = add_chromium_import_to_java_file(contents, JCALLER_IMPORT_STRING)

  class_name = jni_generator.ExtractFullyQualifiedJavaClassName(
      java_file_name, no_comment_content).split('/')[-1]

  replace_patterns = []
  should_add_comma = []

  new_contents = contents

  # 1. Change non-static -> static.
  insertion_offset = 0

  matches = []
  for match in _NON_STATIC_NATIVES_REGEX.finditer(contents):
    if not 'static' in match.group('qualifiers'):
      matches.append(match)
      # Insert static as a keyword.
      qual_end = match.end('qualifiers') + insertion_offset
      insert_str = ' static '
      new_contents = new_contents[:qual_end] + insert_str + new_contents[
          qual_end:]
      insertion_offset += len(insert_str)

      # Insert an object param.
      start = insertion_offset + match.start('params')
      insert_str = '@JCaller %s caller' % class_name
      if match.group('params'):
        insert_str += ', '

      # Match lines that don't have a native keyword.
      replace_patterns.append(r'(^\s*' + match.group('name') + r'\()')
      replace_patterns.append(r'(return ' + match.group('name') + r'\()')
      replace_patterns.append(r'([\:\)\(\+\*\?\&\|,\.\-\=\!\/][ \t]*' +
                              match.group('name') + r'\()')

      add_comma = bool(match.group('params'))
      should_add_comma.extend([add_comma] * 3)

      new_contents = new_contents[:start] + insert_str + new_contents[start:]
      insertion_offset += len(insert_str)

  assert len(matches) == len(non_static_natives), ('Regex missed a native '
                                                   'method that was found by '
                                                   'the jni_generator.')

  # 2. Add a this param to all calls.
  for i, r in enumerate(replace_patterns):
    if should_add_comma[i]:
      new_contents = re.sub(
          r, '\g<1>%s.this, ' % class_name, new_contents, flags=re.MULTILINE)
    else:
      new_contents = re.sub(
          r, '\g<1>%s.this' % class_name, new_contents, flags=re.MULTILINE)

  if dry:
    print(new_contents)
  else:
    with open(java_file_name, 'w') as f:
      f.write(new_contents)


def filter_files_with_natives(files, verbose=True):
  filtered = []
  i = 1
  for java_file_name in files:
    if not os.path.isfile(java_file_name):
      print('does not exist')
      return
    if verbose:
      print('Processing %s/%s - %s ' % (i, len(files), java_file_name))
    with open(java_file_name, 'r') as f:
      contents = f.read()
    no_comment_content = jni_generator.RemoveComments(contents)
    natives = jni_generator.ExtractNatives(no_comment_content, 'long')

    if len(natives) > 1:
      filtered.append(java_file_name)
    i += 1
  return filtered


def convert_file_to_proxy_natives(java_file_name, dry=False, verbose=True):
  if not os.path.isfile(java_file_name):
    if verbose:
      print('%s does not exist', java_file_name)
    return

  with open(java_file_name, 'r') as f:
    contents = f.read()

  no_comment_content = jni_generator.RemoveComments(contents)
  natives = jni_generator.ExtractNatives(no_comment_content, 'long')

  static_natives = [n for n in natives if n.static]

  if not static_natives:
    if verbose:
      print('%s has no static natives.', java_file_name)
    return

  contents = add_chromium_import_to_java_file(contents, JNI_IMPORT_STRING)

  # Extract comments and annotations above native methods.
  native_map = {}
  for itr in re.finditer(_NATIVES_REGEX, contents):
    n_dict = {}
    n_dict['annotations'] = itr.group('annotations').strip()
    n_dict['comments'] = itr.group('comments').strip()
    n_dict['params'] = itr.group('params').strip()
    native_map[itr.group('name')] = n_dict

  # Using static natives here ensures all the methods that are picked up by
  # the JNI generator are also caught by our own regex.
  methods = []
  for n in static_natives:
    new_name = n.name[0].lower() + n.name[1:]
    n_dict = native_map['native' + n.name]
    params = n_dict['params']
    comments = n_dict['comments']
    annotations = n_dict['annotations']
    methods.append(
        build_method_declaration(n.return_type, new_name, params, annotations,
                                 comments))

  fully_qualified_class = jni_generator.ExtractFullyQualifiedJavaClassName(
      java_file_name, contents)
  class_name = fully_qualified_class.split('/')[-1]
  jni_class_name = class_name + 'Jni'

  # Remove all old declarations.
  for n in static_natives:
    pattern = _NATIVES_REGEX
    contents = re.sub(pattern, '', contents)

  # Replace occurences with new signature.
  for n in static_natives:
    # Okay not to match first (.
    # Since even if natives share a prefix, the replacement is the same.
    # E.g. if nativeFoo() and nativeFooBar() are both statics
    # and in nativeFooBar() we replace nativeFoo -> AJni.get().foo
    # the result is the same as replacing nativeFooBar() -> AJni.get().fooBar
    pattern = r'native%s' % n.name
    lower_name = n.name[0].lower() + n.name[1:]
    contents = re.sub(pattern, '%s.get().%s' % (jni_class_name, lower_name),
                      contents)

  # Build and insert the @NativeMethods interface.
  interface = _JNI_INTERFACE_TEMPLATES.substitute({
      'INTERFACE_NAME': 'Natives',
      'METHODS': ''.join(methods)
  })

  # Insert the interface at the bottom of the top level class.
  # Most of the time this will be before the last }.
  insertion_point = contents.rfind('}')
  contents = contents[:insertion_point] + '\n' + interface + contents[
      insertion_point:]

  if not dry:
    with open(java_file_name, 'w') as f:
      f.write(contents)
  else:
    print(contents)
  return contents


def main(argv):
  arg_parser = argparse.ArgumentParser()

  mutually_ex_group = arg_parser.add_mutually_exclusive_group()

  mutually_ex_group.add_argument(
      '-R',
      '--recursive',
      action='store_true',
      help='Run recursively over all java files '
      'descendants of the current directory.',
      default=False)
  mutually_ex_group.add_argument(
      '--read_cache',
      help='Reads paths to refactor from pickled file %s.' % PICKLE_LOCATION,
      action='store_true',
      default=False)
  mutually_ex_group.add_argument(
      '--source', help='Path to refactor single source file.', default=None)

  arg_parser.add_argument(
      '--cache',
      action='store_true',
      help='Finds all java files with native functions recursively from '
      'the current directory, then pickles and saves them to %s and then'
      'exits.' % PICKLE_LOCATION,
      default=False)
  arg_parser.add_argument(
      '--dry_run',
      default=False,
      action='store_true',
      help='Print refactor output to console instead '
      'of replacing the contents of files.')
  arg_parser.add_argument(
      '--nonstatic',
      default=False,
      action='store_true',
      help='If true converts native nonstatic methods to static methods'
      ' instead of converting static methods to new jni.')
  arg_parser.add_argument(
      '--verbose', default=False, action='store_true', help='')

  args = arg_parser.parse_args()

  java_file_paths = []

  if args.source:
    java_file_paths = [args.source]
  elif args.read_cache:
    print('Reading paths from ' + PICKLE_LOCATION)
    with open(PICKLE_LOCATION, 'r') as file:
      java_file_paths = pickle.load(file)
      print('Found %s java paths.' % len(java_file_paths))
  elif args.recursive:
    ignored_paths = [
        'third_party', 'src/out', 'out/Debug', 'library_loader', '.cipd',
        'jni_generator', 'media', 'accessibility', '/vr', 'website',
        'gcm_driver', 'preferences'
    ]

    for root, dirs, files in os.walk(os.path.abspath('.')):

      def getPaths():
        for ignored_path in ignored_paths:
          if ignored_path in root:
            return

          java_file_paths.extend(
              ['%s/%s' % (root, f) for f in files if f.endswith('.java')])

      getPaths()

  else:
    # Get all java files in current dir.
    java_file_paths = filter(lambda x: x.endswith('.java'),
                             map(os.path.abspath, os.listdir('.')))

  if args.cache:
    with open(PICKLE_LOCATION, 'w') as file:
      pickle.dump(filter_files_with_natives(java_file_paths), file)
      print('Java files with proxy natives written to ' + PICKLE_LOCATION)

  i = 1
  for f in java_file_paths:
    print(f)
    if args.nonstatic:
      convert_nonstatic_to_static(f, dry=args.dry_run, verbose=args.verbose)
    else:
      convert_file_to_proxy_natives(f, dry=args.dry_run, verbose=args.verbose)
    print('Done converting %s/%s' % (i, len(java_file_paths)))
    i += 1

  print('Done please run git cl format.')


if __name__ == '__main__':
  sys.exit(main(sys.argv))