blob: f82f502b6db3a3b3853deb55c7702c65da115bdc [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
Parses SQLite source code and produces renaming macros for its exported symbols.
Usage: sqlite.h rename_macros.h
For example, the following renaming macro is produced for sqlite3_initialize().
#define sqlite3_initialize chrome_sqlite3_initialize
import re
import sys
def ExtractLineTuples(string):
'''Returns a list of lines, with start/end whitespace stripped.
Each line is a tuple of (line number, string).
raw_lines = string.split('\n')
stripped_lines = [line.strip() for line in raw_lines]
return list(enumerate(stripped_lines, start=1))
def ExtractPreprocessorDirectives(lines):
'''Extracts preprocessor directives from lines of C code.
Each input line should be a tuple of (line number, string).
Returns a list of preprocessor directives, and a list of C code lines with the
preprocessor directives removed. The returned code lines are a subset of the
input tuples.
code_lines = []
directives = []
in_directive = False
last_directive = []
for line_tuple in lines:
line = line_tuple[1]
# Preprocessor directives start with #.
if not in_directive:
if len(line) > 0 and line[0] == '#':
in_directive = True
last_directive = []
# Preprocessor directives use \ as a line continuation character.
if in_directive:
if line[-1] == '\\':
line = line[:-1]
in_directive = False
if not in_directive:
return directives, code_lines
# Regular expression used to parse a macro definition.
DEFINITION_RE = re.compile(r'^\#\s*define\s+(\w+)(\s|$)')
def ExtractDefineMacroName(line):
'''Extracts the macro name from a non-function preprocessor definition.
Returns None if the preprocessor line is not a preprocessor macro definition.
Macro functions are not considered preprocessor definitions.
match = DEFINITION_RE.match(line)
if match is None:
return None
# Matches C++-style // single-line comments.
SINGLE_LINE_COMMENT_RE = re.compile(r'//.*$')
# Matches C-style /* multi-line comments */.
MULTI_LINE_COMMENT_RE = re.compile(r'/\*.*?\*/', flags=re.MULTILINE|re.DOTALL)
def RemoveLineComments(line):
'''Returns the given C code line with comments removed.
This handles both C-style /* comments */ and C++-style // comments, but cannot
tackle C-style comments that extend over multiple lines.
return SINGLE_LINE_COMMENT_RE.sub('', MULTI_LINE_COMMENT_RE.sub('', line))
def RemoveComments(code_tuples):
'Returns the given C code tuples with all comments removed.'
output_tuples = []
in_comment = False
for line_number, line in code_tuples:
if in_comment:
if '*/' in line:
_, line = line.split('*/', 1)
in_comment = False
if not in_comment:
line = RemoveLineComments(line)
if '/*' in line:
line, _ = line.split('/*', 1)
in_comment = True
output_tuples.append((line_number, line))
return output_tuples
# Splits a line of C code into statement pieces.
STATEMENT_BREAK_RE = re.compile(r'[;{}]')
def ToStatementTuples(code_tuples):
'''Converts C code lines into statements.
The input is tuples of (line number, line code string). The output is
tuples of (min line, max line, statement).
The function considers ; { and } to be statement separators. This is
sufficiently correct, given our goal.
statements = []
current_statement = ''
current_start = 0
for line_number, line in code_tuples:
pieces = STATEMENT_BREAK_RE.split(line)
for piece in pieces[:-1]: # The last piece is an unfinished statement.
if current_statement != '':
current_statement = current_statement + '\n' + piece
(current_start, line_number, current_statement.strip()))
current_statement = ''
statements.append((line_number, line_number, piece.strip()))
if current_statement == '':
current_start = line_number
if pieces[-1] != '':
current_statement = current_statement + '\n' + pieces[-1]
return statements
# Used to break down a line into words.
WHITESPACE_RE = re.compile(r'\s+')
# Features unsupported by our extractor.
# We do not support parsing struct and enum literals because sqlite typedefs
# them before incorporating them into exported symbols. We can avoid matching
# curly braces because we do not support enum, struct, or union, and we only
# need to consider declarations involving typedef names and primitive types.
# Type qualifiers that we can skip over.
# We discard storage-class specifiers and type qualifiers. For purposes of
# finding the end of declaration specifiers, they are not needed. This
# additionally discards any pointer type qualifiers.
# Keywords used in composite primitive types.
# Types using these keywords may have more than one keyword, e.g.
# "long long int".
# Matches an identifier.
IDENTIFIER_RE = re.compile(r'^[a-zA-Z_0-9]+$')
def ExtractApiExport(macro_names, api_export_macro, statement):
'''Extracts the symbol name from a statement exporting a function.
Returns None if the statement does not export a symbol. Throws ValueError if
the parser cannot understand the statement.
# See, section 6.7
# for how to parse C declarations. Note that a declaration is a number of
# declaration-specifiers, followed by a list of declarators with optional
# initializer. Multiple declarators would be a declaration like:
# int a, b;
# While, in principle, one could declare a pair of C functions like this, no
# one does it. We assume there is only one declarator.
# int foo(int), bar(int, int);
# Jumping to section 6.7.5, a declarator includes some optional pointer
# specifiers (which may have type qualifiers like 'const' embedded, e.g. 'int
# * const * const foo') and some grouping. Note, however, that in all cases,
# the declaration name is the first non-type-qualifier identifier.
# Thus our goal is to skip the declaration specifiers and get to the
# declarators.
# Simplification: get rid of pointer characters.
statement = statement.replace('*', ' ')
# Simplification: make sure each open parenthesis is each own word.
statement = statement.replace('(', ' ( ')
statement = statement.replace('[', ' [ ')
words = WHITESPACE_RE.split(statement)
# Ignore statements that don't deal with exporting symbols.
if api_export_macro not in words:
return None
seen_composite_type = False
seen_simple_type = False
for word in words:
raise ValueError("Unsupported keyword %s" % word)
# Per section 6.7.2, we must have at least one type specifier (so the first
# token is one). Moreover, clause 2 implies that if we have a typedef name,
# enum, struct, or union, it is the only type specifier. If we have a
# keyword such as 'int', we may have one or more of such keywords.
if seen_simple_type:
raise ValueError('Mixed simple (struct_name) and composite (int) types')
seen_composite_type = True
# We assume that macros are only used for qualifiers, which can be skipped.
if word in macro_names or word == api_export_macro:
if not seen_composite_type and not seen_simple_type:
seen_simple_type = True
if IDENTIFIER_RE.match(word) is None:
raise ValueError(
"%s parsed as type name, which doesn't make sense" % word)
if IDENTIFIER_RE.match(word) is None:
raise ValueError(
"%s parsed as symbol name, which doesn't make sense" % word)
return word
raise ValueError('Failed to find symbol name')
def ExportedSymbolLine(symbol_prefix, symbol, statement_tuple):
'Returns an output line for an exported symbol.'
if statement_tuple[0] == statement_tuple[1]:
lines = 'Line %d' % statement_tuple[0]
lines = 'Lines %d-%d' % (statement_tuple[0], statement_tuple[1])
return '#define %s %s%s // %s' % (symbol, symbol_prefix, symbol, lines)
def ExportedExceptionLine(exception, statement_tuple):
'Returns an output line for a parsing failure.'
# Output a TODO without a name so the broken parsing result doesn't
# accidentally get checked in.
return '// TODO: Lines %d-%d -- %s' % (
statement_tuple[0], statement_tuple[1], exception.message)
def ProcessSource(api_export_macro, symbol_prefix, header_line, footer_line,
'Returns a list of lines that rename exported symbols in an C program file.'
line_tuples = ExtractLineTuples(file_content)
line_tuples = RemoveComments(line_tuples)
directives, code_tuples = ExtractPreprocessorDirectives(line_tuples)
macro_names = set(
name for name in
[ExtractDefineMacroName(directive) for directive in directives]
if name is not None)
statement_tuples = ToStatementTuples(code_tuples)
output_lines = []
for statement_tuple in statement_tuples:
line = statement_tuple[2]
symbol_name = ExtractApiExport(macro_names, api_export_macro, line)
if symbol_name:
ExportedSymbolLine(symbol_prefix, symbol_name, statement_tuple))
except ValueError as exception:
output_lines.append(ExportedExceptionLine(exception, statement_tuple))
return [header_line] + output_lines + [footer_line]
def ProcessSourceFile(api_export_macro, symbol_prefix, header_line,
footer_line, input_file, output_file):
'Reads in a C program file and outputs macros renaming exported symbols.'
with open(input_file, 'r') as f:
file_content =
output_lines = ProcessSource(api_export_macro, symbol_prefix, header_line,
footer_line, file_content)
with open(output_file, 'w') as f:
header_line='''// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This file is generated by
footer_line ='''
if __name__ == '__main__':
ProcessSourceFile(api_export_macro='SQLITE_API', symbol_prefix='chrome_',
header_line=header_line, footer_line=footer_line,
input_file=sys.argv[1], output_file=sys.argv[2])