| import logging | |
| import sys | |
| import textwrap | |
| from c_common.scriptutil import ( | |
| VERBOSITY, | |
| add_verbosity_cli, | |
| add_traceback_cli, | |
| add_commands_cli, | |
| add_kind_filtering_cli, | |
| add_files_cli, | |
| add_progress_cli, | |
| process_args_by_key, | |
| configure_logger, | |
| get_prog, | |
| ) | |
| from c_parser.info import KIND | |
| import c_parser.__main__ as c_parser | |
| import c_analyzer.__main__ as c_analyzer | |
| import c_analyzer as _c_analyzer | |
| from c_analyzer.info import UNKNOWN | |
| from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT | |
| logger = logging.getLogger(__name__) | |
| CHECK_EXPLANATION = textwrap.dedent(''' | |
| ------------------------- | |
| Non-constant global variables are generally not supported | |
| in the CPython repo. We use a tool to analyze the C code | |
| and report if any unsupported globals are found. The tool | |
| may be run manually with: | |
| ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE] | |
| Occasionally the tool is unable to parse updated code. | |
| If this happens then add the file to the "EXCLUDED" list | |
| in Tools/c-analyzer/cpython/_parser.py and create a new | |
| issue for fixing the tool (and CC ericsnowcurrently | |
| on the issue). | |
| If the tool reports an unsupported global variable and | |
| it is actually const (and thus supported) then first try | |
| fixing the declaration appropriately in the code. If that | |
| doesn't work then add the variable to the "should be const" | |
| section of Tools/c-analyzer/cpython/ignored.tsv. | |
| If the tool otherwise reports an unsupported global variable | |
| then first try to make it non-global, possibly adding to | |
| PyInterpreterState (for core code) or module state (for | |
| extension modules). In an emergency, you can add the | |
| variable to Tools/c-analyzer/cpython/globals-to-fix.tsv | |
| to get CI passing, but doing so should be avoided. If | |
| this course it taken, be sure to create an issue for | |
| eliminating the global (and CC ericsnowcurrently). | |
| ''') | |
| def _resolve_filenames(filenames): | |
| if filenames: | |
| resolved = (_files.resolve_filename(f) for f in filenames) | |
| else: | |
| resolved = _files.iter_filenames() | |
| return resolved | |
| ####################################### | |
| # the formats | |
| def fmt_summary(analysis): | |
| # XXX Support sorting and grouping. | |
| supported = [] | |
| unsupported = [] | |
| for item in analysis: | |
| if item.supported: | |
| supported.append(item) | |
| else: | |
| unsupported.append(item) | |
| total = 0 | |
| def section(name, groupitems): | |
| nonlocal total | |
| items, render = c_analyzer.build_section(name, groupitems, | |
| relroot=REPO_ROOT) | |
| yield from render() | |
| total += len(items) | |
| yield '' | |
| yield '====================' | |
| yield 'supported' | |
| yield '====================' | |
| yield from section('types', supported) | |
| yield from section('variables', supported) | |
| yield '' | |
| yield '====================' | |
| yield 'unsupported' | |
| yield '====================' | |
| yield from section('types', unsupported) | |
| yield from section('variables', unsupported) | |
| yield '' | |
| yield f'grand total: {total}' | |
| ####################################### | |
| # the checks | |
| CHECKS = dict(c_analyzer.CHECKS, **{ | |
| 'globals': _analyzer.check_globals, | |
| }) | |
| ####################################### | |
| # the commands | |
| FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*') | |
| def _cli_parse(parser): | |
| process_output = c_parser.add_output_cli(parser) | |
| process_kind = add_kind_filtering_cli(parser) | |
| process_preprocessor = c_parser.add_preprocessor_cli( | |
| parser, | |
| get_preprocessor=_parser.get_preprocessor, | |
| ) | |
| process_files = add_files_cli(parser, **FILES_KWARGS) | |
| return [ | |
| process_output, | |
| process_kind, | |
| process_preprocessor, | |
| process_files, | |
| ] | |
| def cmd_parse(filenames=None, **kwargs): | |
| filenames = _resolve_filenames(filenames) | |
| if 'get_file_preprocessor' not in kwargs: | |
| kwargs['get_file_preprocessor'] = _parser.get_preprocessor() | |
| c_parser.cmd_parse( | |
| filenames, | |
| relroot=REPO_ROOT, | |
| file_maxsizes=_parser.MAX_SIZES, | |
| **kwargs | |
| ) | |
| def _cli_check(parser, **kwargs): | |
| return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS) | |
| def cmd_check(filenames=None, **kwargs): | |
| filenames = _resolve_filenames(filenames) | |
| kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) | |
| try: | |
| c_analyzer.cmd_check( | |
| filenames, | |
| relroot=REPO_ROOT, | |
| _analyze=_analyzer.analyze, | |
| _CHECKS=CHECKS, | |
| file_maxsizes=_parser.MAX_SIZES, | |
| **kwargs | |
| ) | |
| except SystemExit as exc: | |
| num_failed = exc.args[0] if getattr(exc, 'args', None) else None | |
| if isinstance(num_failed, int): | |
| if num_failed > 0: | |
| sys.stderr.flush() | |
| print(CHECK_EXPLANATION, flush=True) | |
| raise # re-raise | |
| except Exception: | |
| sys.stderr.flush() | |
| print(CHECK_EXPLANATION, flush=True) | |
| raise # re-raise | |
| def cmd_analyze(filenames=None, **kwargs): | |
| formats = dict(c_analyzer.FORMATS) | |
| formats['summary'] = fmt_summary | |
| filenames = _resolve_filenames(filenames) | |
| kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) | |
| c_analyzer.cmd_analyze( | |
| filenames, | |
| relroot=REPO_ROOT, | |
| _analyze=_analyzer.analyze, | |
| formats=formats, | |
| file_maxsizes=_parser.MAX_SIZES, | |
| **kwargs | |
| ) | |
| def _cli_data(parser): | |
| filenames = False | |
| known = True | |
| return c_analyzer._cli_data(parser, filenames, known) | |
| def cmd_data(datacmd, **kwargs): | |
| formats = dict(c_analyzer.FORMATS) | |
| formats['summary'] = fmt_summary | |
| filenames = (file | |
| for file in _resolve_filenames(None) | |
| if file not in _parser.EXCLUDED) | |
| kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) | |
| if datacmd == 'show': | |
| types = _analyzer.read_known() | |
| results = [] | |
| for decl, info in types.items(): | |
| if info is UNKNOWN: | |
| if decl.kind in (KIND.STRUCT, KIND.UNION): | |
| extra = {'unsupported': ['type unknown'] * len(decl.members)} | |
| else: | |
| extra = {'unsupported': ['type unknown']} | |
| info = (info, extra) | |
| results.append((decl, info)) | |
| if decl.shortkey == 'struct _object': | |
| tempinfo = info | |
| known = _analyzer.Analysis.from_results(results) | |
| analyze = None | |
| elif datacmd == 'dump': | |
| known = _analyzer.KNOWN_FILE | |
| def analyze(files, **kwargs): | |
| decls = [] | |
| for decl in _analyzer.iter_decls(files, **kwargs): | |
| if not KIND.is_type_decl(decl.kind): | |
| continue | |
| if not decl.filename.endswith('.h'): | |
| if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C: | |
| continue | |
| decls.append(decl) | |
| results = _c_analyzer.analyze_decls( | |
| decls, | |
| known={}, | |
| analyze_resolved=_analyzer.analyze_resolved, | |
| ) | |
| return _analyzer.Analysis.from_results(results) | |
| else: # check | |
| known = _analyzer.read_known() | |
| def analyze(files, **kwargs): | |
| return _analyzer.iter_decls(files, **kwargs) | |
| extracolumns = None | |
| c_analyzer.cmd_data( | |
| datacmd, | |
| filenames, | |
| known, | |
| _analyze=analyze, | |
| formats=formats, | |
| extracolumns=extracolumns, | |
| relroot=REPO_ROOT, | |
| **kwargs | |
| ) | |
| def _cli_capi(parser): | |
| parser.add_argument('--levels', action='append', metavar='LEVEL[,...]') | |
| parser.add_argument(f'--public', dest='levels', | |
| action='append_const', const='public') | |
| parser.add_argument(f'--no-public', dest='levels', | |
| action='append_const', const='no-public') | |
| for level in _capi.LEVELS: | |
| parser.add_argument(f'--{level}', dest='levels', | |
| action='append_const', const=level) | |
| def process_levels(args, *, argv=None): | |
| levels = [] | |
| for raw in args.levels or (): | |
| for level in raw.replace(',', ' ').strip().split(): | |
| if level == 'public': | |
| levels.append('stable') | |
| levels.append('cpython') | |
| elif level == 'no-public': | |
| levels.append('private') | |
| levels.append('internal') | |
| elif level in _capi.LEVELS: | |
| levels.append(level) | |
| else: | |
| parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}') | |
| args.levels = set(levels) | |
| parser.add_argument('--kinds', action='append', metavar='KIND[,...]') | |
| for kind in _capi.KINDS: | |
| parser.add_argument(f'--{kind}', dest='kinds', | |
| action='append_const', const=kind) | |
| def process_kinds(args, *, argv=None): | |
| kinds = [] | |
| for raw in args.kinds or (): | |
| for kind in raw.replace(',', ' ').strip().split(): | |
| if kind in _capi.KINDS: | |
| kinds.append(kind) | |
| else: | |
| parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}') | |
| args.kinds = set(kinds) | |
| parser.add_argument('--group-by', dest='groupby', | |
| choices=['level', 'kind']) | |
| parser.add_argument('--format', default='table') | |
| parser.add_argument('--summary', dest='format', | |
| action='store_const', const='summary') | |
| def process_format(args, *, argv=None): | |
| orig = args.format | |
| args.format = _capi.resolve_format(args.format) | |
| if isinstance(args.format, str): | |
| if args.format not in _capi._FORMATS: | |
| parser.error(f'unsupported format {orig!r}') | |
| parser.add_argument('--show-empty', dest='showempty', action='store_true') | |
| parser.add_argument('--no-show-empty', dest='showempty', action='store_false') | |
| parser.set_defaults(showempty=None) | |
| # XXX Add --sort-by, --sort and --no-sort. | |
| parser.add_argument('--ignore', dest='ignored', action='append') | |
| def process_ignored(args, *, argv=None): | |
| ignored = [] | |
| for raw in args.ignored or (): | |
| ignored.extend(raw.replace(',', ' ').strip().split()) | |
| args.ignored = ignored or None | |
| parser.add_argument('filenames', nargs='*', metavar='FILENAME') | |
| process_progress = add_progress_cli(parser) | |
| return [ | |
| process_levels, | |
| process_kinds, | |
| process_format, | |
| process_ignored, | |
| process_progress, | |
| ] | |
| def cmd_capi(filenames=None, *, | |
| levels=None, | |
| kinds=None, | |
| groupby='kind', | |
| format='table', | |
| showempty=None, | |
| ignored=None, | |
| track_progress=None, | |
| verbosity=VERBOSITY, | |
| **kwargs | |
| ): | |
| render = _capi.get_renderer(format) | |
| filenames = _files.iter_header_files(filenames, levels=levels) | |
| #filenames = (file for file, _ in main_for_filenames(filenames)) | |
| if track_progress: | |
| filenames = track_progress(filenames) | |
| items = _capi.iter_capi(filenames) | |
| if levels: | |
| items = (item for item in items if item.level in levels) | |
| if kinds: | |
| items = (item for item in items if item.kind in kinds) | |
| filter = _capi.resolve_filter(ignored) | |
| if filter: | |
| items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg))) | |
| lines = render( | |
| items, | |
| groupby=groupby, | |
| showempty=showempty, | |
| verbose=verbosity > VERBOSITY, | |
| ) | |
| print() | |
| for line in lines: | |
| print(line) | |
| def _cli_builtin_types(parser): | |
| parser.add_argument('--format', dest='fmt', default='table') | |
| # parser.add_argument('--summary', dest='format', | |
| # action='store_const', const='summary') | |
| def process_format(args, *, argv=None): | |
| orig = args.fmt | |
| args.fmt = _builtin_types.resolve_format(args.fmt) | |
| if isinstance(args.fmt, str): | |
| if args.fmt not in _builtin_types._FORMATS: | |
| parser.error(f'unsupported format {orig!r}') | |
| parser.add_argument('--include-modules', dest='showmodules', | |
| action='store_true') | |
| def process_modules(args, *, argv=None): | |
| pass | |
| return [ | |
| process_format, | |
| process_modules, | |
| ] | |
| def cmd_builtin_types(fmt, *, | |
| showmodules=False, | |
| verbosity=VERBOSITY, | |
| ): | |
| render = _builtin_types.get_renderer(fmt) | |
| types = _builtin_types.iter_builtin_types() | |
| match = _builtin_types.resolve_matcher(showmodules) | |
| if match: | |
| types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg))) | |
| lines = render( | |
| types, | |
| # verbose=verbosity > VERBOSITY, | |
| ) | |
| print() | |
| for line in lines: | |
| print(line) | |
| # We do not define any other cmd_*() handlers here, | |
| # favoring those defined elsewhere. | |
| COMMANDS = { | |
| 'check': ( | |
| 'analyze and fail if the CPython source code has any problems', | |
| [_cli_check], | |
| cmd_check, | |
| ), | |
| 'analyze': ( | |
| 'report on the state of the CPython source code', | |
| [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))], | |
| cmd_analyze, | |
| ), | |
| 'parse': ( | |
| 'parse the CPython source files', | |
| [_cli_parse], | |
| cmd_parse, | |
| ), | |
| 'data': ( | |
| 'check/manage local data (e.g. known types, ignored vars, caches)', | |
| [_cli_data], | |
| cmd_data, | |
| ), | |
| 'capi': ( | |
| 'inspect the C-API', | |
| [_cli_capi], | |
| cmd_capi, | |
| ), | |
| 'builtin-types': ( | |
| 'show the builtin types', | |
| [_cli_builtin_types], | |
| cmd_builtin_types, | |
| ), | |
| } | |
| ####################################### | |
| # the script | |
| def parse_args(argv=sys.argv[1:], prog=None, *, subset=None): | |
| import argparse | |
| parser = argparse.ArgumentParser( | |
| prog=prog or get_prog(), | |
| ) | |
| # if subset == 'check' or subset == ['check']: | |
| # if checks is not None: | |
| # commands = dict(COMMANDS) | |
| # commands['check'] = list(commands['check']) | |
| # cli = commands['check'][1][0] | |
| # commands['check'][1][0] = (lambda p: cli(p, checks=checks)) | |
| processors = add_commands_cli( | |
| parser, | |
| commands=COMMANDS, | |
| commonspecs=[ | |
| add_verbosity_cli, | |
| add_traceback_cli, | |
| ], | |
| subset=subset, | |
| ) | |
| args = parser.parse_args(argv) | |
| ns = vars(args) | |
| cmd = ns.pop('cmd') | |
| verbosity, traceback_cm = process_args_by_key( | |
| args, | |
| argv, | |
| processors[cmd], | |
| ['verbosity', 'traceback_cm'], | |
| ) | |
| if cmd != 'parse': | |
| # "verbosity" is sent to the commands, so we put it back. | |
| args.verbosity = verbosity | |
| return cmd, ns, verbosity, traceback_cm | |
| def main(cmd, cmd_kwargs): | |
| try: | |
| run_cmd = COMMANDS[cmd][-1] | |
| except KeyError: | |
| raise ValueError(f'unsupported cmd {cmd!r}') | |
| run_cmd(**cmd_kwargs) | |
| if __name__ == '__main__': | |
| cmd, cmd_kwargs, verbosity, traceback_cm = parse_args() | |
| configure_logger(verbosity) | |
| with traceback_cm: | |
| main(cmd, cmd_kwargs) |