| import io | |
| import logging | |
| import os | |
| import os.path | |
| import re | |
| import sys | |
| from c_common import fsutil | |
| from c_common.logging import VERBOSITY, Printer | |
| from c_common.scriptutil import ( | |
| add_verbosity_cli, | |
| add_traceback_cli, | |
| add_sepval_cli, | |
| add_progress_cli, | |
| add_files_cli, | |
| add_commands_cli, | |
| process_args_by_key, | |
| configure_logger, | |
| get_prog, | |
| filter_filenames, | |
| ) | |
| from c_parser.info import KIND | |
| from .match import filter_forward | |
| from . import ( | |
| analyze as _analyze, | |
| datafiles as _datafiles, | |
| check_all as _check_all, | |
| ) | |
| KINDS = [ | |
| KIND.TYPEDEF, | |
| KIND.STRUCT, | |
| KIND.UNION, | |
| KIND.ENUM, | |
| KIND.FUNCTION, | |
| KIND.VARIABLE, | |
| KIND.STATEMENT, | |
| ] | |
| logger = logging.getLogger(__name__) | |
| ####################################### | |
| # table helpers | |
| TABLE_SECTIONS = { | |
| 'types': ( | |
| ['kind', 'name', 'data', 'file'], | |
| KIND.is_type_decl, | |
| (lambda v: (v.kind.value, v.filename or '', v.name)), | |
| ), | |
| 'typedefs': 'types', | |
| 'structs': 'types', | |
| 'unions': 'types', | |
| 'enums': 'types', | |
| 'functions': ( | |
| ['name', 'data', 'file'], | |
| (lambda kind: kind is KIND.FUNCTION), | |
| (lambda v: (v.filename or '', v.name)), | |
| ), | |
| 'variables': ( | |
| ['name', 'parent', 'data', 'file'], | |
| (lambda kind: kind is KIND.VARIABLE), | |
| (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)), | |
| ), | |
| 'statements': ( | |
| ['file', 'parent', 'data'], | |
| (lambda kind: kind is KIND.STATEMENT), | |
| (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)), | |
| ), | |
| KIND.TYPEDEF: 'typedefs', | |
| KIND.STRUCT: 'structs', | |
| KIND.UNION: 'unions', | |
| KIND.ENUM: 'enums', | |
| KIND.FUNCTION: 'functions', | |
| KIND.VARIABLE: 'variables', | |
| KIND.STATEMENT: 'statements', | |
| } | |
| def _render_table(items, columns, relroot=None): | |
| # XXX improve this | |
| header = '\t'.join(columns) | |
| div = '--------------------' | |
| yield header | |
| yield div | |
| total = 0 | |
| for item in items: | |
| rowdata = item.render_rowdata(columns) | |
| row = [rowdata[c] for c in columns] | |
| if relroot and 'file' in columns: | |
| index = columns.index('file') | |
| row[index] = os.path.relpath(row[index], relroot) | |
| yield '\t'.join(row) | |
| total += 1 | |
| yield div | |
| yield f'total: {total}' | |
| def build_section(name, groupitems, *, relroot=None): | |
| info = TABLE_SECTIONS[name] | |
| while type(info) is not tuple: | |
| if name in KINDS: | |
| name = info | |
| info = TABLE_SECTIONS[info] | |
| columns, match_kind, sortkey = info | |
| items = (v for v in groupitems if match_kind(v.kind)) | |
| items = sorted(items, key=sortkey) | |
| def render(): | |
| yield '' | |
| yield f'{name}:' | |
| yield '' | |
| for line in _render_table(items, columns, relroot): | |
| yield line | |
| return items, render | |
| ####################################### | |
| # the checks | |
| CHECKS = { | |
| #'globals': _check_globals, | |
| } | |
| def add_checks_cli(parser, checks=None, *, add_flags=None): | |
| default = False | |
| if not checks: | |
| checks = list(CHECKS) | |
| default = True | |
| elif isinstance(checks, str): | |
| checks = [checks] | |
| if (add_flags is None and len(checks) > 1) or default: | |
| add_flags = True | |
| process_checks = add_sepval_cli(parser, '--check', 'checks', checks) | |
| if add_flags: | |
| for check in checks: | |
| parser.add_argument(f'--{check}', dest='checks', | |
| action='append_const', const=check) | |
| return [ | |
| process_checks, | |
| ] | |
| def _get_check_handlers(fmt, printer, verbosity=VERBOSITY): | |
| div = None | |
| def handle_after(): | |
| pass | |
| if not fmt: | |
| div = '' | |
| def handle_failure(failure, data): | |
| data = repr(data) | |
| if verbosity >= 3: | |
| logger.info(f'failure: {failure}') | |
| logger.info(f'data: {data}') | |
| else: | |
| logger.warn(f'failure: {failure} (data: {data})') | |
| elif fmt == 'raw': | |
| def handle_failure(failure, data): | |
| print(f'{failure!r} {data!r}') | |
| elif fmt == 'brief': | |
| def handle_failure(failure, data): | |
| parent = data.parent or '' | |
| funcname = parent if isinstance(parent, str) else parent.name | |
| name = f'({funcname}).{data.name}' if funcname else data.name | |
| failure = failure.split('\t')[0] | |
| print(f'{data.filename}:{name} - {failure}') | |
| elif fmt == 'summary': | |
| def handle_failure(failure, data): | |
| print(_fmt_one_summary(data, failure)) | |
| elif fmt == 'full': | |
| div = '' | |
| def handle_failure(failure, data): | |
| name = data.shortkey if data.kind is KIND.VARIABLE else data.name | |
| parent = data.parent or '' | |
| funcname = parent if isinstance(parent, str) else parent.name | |
| known = 'yes' if data.is_known else '*** NO ***' | |
| print(f'{data.kind.value} {name!r} failed ({failure})') | |
| print(f' file: {data.filename}') | |
| print(f' func: {funcname or "-"}') | |
| print(f' name: {data.name}') | |
| print(f' data: ...') | |
| print(f' type unknown: {known}') | |
| else: | |
| if fmt in FORMATS: | |
| raise NotImplementedError(fmt) | |
| raise ValueError(f'unsupported fmt {fmt!r}') | |
| return handle_failure, handle_after, div | |
| ####################################### | |
| # the formats | |
| def fmt_raw(analysis): | |
| for item in analysis: | |
| yield from item.render('raw') | |
| def fmt_brief(analysis): | |
| # XXX Support sorting. | |
| items = sorted(analysis) | |
| for kind in KINDS: | |
| if kind is KIND.STATEMENT: | |
| continue | |
| for item in items: | |
| if item.kind is not kind: | |
| continue | |
| yield from item.render('brief') | |
| yield f' total: {len(items)}' | |
| def fmt_summary(analysis): | |
| # XXX Support sorting and grouping. | |
| items = list(analysis) | |
| total = len(items) | |
| def section(name): | |
| _, render = build_section(name, items) | |
| yield from render() | |
| yield from section('types') | |
| yield from section('functions') | |
| yield from section('variables') | |
| yield from section('statements') | |
| yield '' | |
| # yield f'grand total: {len(supported) + len(unsupported)}' | |
| yield f'grand total: {total}' | |
| def _fmt_one_summary(item, extra=None): | |
| parent = item.parent or '' | |
| funcname = parent if isinstance(parent, str) else parent.name | |
| if extra: | |
| return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}' | |
| else: | |
| return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}' | |
| def fmt_full(analysis): | |
| # XXX Support sorting. | |
| items = sorted(analysis, key=lambda v: v.key) | |
| yield '' | |
| for item in items: | |
| yield from item.render('full') | |
| yield '' | |
| yield f'total: {len(items)}' | |
| FORMATS = { | |
| 'raw': fmt_raw, | |
| 'brief': fmt_brief, | |
| 'summary': fmt_summary, | |
| 'full': fmt_full, | |
| } | |
| def add_output_cli(parser, *, default='summary'): | |
| parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS)) | |
| def process_args(args, *, argv=None): | |
| pass | |
| return process_args | |
| ####################################### | |
| # the commands | |
| def _cli_check(parser, checks=None, **kwargs): | |
| if isinstance(checks, str): | |
| checks = [checks] | |
| if checks is False: | |
| process_checks = None | |
| elif checks is None: | |
| process_checks = add_checks_cli(parser) | |
| elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]): | |
| check = checks[0][1:-1] | |
| def process_checks(args, *, argv=None): | |
| args.checks = [check] | |
| else: | |
| process_checks = add_checks_cli(parser, checks=checks) | |
| process_progress = add_progress_cli(parser) | |
| process_output = add_output_cli(parser, default=None) | |
| process_files = add_files_cli(parser, **kwargs) | |
| return [ | |
| process_checks, | |
| process_progress, | |
| process_output, | |
| process_files, | |
| ] | |
| def cmd_check(filenames, *, | |
| checks=None, | |
| ignored=None, | |
| fmt=None, | |
| failfast=False, | |
| iter_filenames=None, | |
| relroot=fsutil.USE_CWD, | |
| track_progress=None, | |
| verbosity=VERBOSITY, | |
| _analyze=_analyze, | |
| _CHECKS=CHECKS, | |
| **kwargs | |
| ): | |
| if not checks: | |
| checks = _CHECKS | |
| elif isinstance(checks, str): | |
| checks = [checks] | |
| checks = [_CHECKS[c] if isinstance(c, str) else c | |
| for c in checks] | |
| printer = Printer(verbosity) | |
| (handle_failure, handle_after, div | |
| ) = _get_check_handlers(fmt, printer, verbosity) | |
| filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot) | |
| filenames = filter_filenames(filenames, iter_filenames, relroot) | |
| if track_progress: | |
| filenames = track_progress(filenames) | |
| logger.info('analyzing files...') | |
| analyzed = _analyze(filenames, **kwargs) | |
| analyzed.fix_filenames(relroot, normalize=False) | |
| decls = filter_forward(analyzed, markpublic=True) | |
| logger.info('checking analysis results...') | |
| failed = [] | |
| for data, failure in _check_all(decls, checks, failfast=failfast): | |
| if data is None: | |
| printer.info('stopping after one failure') | |
| break | |
| if div is not None and len(failed) > 0: | |
| printer.info(div) | |
| failed.append(data) | |
| handle_failure(failure, data) | |
| handle_after() | |
| printer.info('-------------------------') | |
| logger.info(f'total failures: {len(failed)}') | |
| logger.info('done checking') | |
| if fmt == 'summary': | |
| print('Categorized by storage:') | |
| print() | |
| from .match import group_by_storage | |
| grouped = group_by_storage(failed, ignore_non_match=False) | |
| for group, decls in grouped.items(): | |
| print() | |
| print(group) | |
| for decl in decls: | |
| print(' ', _fmt_one_summary(decl)) | |
| print(f'subtotal: {len(decls)}') | |
| if len(failed) > 0: | |
| sys.exit(len(failed)) | |
| def _cli_analyze(parser, **kwargs): | |
| process_progress = add_progress_cli(parser) | |
| process_output = add_output_cli(parser) | |
| process_files = add_files_cli(parser, **kwargs) | |
| return [ | |
| process_progress, | |
| process_output, | |
| process_files, | |
| ] | |
| # XXX Support filtering by kind. | |
| def cmd_analyze(filenames, *, | |
| fmt=None, | |
| iter_filenames=None, | |
| relroot=fsutil.USE_CWD, | |
| track_progress=None, | |
| verbosity=None, | |
| _analyze=_analyze, | |
| formats=FORMATS, | |
| **kwargs | |
| ): | |
| verbosity = verbosity if verbosity is not None else 3 | |
| try: | |
| do_fmt = formats[fmt] | |
| except KeyError: | |
| raise ValueError(f'unsupported fmt {fmt!r}') | |
| filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot) | |
| filenames = filter_filenames(filenames, iter_filenames, relroot) | |
| if track_progress: | |
| filenames = track_progress(filenames) | |
| logger.info('analyzing files...') | |
| analyzed = _analyze(filenames, **kwargs) | |
| analyzed.fix_filenames(relroot, normalize=False) | |
| decls = filter_forward(analyzed, markpublic=True) | |
| for line in do_fmt(decls): | |
| print(line) | |
| def _cli_data(parser, filenames=None, known=None): | |
| ArgumentParser = type(parser) | |
| common = ArgumentParser(add_help=False) | |
| # These flags will get processed by the top-level parse_args(). | |
| add_verbosity_cli(common) | |
| add_traceback_cli(common) | |
| subs = parser.add_subparsers(dest='datacmd') | |
| sub = subs.add_parser('show', parents=[common]) | |
| if known is None: | |
| sub.add_argument('--known', required=True) | |
| if filenames is None: | |
| sub.add_argument('filenames', metavar='FILE', nargs='+') | |
| sub = subs.add_parser('dump', parents=[common]) | |
| if known is None: | |
| sub.add_argument('--known') | |
| sub.add_argument('--show', action='store_true') | |
| process_progress = add_progress_cli(sub) | |
| sub = subs.add_parser('check', parents=[common]) | |
| if known is None: | |
| sub.add_argument('--known', required=True) | |
| def process_args(args, *, argv): | |
| if args.datacmd == 'dump': | |
| process_progress(args, argv) | |
| return process_args | |
| def cmd_data(datacmd, filenames, known=None, *, | |
| _analyze=_analyze, | |
| formats=FORMATS, | |
| extracolumns=None, | |
| relroot=fsutil.USE_CWD, | |
| track_progress=None, | |
| **kwargs | |
| ): | |
| kwargs.pop('verbosity', None) | |
| usestdout = kwargs.pop('show', None) | |
| if datacmd == 'show': | |
| do_fmt = formats['summary'] | |
| if isinstance(known, str): | |
| known, _ = _datafiles.get_known(known, extracolumns, relroot) | |
| for line in do_fmt(known): | |
| print(line) | |
| elif datacmd == 'dump': | |
| filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot) | |
| if track_progress: | |
| filenames = track_progress(filenames) | |
| analyzed = _analyze(filenames, **kwargs) | |
| analyzed.fix_filenames(relroot, normalize=False) | |
| if known is None or usestdout: | |
| outfile = io.StringIO() | |
| _datafiles.write_known(analyzed, outfile, extracolumns, | |
| relroot=relroot) | |
| print(outfile.getvalue()) | |
| else: | |
| _datafiles.write_known(analyzed, known, extracolumns, | |
| relroot=relroot) | |
| elif datacmd == 'check': | |
| raise NotImplementedError(datacmd) | |
| else: | |
| raise ValueError(f'unsupported data command {datacmd!r}') | |
| COMMANDS = { | |
| 'check': ( | |
| 'analyze and fail if the given C source/header files have any problems', | |
| [_cli_check], | |
| cmd_check, | |
| ), | |
| 'analyze': ( | |
| 'report on the state of the given C source/header files', | |
| [_cli_analyze], | |
| cmd_analyze, | |
| ), | |
| 'data': ( | |
| 'check/manage local data (e.g. known types, ignored vars, caches)', | |
| [_cli_data], | |
| cmd_data, | |
| ), | |
| } | |
| ####################################### | |
| # the script | |
| def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None): | |
| import argparse | |
| parser = argparse.ArgumentParser( | |
| prog=prog or get_prog(), | |
| ) | |
| processors = add_commands_cli( | |
| parser, | |
| commands={k: v[1] for k, v in COMMANDS.items()}, | |
| commonspecs=[ | |
| add_verbosity_cli, | |
| add_traceback_cli, | |
| ], | |
| subset=subset, | |
| ) | |
| args = parser.parse_args(argv) | |
| ns = vars(args) | |
| cmd = ns.pop('cmd') | |
| verbosity, traceback_cm = process_args_by_key( | |
| args, | |
| argv, | |
| processors[cmd], | |
| ['verbosity', 'traceback_cm'], | |
| ) | |
| # "verbosity" is sent to the commands, so we put it back. | |
| args.verbosity = verbosity | |
| return cmd, ns, verbosity, traceback_cm | |
| def main(cmd, cmd_kwargs): | |
| try: | |
| run_cmd = COMMANDS[cmd][0] | |
| except KeyError: | |
| raise ValueError(f'unsupported cmd {cmd!r}') | |
| run_cmd(**cmd_kwargs) | |
| if __name__ == '__main__': | |
| cmd, cmd_kwargs, verbosity, traceback_cm = parse_args() | |
| configure_logger(verbosity) | |
| with traceback_cm: | |
| main(cmd, cmd_kwargs) |