| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """An interactive console for looking analyzing .size files.""" |
| |
| import argparse |
| import bisect |
| import code |
| import collections |
| import contextlib |
| import itertools |
| import logging |
| import os |
| import readline |
| import subprocess |
| import sys |
| import types |
| |
| import archive |
| import canned_queries |
| import data_quality |
| import describe |
| import dex_disassembly |
| import diff |
| import file_format |
| import match_util |
| import models |
| import native_disassembly |
| import path_util |
| import readelf |
| import string_extract |
| |
| |
| # Number of lines before using less for Print(). |
| _THRESHOLD_FOR_PAGER = 50 |
| |
| @contextlib.contextmanager |
| def _LessPipe(): |
| """Output to `less`. Yields a file object to write to.""" |
| try: |
| # pylint: disable=unexpected-keyword-arg |
| proc = subprocess.Popen(['less'], |
| stdin=subprocess.PIPE, |
| stdout=sys.stdout, |
| encoding='utf-8') |
| yield proc.stdin |
| proc.stdin.close() |
| proc.wait() |
| except IOError: |
| pass # Happens when less is quit before all data is written. |
| except KeyboardInterrupt: |
| pass # Assume used to break out of less. |
| |
| |
| def _WriteToStream(lines, has_newlines=False, use_pager=None, to_file=None): |
| if to_file: |
| use_pager = False |
| if use_pager is None and sys.stdout.isatty(): |
| # Does not take into account line-wrapping... Oh well. |
| first_lines = list(itertools.islice(lines, _THRESHOLD_FOR_PAGER)) |
| use_pager = len(first_lines) == _THRESHOLD_FOR_PAGER |
| lines = itertools.chain(first_lines, lines) |
| |
| if has_newlines: |
| func = lambda lines, obj: obj.writelines(lines) |
| else: |
| func = lambda lines, obj: describe.WriteLines(lines, obj.write) |
| |
| if use_pager: |
| with _LessPipe() as stdin: |
| func(lines, stdin) |
| elif to_file: |
| with open(to_file, 'w') as file_obj: |
| func(lines, file_obj) |
| else: |
| func(lines, sys.stdout) |
| |
| |
| @contextlib.contextmanager |
| def _ReadlineSession(): |
| history_file = os.path.join(os.path.expanduser('~'), |
| '.binary_size_query_history') |
| # Without initializing readline, arrow keys don't even work! |
| readline.parse_and_bind('tab: complete') |
| if os.path.exists(history_file): |
| readline.read_history_file(history_file) |
| yield |
| readline.write_history_file(history_file) |
| |
| |
| class _Session: |
| |
| def __init__(self, size_infos, output_directory_finder): |
| self._printed_variables = [] |
| self._variables = { |
| 'Print': self._PrintFunc, |
| 'CheckDataQuality': self._CheckDataQuality, |
| 'Csv': self._CsvFunc, |
| 'Diff': self._DiffFunc, |
| 'SaveSizeInfo': self._SaveSizeInfo, |
| 'SaveDeltaSizeInfo': self._SaveDeltaSizeInfo, |
| 'ReadStringLiterals': self._ReadStringLiterals, |
| 'ReplaceWithRelocations': self._ReplaceWithRelocations, |
| 'Disassemble': self._DisassembleFunc, |
| 'ExpandRegex': match_util.ExpandRegexIdentifierPlaceholder, |
| 'SizeStats': self._SizeStats, |
| 'ShowExamples': self._ShowExamplesFunc, |
| 'canned_queries': canned_queries.CannedQueries(size_infos), |
| 'printed': self._printed_variables, |
| 'models': models, |
| } |
| self._output_directory_finder = output_directory_finder |
| self._size_infos = size_infos |
| self._dex_disassembly_cache_by_size_info = collections.defaultdict( |
| dex_disassembly.CreateCache) |
| |
| if len(size_infos) == 1: |
| self._variables['size_info'] = size_infos[0] |
| else: |
| for i, size_info in enumerate(size_infos): |
| self._variables['size_info%d' % (i + 1)] = size_info |
| |
| def _ReadStringLiterals(self, thing=None, all_rodata=False, elf_path=None): |
| """Returns a list of (symbol, string value) for all string literal symbols. |
| |
| E.g.: |
| # Print sorted list of all string literals: |
| Print(sorted(x[1] for x in ReadStringLiterals())) |
| Args: |
| thing: Can be a Symbol, iterable of symbols, or SizeInfo. |
| Defaults to the current SizeInfo. |
| all_rodata: Assume every symbol within .rodata that ends in a \0 is a |
| string literal. |
| elf_path: Path to the executable containing the symbol. Required only |
| when auto-detection fails. |
| """ |
| if thing is None: |
| thing = self._size_infos[-1] |
| if isinstance(thing, models.SizeInfo): |
| thing = thing.raw_symbols.IterUniqueSymbols() |
| elif isinstance(thing, models.BaseSymbol): |
| thing = thing.IterLeafSymbols() |
| |
| thing, thing_clone = itertools.tee(thing) |
| first_sym = next(thing_clone, None) |
| if not first_sym: |
| return [] |
| size_info = self._SizeInfoForSymbol(first_sym) |
| container = first_sym.container |
| elf_path = self._ElfPathForSymbol(size_info, container, elf_path) |
| |
| return string_extract.ReadStringLiterals(thing, |
| elf_path, |
| all_rodata=all_rodata) |
| |
| def _DiffFunc(self, before=None, after=None, sort=True): |
| """Diffs two SizeInfo objects. Returns a DeltaSizeInfo. |
| |
| Args: |
| before: Defaults to size_infos[0]. |
| after: Defaults to size_infos[1]. |
| sort: When True (default), calls SymbolGroup.Sorted() after diffing. |
| """ |
| before = before if before is not None else self._size_infos[0] |
| after = after if after is not None else self._size_infos[1] |
| return diff.Diff(before, after, sort=sort) |
| |
| def _PrintUploadCommand(self, file_to_upload, is_internal=False): |
| oneoffs_dir = 'oneoffs' |
| visibility = '-a public-read ' |
| if is_internal: |
| oneoffs_dir = 'private-oneoffs' |
| visibility = '' |
| |
| shortname = os.path.basename(os.path.normpath(file_to_upload)) |
| msg = ( |
| 'Saved locally to {local}. To share, run:\n' |
| '> gsutil.py cp {visibility}{local} gs://chrome-supersize/' |
| '{oneoffs_dir}\n' |
| ' Then view it at https://chrome-supersize.firebaseapp.com/viewer.html' |
| '?load_url=https://storage.googleapis.com/chrome-supersize/' |
| '{oneoffs_dir}/{shortname}') |
| print(msg.format(local=file_to_upload, |
| shortname=shortname, |
| oneoffs_dir=oneoffs_dir, |
| visibility=visibility)) |
| |
| def _SaveSizeInfo(self, size_info=None, to_file=None): |
| """Writes a .size file. |
| |
| Args: |
| size_info: The size_info to filter. Defaults to size_infos[0]. |
| to_file: Defaults to default.size |
| """ |
| size_info = size_info or self._size_infos[0] |
| to_file = to_file or 'default.size' |
| assert to_file.endswith('.size'), 'to_file should end with .size' |
| |
| file_format.SaveSizeInfo(size_info, to_file) |
| |
| is_internal = len(size_info.symbols.WherePathMatches('^clank')) > 0 |
| self._PrintUploadCommand(to_file, is_internal) |
| |
| def _SaveDeltaSizeInfo(self, size_info, to_file=None): |
| """Writes a .sizediff file. |
| |
| Args: |
| delta_size_info: The delta_size_info to filter. |
| to_file: Defaults to default.sizediff |
| """ |
| to_file = to_file or 'default.sizediff' |
| assert to_file.endswith('.sizediff'), 'to_file should end with .sizediff' |
| |
| file_format.SaveDeltaSizeInfo(size_info, to_file) |
| is_internal = len(size_info.symbols.WherePathMatches('^clank')) > 0 |
| self._PrintUploadCommand(to_file, is_internal) |
| |
| def _SizeStats(self, size_info=None): |
| """Prints some statistics for the given size info. |
| |
| Args: |
| size_info: Defaults to size_infos[0]. |
| """ |
| size_info = size_info or self._size_infos[0] |
| describe.WriteLines(data_quality.DescribeSizeInfoCoverage(size_info), |
| sys.stdout.write) |
| |
| def _CheckDataQuality(self, size_info=None, track_string_literals=True): |
| """Performs checks that run as part of --check-data-quality.""" |
| size_info = size_info or self._size_infos[0] |
| data_quality.CheckDataQuality(size_info, track_string_literals) |
| |
| def _PrintFunc(self, obj=None, verbose=False, summarize=True, recursive=False, |
| use_pager=None, to_file=None): |
| """Prints out the given Symbol / SymbolGroup / SizeInfo. |
| |
| For convenience, |obj| will be appended to the global "printed" list. |
| |
| Args: |
| obj: The object to be printed. |
| verbose: Show more detailed output. |
| summarize: If False, show symbols only (no headers / summaries). |
| recursive: Print children of nested SymbolGroups. |
| use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol. |
| default is to automatically pipe when output is long. |
| to_file: Rather than print to stdio, write to the given file. |
| """ |
| if obj is not None: |
| self._printed_variables.append(obj) |
| lines = describe.GenerateLines( |
| obj, verbose=verbose, recursive=recursive, summarize=summarize, |
| format_name='text') |
| _WriteToStream(lines, use_pager=use_pager, to_file=to_file) |
| |
| def _CsvFunc(self, obj=None, verbose=False, use_pager=None, to_file=None): |
| """Prints out the given Symbol / SymbolGroup / SizeInfo in CSV format. |
| |
| For convenience, |obj| will be appended to the global "printed" list. |
| |
| Args: |
| obj: The object to be printed as CSV. |
| use_pager: Pipe output through `less`. Ignored when |obj| is a Symbol. |
| default is to automatically pipe when output is long. |
| to_file: Rather than print to stdio, write to the given file. |
| """ |
| if obj is not None: |
| self._printed_variables.append(obj) |
| lines = describe.GenerateLines(obj, verbose=verbose, recursive=False, |
| format_name='csv') |
| _WriteToStream(lines, use_pager=use_pager, to_file=to_file) |
| |
| def _ElfPathForSymbol(self, size_info, container, elf_path=None): |
| def build_id_matches(elf_path): |
| found_build_id = readelf.BuildIdFromElf(elf_path) |
| expected_build_id = container.metadata.get(models.METADATA_ELF_BUILD_ID) |
| return found_build_id == expected_build_id |
| |
| filename = container.metadata[models.METADATA_ELF_FILENAME] |
| paths_to_try = [] |
| if elf_path: |
| paths_to_try.append(elf_path) |
| else: |
| auto_output_directory_finders = [ |
| path_util.OutputDirectoryFinder( |
| any_path_within_output_directory=s.size_path) |
| for s in self._size_infos] + [self._output_directory_finder] |
| for output_directory_finder in auto_output_directory_finders: |
| output_dir = output_directory_finder.Tentative() |
| if output_dir: |
| # Local build: File is located in output directory. |
| paths_to_try.append( |
| os.path.normpath(os.path.join(output_dir, filename))) |
| # Downloaded build: File is located beside .size file. |
| paths_to_try.append(os.path.normpath(os.path.join( |
| os.path.dirname(size_info.size_path), os.path.basename(filename)))) |
| |
| paths_to_try = [p for p in paths_to_try if os.path.exists(p)] |
| |
| for i, path in enumerate(paths_to_try): |
| if build_id_matches(path): |
| return path |
| |
| # Show an error only once all paths are tried. |
| if i + 1 == len(paths_to_try): |
| assert False, 'Build ID does not match for %s' % elf_path |
| |
| assert False, ( |
| 'Could not locate ELF file. If binary was built locally, ensure ' |
| '--output-directory is set. If output directory is unavailable, ' |
| 'ensure {} is located beside {}, or pass its path explicitly using ' |
| 'elf_path=').format(os.path.basename(filename), size_info.size_path) |
| return None |
| |
| def _SizeInfoForSymbol(self, symbol): |
| for size_info in self._size_infos: |
| if symbol in size_info.raw_symbols: |
| return size_info |
| assert False, 'Symbol does not belong to a size_info.' |
| return None |
| |
| def _DisassembleNative(self, |
| symbol, |
| output_directory, |
| elf_path=None, |
| use_pager=None, |
| to_file=None): |
| if output_directory is None: |
| # If we do not know/guess the output directory, run from any directory 2 |
| # levels below src since it is better than a random cwd (because usually |
| # source file paths are relative to an output directory two levels below |
| # src and start with ../../). |
| output_directory = path_util.FromToolsSrcRoot('tools', 'binary_size') |
| |
| size_info = self._SizeInfoForSymbol(symbol) |
| elf_path = self._ElfPathForSymbol(size_info, symbol.container, elf_path) |
| |
| with native_disassembly.Disassemble(symbol, |
| output_directory, |
| elf_path, |
| max_bytes=None) as lines: |
| _WriteToStream(lines, |
| has_newlines=True, |
| use_pager=use_pager, |
| to_file=to_file) |
| |
| def _DisassembleDex(self, |
| symbol, |
| output_directory, |
| use_pager=None, |
| to_file=None): |
| |
| def path_resolver(path): |
| return os.path.join(output_directory, path) |
| |
| size_info = self._SizeInfoForSymbol(symbol) |
| cache = self._dex_disassembly_cache_by_size_info[size_info] |
| lines = dex_disassembly.Disassemble(symbol, path_resolver, cache) |
| _WriteToStream(iter(lines), |
| has_newlines=True, |
| use_pager=use_pager, |
| to_file=to_file) |
| |
| def _DisassembleFunc(self, |
| symbol, |
| elf_or_apk_path=None, |
| use_pager=None, |
| to_file=None): |
| """Shows objdump disassembly for the given symbol. |
| |
| Args: |
| symbol: Must be a .text symbol and not a SymbolGroup. |
| elf_or_apk_path: Path to the executable containing the symbol. Required |
| only when auto-detection fails. |
| """ |
| assert not symbol.IsGroup() |
| assert not symbol.IsDelta(), ('Cannot disasseble a Diff\'ed symbol. Try ' |
| 'passing .before_symbol or .after_symbol.') |
| |
| output_directory_finder = self._output_directory_finder |
| if not output_directory_finder.Tentative(): |
| output_directory_finder = path_util.OutputDirectoryFinder( |
| any_path_within_output_directory=elf_or_apk_path) |
| output_directory = output_directory_finder.Tentative() |
| |
| if symbol.section_name == models.SECTION_TEXT: |
| assert symbol.address, 'Symbol is missing address' |
| self._DisassembleNative(symbol, |
| output_directory, |
| elf_path=elf_or_apk_path, |
| use_pager=use_pager, |
| to_file=to_file) |
| elif symbol.section_name == models.SECTION_DEX_METHOD: |
| self._DisassembleDex(symbol, |
| output_directory, |
| use_pager=use_pager, |
| to_file=to_file) |
| else: |
| raise Exception('Symbol type is not supported: ' + symbol.section_name) |
| |
| |
| def _ReplaceWithRelocations(self, size_info=None): |
| """Replace all symbol sizes with counts of native relocations. |
| |
| Removes all symbols that do not contain relocations. |
| |
| Args: |
| size_info: The size_info to filter. Defaults to size_infos[0]. |
| |
| Returns: |
| A new SizeInfo. |
| """ |
| size_info = size_info or self._size_infos[0] |
| |
| new_syms = [] |
| new_containers = [] |
| |
| for container, group in itertools.groupby( |
| size_info.raw_symbols, lambda s: s.container): |
| if models.METADATA_ELF_FILENAME not in container.metadata: |
| continue |
| |
| raw_symbols = [s for s in group if s.IsNative()] |
| if not raw_symbols: |
| continue |
| |
| new_containers.append(container) |
| |
| elf_path = self._ElfPathForSymbol(size_info, container) |
| relro_addresses = readelf.CollectRelocationAddresses(elf_path) |
| |
| # More likely for there to be a bug in supersize than an ELF to have any |
| # relative relocations. |
| assert relro_addresses |
| |
| # Last symbol address is the end of the last symbol, so we don't |
| # misattribute all relros after the last symbol to that symbol. |
| symbol_addresses = [s.address for s in raw_symbols] |
| symbol_addresses.append(raw_symbols[-1].end_address) |
| |
| for symbol in raw_symbols: |
| symbol.address = 0 |
| symbol.size = 0 |
| symbol.padding = 0 |
| |
| logging.info('Adding %d relocations', len(relro_addresses)) |
| for addr in relro_addresses: |
| # Attribute relros to largest symbol start address that precede them. |
| idx = bisect.bisect_right(symbol_addresses, addr) - 1 |
| if 0 <= idx < len(raw_symbols): |
| symbol = raw_symbols[idx] |
| for alias in symbol.aliases or [symbol]: |
| alias.size += 1 |
| |
| new_syms.extend(s for s in raw_symbols if s.size) |
| |
| return models.SizeInfo(size_info.build_config, |
| new_containers, |
| models.SymbolGroup(new_syms), |
| size_path=size_info.size_path) |
| |
| def _ShowExamplesFunc(self): |
| print(self._CreateBanner()) |
| print('\n'.join([ |
| '# Show pydoc for main types:', |
| 'import models', |
| 'help(models)', |
| '', |
| '# Show all attributes of all symbols & per-section totals:', |
| 'Print(size_info, verbose=True)', |
| '', |
| '# Dump section info and all symbols in CSV format:', |
| 'Csv(size_info)', |
| '', |
| '# Print sorted list of all string literals:', |
| 'Print(sorted(x[1] for x in ReadStringLiterals()))', |
| '', |
| '# Show two levels of .text, grouped by first two subdirectories', |
| 'text_syms = size_info.symbols.WhereInSection("t")', |
| 'by_path = text_syms.GroupedByPath(depth=2)', |
| 'Print(by_path.WherePssBiggerThan(1024))', |
| '', |
| '# Show all generated symbols, then show only non-vtable ones', |
| 'Print(size_info.symbols.WhereGeneratedByToolchain())', |
| 'Print(printed[-1].WhereNameMatches(r"vtable").Inverted().Sorted())', |
| '', |
| '# Show all symbols that have "print" in their name or path, except', |
| '# those within components/.', |
| '# Note: Could have also used Inverted(), as above.', |
| '# Note: Use "help(ExpandRegex)" for more about what {{_print_}} does.', |
| 'print_syms = size_info.symbols.WhereMatches(r"{{_print_}}")', |
| 'Print(print_syms - print_syms.WherePathMatches(r"^components/"))', |
| '', |
| '# Diff two .size files and save result to a file:', |
| 'Print(Diff(size_info1, size_info2), to_file="output.txt")', |
| '', |
| '# Save a .size containing only the filtered symbols', |
| 'filtered_symbols = size_info.raw_symbols.Filter(lambda l: l.IsPak())', |
| 'size_info.MakeSparse(filtered_symbols)', |
| 'SaveSizeInfo(size_info, to_file="oneoff_paks.size")', |
| '', |
| '# View per-component breakdowns, then drill into the last entry.', |
| 'c = canned_queries.CategorizeByChromeComponent()', |
| 'Print(c)', |
| 'Print(c[-1].GroupedByPath(depth=2).Sorted())', |
| '', |
| '# For even more inspiration, look at canned_queries.py', |
| '# (and feel free to add your own!).', |
| ])) |
| |
| def _CreateBanner(self): |
| def keys(cls, super_keys=None): |
| ret = sorted(m for m in dir(cls) if m[0] != '_') |
| if super_keys: |
| ret = sorted(m for m in ret if m not in super_keys) |
| return ret |
| |
| symbol_info_keys = keys(models.SizeInfo) |
| symbol_keys = keys(models.Symbol) |
| symbol_group_keys = keys(models.SymbolGroup, symbol_keys) |
| delta_size_info_keys = keys(models.DeltaSizeInfo) |
| delta_symbol_keys = keys(models.DeltaSymbol, symbol_keys) |
| delta_symbol_group_keys = keys(models.DeltaSymbolGroup, |
| symbol_keys + symbol_group_keys) |
| canned_queries_keys = keys(canned_queries.CannedQueries) |
| |
| functions = sorted(k for k in self._variables if k[0].isupper()) |
| lines = [ |
| '*' * 80, |
| 'Entering interactive Python shell. Quick reference:', |
| '', |
| 'SizeInfo: %s' % ', '.join(symbol_info_keys), |
| 'Symbol: %s' % ', '.join(symbol_keys), |
| '', |
| 'SymbolGroup (extends Symbol): %s' % ', '.join(symbol_group_keys), |
| '', |
| 'DeltaSizeInfo: %s' % ', '.join(delta_size_info_keys), |
| 'DeltaSymbol (extends Symbol): %s' % ', '.join(delta_symbol_keys), |
| 'DeltaSymbolGroup (extends SymbolGroup): %s' % ', '.join( |
| delta_symbol_group_keys), |
| '', |
| 'canned_queries: %s' % ', '.join(canned_queries_keys), |
| '', |
| 'Functions: %s' % ', '.join('%s()' % f for f in functions), |
| 'Variables:', |
| ' printed: List of objects passed to Print().', |
| ] |
| for key, value in self._variables.items(): |
| if isinstance(value, types.ModuleType): |
| continue |
| if key.startswith('size_info'): |
| # pylint: disable=no-member |
| lines.append(f' {key}: Loaded from {value.size_path}') |
| # pylint: enable=no-member |
| lines.append('*' * 80) |
| return '\n'.join(lines) |
| |
| def Eval(self, query): |
| exec (query, self._variables) |
| |
| def GoInteractive(self): |
| with _ReadlineSession(): |
| code.InteractiveConsole(self._variables).interact(self._CreateBanner()) |
| |
| |
| def AddArguments(parser): |
| parser.add_argument( |
| 'inputs', nargs='+', |
| help='Input .size files to load. For a single file, it will be mapped to ' |
| 'the variable "size_info". For multiple inputs, the names will be ' |
| 'size_info1, size_info2, etc.') |
| parser.add_argument('--query', |
| help='Execute the given snippet. ' |
| 'Example: Print(size_info)') |
| parser.add_argument('--output-directory', |
| help='Path to the root build directory. Used only for ' |
| 'Disassemble().') |
| |
| |
| def Run(args, on_config_error): |
| # Up-front check for faster error-checking. |
| for path in args.inputs: |
| if not path.endswith('.size') and not path.endswith('.sizediff'): |
| on_config_error('All inputs must end with ".size" or ".sizediff"') |
| |
| size_infos = [] |
| for path in args.inputs: |
| logging.warning('Loading %s', path) |
| if path.endswith('.sizediff'): |
| size_infos.extend(archive.LoadAndPostProcessDeltaSizeInfo(path)) |
| else: |
| size_infos.append(archive.LoadAndPostProcessSizeInfo(path)) |
| output_directory_finder = path_util.OutputDirectoryFinder( |
| value=args.output_directory, |
| any_path_within_output_directory=args.inputs[0]) |
| session = _Session(size_infos, output_directory_finder) |
| |
| if args.query: |
| logging.info('Running query from command-line.') |
| session.Eval(args.query) |
| else: |
| logging.info('Entering interactive console.') |
| session.GoInteractive() |
| |
| # Exit without running GC, which can save multiple seconds due the large |
| # number of objects created. It meants atexit and __del__ calls are not |
| # made, but this shouldn't matter for console. |
| sys.stdout.flush() |
| sys.stderr.flush() |
| os._exit(0) |
