| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Main Python API for analyzing binary size.""" |
| |
| import argparse |
| import collections |
| import dataclasses |
| import functools |
| import logging |
| import os |
| import posixpath |
| import re |
| import shlex |
| import subprocess |
| import time |
| import zipfile |
| |
| import apk |
| import apkanalyzer |
| import archive_util |
| import data_quality |
| import describe |
| import dex_deobfuscate |
| import dir_metadata |
| import file_format |
| import function_signature |
| import json_config_parser |
| import models |
| import native |
| import pakfile |
| import parallel |
| import path_util |
| import readelf |
| import zip_util |
| |
| |
| @dataclasses.dataclass |
| class NativeSpec: |
| # One (or more) of apk_so_path, map_path, elf_path must be non-None. |
| # Path within the .apk of the .so file. Non-None only when apk_spec is. |
| apk_so_path: str = None |
| # Path to linker map file (if present). |
| map_path: str = None |
| # Path to unstripped ELF file (if present). |
| elf_path: str = None |
| # Path to unstripped ELF file before being split into partitions (if present). |
| combined_elf_path: str = None |
| # Whether to create symbols for each string literal. |
| track_string_literals: bool = True |
| # component to use for all symbols. |
| component: str = None |
| # Regular expression that will match generated files. |
| gen_dir_regex: str = None |
| # source_path prefix to use for all symbols. |
| source_path_prefix: str = None |
| |
| @property |
| def algorithm(self): |
| if self.map_path: |
| return 'linker_map' |
| if self.elf_path: |
| return 'dwarf' |
| return 'sections' |
| |
| |
| @dataclasses.dataclass |
| class PakSpec: |
| # One of pak_paths or apk_pak_paths must be non-None. |
| pak_paths: list = None |
| apk_pak_paths: list = None |
| pak_info_path: str = None |
| |
| |
| @dataclasses.dataclass |
| class ApkSpec: |
| # Path the .apk file. Never None. |
| # This is a temp file when .apks is being analyzed. |
| apk_path: str |
| # Path to .minimal.apks (when analyzing bundles). |
| minimal_apks_path: str = None |
| # Proguard mapping path. |
| mapping_path: str = None |
| # Path to the .pathmap.txt file for the apk. Used to deobfuscate res/ files. |
| resources_pathmap_path: str = None |
| # Name of the apk split when .apks is being analyzed. |
| split_name: str = None |
| # Path such as: out/Release/size-info/BaseName |
| size_info_prefix: str = None |
| # Whether to break down classes.dex. |
| analyze_dex: bool = True |
| # Whether to create symbols for each string literal. |
| track_string_literals: bool = True |
| # Dict of apk_path -> source_path, provided by json config. |
| path_defaults: dict = None |
| # Component to use for symbols when not specified by DIR_METADATA, provided by |
| # json config. |
| default_component: str = '' |
| # Paths to not create .other symbols for. |
| ignore_apk_paths: set = dataclasses.field(default_factory=set) |
| |
| |
| @dataclasses.dataclass |
| class ContainerSpec: |
| container_name: str |
| apk_spec: ApkSpec |
| pak_spec: PakSpec |
| native_spec: NativeSpec |
| source_directory: str |
| output_directory: str |
| |
| |
| def _NormalizeNames(raw_symbols): |
| """Ensures that all names are formatted in a useful way. |
| |
| This includes: |
| - Deriving |name| and |template_name| from |full_name|. |
| - Stripping of return types (for functions). |
| - Moving "vtable for" and the like to be suffixes rather than prefixes. |
| """ |
| found_prefixes = set() |
| for symbol in raw_symbols: |
| full_name = symbol.full_name |
| |
| # See comment in _CalculatePadding() about when this can happen. Don't |
| # process names for non-native sections. |
| if symbol.IsPak(): |
| # full_name: "about_ui_resources.grdp: IDR_ABOUT_UI_CREDITS_HTML". |
| space_idx = full_name.rindex(' ') |
| name = full_name[space_idx + 1:] |
| symbol.template_name = name |
| symbol.name = name |
| elif (full_name.startswith('**') or symbol.IsOverhead() |
| or symbol.IsOther()): |
| symbol.template_name = full_name |
| symbol.name = full_name |
| elif symbol.IsStringLiteral(): # Handles native and DEX strings. |
| symbol.full_name = full_name |
| symbol.template_name = full_name |
| symbol.name = full_name |
| elif symbol.IsDex(): |
| symbol.full_name, symbol.template_name, symbol.name = ( |
| function_signature.ParseJava(full_name)) |
| elif symbol.IsNative(): |
| # Remove [clone] suffix, and set flag accordingly. |
| # Search from left-to-right, as multiple [clone]s can exist. |
| # Example name suffixes: |
| # [clone .part.322] # GCC |
| # [clone .isra.322] # GCC |
| # [clone .constprop.1064] # GCC |
| # [clone .11064] # clang |
| # http://unix.stackexchange.com/questions/223013/function-symbol-gets-part-suffix-after-compilation |
| idx = full_name.find(' [clone ') |
| if idx != -1: |
| full_name = full_name[:idx] |
| symbol.flags |= models.FLAG_CLONE |
| |
| # Clones for C symbols. |
| if symbol.section == 't': |
| idx = full_name.rfind('.') |
| if idx != -1 and full_name[idx + 1:].isdigit(): |
| new_name = full_name[:idx] |
| # Generated symbols that end with .123 but are not clones. |
| # Find these via: |
| # size_info.symbols.WhereInSection('t').WhereIsGroup().SortedByCount() |
| if new_name not in ('__tcf_0', 'startup'): |
| full_name = new_name |
| symbol.flags |= models.FLAG_CLONE |
| # Remove .part / .isra / .constprop. |
| idx = full_name.rfind('.', 0, idx) |
| if idx != -1: |
| full_name = full_name[:idx] |
| |
| # E.g.: vtable for FOO |
| idx = full_name.find(' for ', 0, 30) |
| if idx != -1: |
| found_prefixes.add(full_name[:idx + 4]) |
| full_name = '{} [{}]'.format(full_name[idx + 5:], full_name[:idx]) |
| |
| # E.g.: virtual thunk to FOO |
| idx = full_name.find(' to ', 0, 30) |
| if idx != -1: |
| found_prefixes.add(full_name[:idx + 3]) |
| full_name = '{} [{}]'.format(full_name[idx + 4:], full_name[:idx]) |
| |
| # Strip out return type, and split out name, template_name. |
| # Function parsing also applies to non-text symbols. |
| # E.g. Function statics. |
| symbol.full_name, symbol.template_name, symbol.name = ( |
| function_signature.Parse(full_name)) |
| |
| # Remove anonymous namespaces (they just harm clustering). |
| symbol.template_name = symbol.template_name.replace( |
| '(anonymous namespace)::', '') |
| symbol.full_name = symbol.full_name.replace( |
| '(anonymous namespace)::', '') |
| non_anonymous_name = symbol.name.replace('(anonymous namespace)::', '') |
| if symbol.name != non_anonymous_name: |
| symbol.flags |= models.FLAG_ANONYMOUS |
| symbol.name = non_anonymous_name |
| |
| # Allow using "is" to compare names (and should help with RAM). This applies |
| # to all symbols. |
| function_signature.InternSameNames(symbol) |
| |
| logging.debug('Found name prefixes of: %r', found_prefixes) |
| |
| |
| def LoadAndPostProcessSizeInfo(path, file_obj=None): |
| """Returns a SizeInfo for the given |path|.""" |
| logging.debug('Loading results from: %s', path) |
| size_info = file_format.LoadSizeInfo(path, file_obj=file_obj) |
| logging.info('Normalizing symbol names') |
| _NormalizeNames(size_info.raw_symbols) |
| logging.info('Loaded %d symbols', len(size_info.raw_symbols)) |
| return size_info |
| |
| |
| def LoadAndPostProcessDeltaSizeInfo(path, file_obj=None): |
| """Returns a tuple of SizeInfos for the given |path|.""" |
| logging.debug('Loading results from: %s', path) |
| before_size_info, after_size_info, _, _ = (file_format.LoadDeltaSizeInfo( |
| path, file_obj=file_obj)) |
| logging.info('Normalizing symbol names') |
| _NormalizeNames(before_size_info.raw_symbols) |
| _NormalizeNames(after_size_info.raw_symbols) |
| logging.info('Loaded %d + %d symbols', len(before_size_info.raw_symbols), |
| len(after_size_info.raw_symbols)) |
| return before_size_info, after_size_info |
| |
| |
| def CreateBuildConfig(output_directory, source_directory, url=None, title=None): |
| """Creates the dict to use for SizeInfo.build_info.""" |
| logging.debug('Constructing build_config') |
| build_config = {} |
| if output_directory: |
| gn_args = _ParseGnArgs(os.path.join(output_directory, 'args.gn')) |
| build_config[models.BUILD_CONFIG_GN_ARGS] = gn_args |
| build_config[models.BUILD_CONFIG_OUT_DIRECTORY] = os.path.relpath( |
| output_directory, start=source_directory) |
| git_rev = _DetectGitRevision(source_directory) |
| if git_rev: |
| build_config[models.BUILD_CONFIG_GIT_REVISION] = git_rev |
| if url is not None: |
| build_config[models.BUILD_CONFIG_URL] = url |
| if title is not None: |
| build_config[models.BUILD_CONFIG_TITLE] = title |
| |
| return build_config |
| |
| |
| def _CreateMetadata(container_spec, elf_info): |
| logging.debug('Constructing metadata') |
| metadata = {} |
| apk_spec = container_spec.apk_spec |
| native_spec = container_spec.native_spec |
| output_directory = container_spec.output_directory |
| |
| # Ensure all paths are relative to output directory to make them hermetic. |
| if output_directory: |
| shorten_path = lambda path: os.path.relpath(path, output_directory) |
| else: |
| # If output directory is unavailable, just store basenames. |
| shorten_path = os.path.basename |
| |
| if apk_spec: |
| apk_metadata = apk.CreateMetadata(apk_spec=apk_spec, |
| include_file_details=not native_spec, |
| shorten_path=shorten_path) |
| assert not (metadata.keys() & apk_metadata.keys()) |
| metadata.update(apk_metadata) |
| |
| if native_spec: |
| native_metadata = native.CreateMetadata(native_spec=native_spec, |
| elf_info=elf_info, |
| shorten_path=shorten_path) |
| assert not (metadata.keys() & native_metadata.keys()) |
| metadata.update(native_metadata) |
| |
| logging.debug('Constructing metadata (done)') |
| return metadata |
| |
| |
| def _CreatePakSymbols(*, pak_spec, pak_id_map, apk_spec, output_directory): |
| logging.debug('Creating Pak symbols') |
| section_ranges = {} |
| if apk_spec: |
| assert apk_spec.size_info_prefix |
| # Can modify |section_ranges|. |
| raw_symbols = pakfile.CreatePakSymbolsFromApk(section_ranges, |
| apk_spec.apk_path, |
| pak_spec.apk_pak_paths, |
| pak_spec.pak_info_path, |
| pak_id_map) |
| else: |
| # Can modify |section_ranges|. |
| raw_symbols = pakfile.CreatePakSymbolsFromFiles(section_ranges, |
| pak_spec.pak_paths, |
| pak_spec.pak_info_path, |
| output_directory, |
| pak_id_map) |
| return section_ranges, raw_symbols |
| |
| |
| def _CreateContainerSymbols(container_spec, apk_file_manager, |
| apk_analyzer_results, ninja_source_mapper, |
| pak_id_map, component_overrides, |
| dex_deobfuscator_cache): |
| container_name = container_spec.container_name |
| apk_spec = container_spec.apk_spec |
| pak_spec = container_spec.pak_spec |
| native_spec = container_spec.native_spec |
| output_directory = container_spec.output_directory |
| source_directory = container_spec.source_directory |
| |
| logging.info('Starting on container: %s', container_spec) |
| |
| raw_symbols = [] |
| section_sizes = {} |
| metrics_by_file = {} |
| default_component = apk_spec.default_component if apk_spec else '' |
| |
| def add_syms(section_ranges, |
| new_raw_symbols, |
| source_path_prefix=None, |
| component=None, |
| paths_already_normalized=False): |
| new_section_sizes = { |
| k: size |
| for k, (address, size) in section_ranges.items() |
| } |
| if models.SECTION_OTHER in new_section_sizes: |
| section_sizes[models.SECTION_OTHER] = section_sizes.get( |
| models.SECTION_OTHER, 0) + new_section_sizes[models.SECTION_OTHER] |
| del new_section_sizes[models.SECTION_OTHER] |
| |
| assert not (set(section_sizes) & set(new_section_sizes)), ( |
| 'Section collision: {}\n\n {}'.format(section_sizes, new_section_sizes)) |
| section_sizes.update(new_section_sizes) |
| |
| # E.g.: native.CreateSymbols() already calls NormalizePaths(). |
| if not paths_already_normalized: |
| archive_util.NormalizePaths(new_raw_symbols) |
| |
| if source_path_prefix: |
| # Prefix the source_path for all symbols that have a source_path assigned, |
| # and that don't have it set to $APK or $GOOGLE3. |
| for s in new_raw_symbols: |
| if s.source_path and s.source_path[0] != '$': |
| s.source_path = source_path_prefix + s.source_path |
| |
| if component is not None: |
| for s in new_raw_symbols: |
| s.component = component |
| else: |
| dir_metadata.PopulateComponents(new_raw_symbols, |
| source_directory, |
| component_overrides, |
| default_component=default_component) |
| raw_symbols.extend(new_raw_symbols) |
| |
| elf_info = None |
| if native_spec: |
| section_ranges, native_symbols, elf_info, native_metrics_by_file = ( |
| native.CreateSymbols(apk_spec=apk_spec, |
| native_spec=native_spec, |
| output_directory=output_directory, |
| ninja_source_mapper=ninja_source_mapper, |
| pak_id_map=pak_id_map)) |
| add_syms(section_ranges, |
| native_symbols, |
| source_path_prefix=native_spec.source_path_prefix, |
| component=native_spec.component, |
| paths_already_normalized=True) |
| metrics_by_file.update(native_metrics_by_file) |
| elif apk_spec and apk_spec.analyze_dex: |
| logging.info('Analyzing DEX') |
| apk_infolist = apk_file_manager.InfoList(apk_spec.apk_path) |
| dex_total_size = sum(i.file_size for i in apk_infolist |
| if i.filename.endswith('.dex')) |
| if dex_total_size > 0: |
| mapping_path = apk_spec.mapping_path # May be None. |
| class_deobfuscation_map = ( |
| dex_deobfuscator_cache.GetForMappingFile(mapping_path)) |
| section_ranges, dex_symbols, dex_metrics_by_file = ( |
| apkanalyzer.CreateDexSymbols(apk_spec.apk_path, |
| apk_analyzer_results[container_name], |
| dex_total_size, class_deobfuscation_map, |
| apk_spec.size_info_prefix, |
| apk_spec.track_string_literals)) |
| add_syms(section_ranges, dex_symbols) |
| metrics_by_file.update(dex_metrics_by_file) |
| |
| if pak_spec: |
| section_ranges, pak_symbols = _CreatePakSymbols( |
| pak_spec=pak_spec, |
| pak_id_map=pak_id_map, |
| apk_spec=apk_spec, |
| output_directory=output_directory) |
| add_syms(section_ranges, pak_symbols) |
| apk_metadata = {} |
| |
| # This function can get called multiple times for the same APK file, to |
| # process .so files that are treated as containers. The |not native_spec| |
| # condition below skips these cases to prevent redundant symbol creation. |
| if not native_spec and apk_spec: |
| logging.info('Analyzing ARSC') |
| arsc_section_ranges, arsc_symbols, arsc_metrics_by_file = ( |
| apk.CreateArscSymbols(apk_spec)) |
| add_syms(arsc_section_ranges, arsc_symbols) |
| metrics_by_file.update(arsc_metrics_by_file) |
| |
| other_section_ranges, other_symbols, apk_metadata, apk_metrics_by_file = ( |
| apk.CreateApkOtherSymbols(apk_spec)) |
| add_syms(other_section_ranges, other_symbols) |
| metrics_by_file.update(apk_metrics_by_file) |
| |
| metadata = _CreateMetadata(container_spec, elf_info) |
| assert not (metadata.keys() & apk_metadata.keys()) |
| metadata.update(apk_metadata) |
| container = models.Container(name=container_name, |
| metadata=metadata, |
| section_sizes=section_sizes, |
| metrics_by_file=metrics_by_file) |
| for symbol in raw_symbols: |
| symbol.container = container |
| |
| return raw_symbols |
| |
| |
| def _DetectGitRevision(directory): |
| """Runs git rev-parse to get the SHA1 hash of the current revision. |
| |
| Args: |
| directory: Path to directory where rev-parse command will be run. |
| |
| Returns: |
| A string with the SHA1 hash, or None if an error occured. |
| """ |
| try: |
| git_rev = subprocess.check_output( |
| ['git', '-C', directory, 'rev-parse', 'HEAD']).decode('ascii') |
| return git_rev.rstrip() |
| except Exception: |
| logging.warning('Failed to detect git revision for file metadata.') |
| return None |
| |
| |
| def _ParseGnArgs(args_path): |
| """Returns a list of normalized "key=value" strings.""" |
| args = {} |
| with open(args_path) as f: |
| for l in f: |
| # Strips #s even if within string literal. Not a problem in practice. |
| parts = l.split('#')[0].split('=') |
| if len(parts) != 2: |
| continue |
| args[parts[0].strip()] = parts[1].strip() |
| return ["%s=%s" % x for x in sorted(args.items())] |
| |
| |
| def _AddContainerArguments(parser, is_top_args=False): |
| """Add arguments applicable to a single container.""" |
| |
| # Main file argument: Exactly one should be specified (perhaps via -f). |
| # _IdentifyInputFile() should be kept updated. |
| group = parser.add_argument_group(title='Main Input') |
| group = group.add_mutually_exclusive_group(required=True) |
| group.add_argument('-f', |
| metavar='FILE', |
| help='Auto-identify input file type.') |
| group.add_argument('--apk-file', |
| help='.apk file to measure. Other flags can generally be ' |
| 'derived when this is used.') |
| group.add_argument('--minimal-apks-file', |
| help='.minimal.apks file to measure. Other flags can ' |
| 'generally be derived when this is used.') |
| group.add_argument('--elf-file', help='Path to input ELF file.') |
| group.add_argument('--map-file', |
| help='Path to input .map(.gz) file. Defaults to ' |
| '{{elf_file}}.map(.gz)?. If given without ' |
| '--elf-file, no size metadata will be recorded.') |
| group.add_argument('--pak-file', |
| action='append', |
| default=[], |
| dest='pak_files', |
| help='Paths to pak files.') |
| if is_top_args: |
| group.add_argument('--ssargs-file', |
| help='Path to SuperSize multi-container arguments file.') |
| |
| group = parser.add_argument_group(title='What to Analyze') |
| group.add_argument('--java-only', |
| action='store_true', |
| help='Run on only Java symbols') |
| group.add_argument('--native-only', |
| action='store_true', |
| help='Run on only native symbols') |
| group.add_argument('--no-java', |
| action='store_true', |
| help='Do not run on Java symbols') |
| group.add_argument('--no-native', |
| action='store_true', |
| help='Do not run on native symbols') |
| if is_top_args: |
| group.add_argument('--container-filter', |
| help='Regular expression for which containers to create') |
| |
| group = parser.add_argument_group(title='Analysis Options for Native Code') |
| group.add_argument('--no-map-file', |
| dest='ignore_linker_map', |
| action='store_true', |
| help='Use debug information to capture symbol sizes ' |
| 'instead of linker map file.') |
| # Used by tests to override path to APK-discovered files. |
| group.add_argument('--aux-elf-file', help=argparse.SUPPRESS) |
| group.add_argument( |
| '--aux-map-file', |
| help='Path to linker map to use when --elf-file is provided') |
| |
| group = parser.add_argument_group(title='APK options') |
| group.add_argument('--mapping-file', |
| help='Proguard .mapping file for deobfuscation.') |
| group.add_argument('--resources-pathmap-file', |
| help='.pathmap.txt file that contains a maping from ' |
| 'original resource paths to shortened resource paths.') |
| group.add_argument('--abi-filter', |
| dest='abi_filters', |
| action='append', |
| help='For apks with multiple ABIs, break down native ' |
| 'libraries for this ABI. Defaults to 64-bit when both ' |
| '32 and 64 bit are present.') |
| |
| group = parser.add_argument_group(title='Analysis Options for Pak Files') |
| group.add_argument('--pak-info-file', |
| help='This file should contain all ids found in the pak ' |
| 'files that have been passed in. If not specified, ' |
| '${pak_file}.info is assumed.') |
| |
| group = parser.add_argument_group(title='Analysis Options (shared)') |
| group.add_argument('--source-directory', |
| help='Custom path to the root source directory.') |
| group.add_argument('--output-directory', |
| help='Path to the root build directory.') |
| group.add_argument('--symbols-dir', |
| default='lib.unstripped', |
| help='Relative path containing unstripped .so files ' |
| '(for symbols) w.r.t. the output directory.') |
| group.add_argument('--no-string-literals', |
| action='store_true', |
| help=('Do not create symbols for string literals ' |
| '(applies to DEX and Native).')) |
| if is_top_args: |
| group.add_argument('--json-config', help='Path to a supersize.json.') |
| group.add_argument('--no-output-directory', |
| action='store_true', |
| help='Do not auto-detect --output-directory.') |
| group.add_argument('--check-data-quality', |
| action='store_true', |
| help='Perform sanity checks to ensure there is no ' |
| 'missing data.') |
| |
| |
| def AddArguments(parser): |
| parser.add_argument('size_file', help='Path to output .size file.') |
| parser.add_argument('--title', |
| help='Value for the "title" build_config entry.') |
| parser.add_argument('--url', help='Value for the "url" build_config entry.') |
| _AddContainerArguments(parser, is_top_args=True) |
| |
| |
| def _IdentifyInputFile(args, on_config_error): |
| """Identifies main input file type from |args.f|, and updates |args|. |
| |
| Identification is performed on filename alone, i.e., the file need not exist. |
| The result is written to a field in |args|. If the field exists then it |
| simply gets overwritten. |
| |
| If '.' is missing from |args.f| then --elf-file is assumed. |
| |
| Returns: |
| The primary input file. |
| """ |
| if args.f: |
| if args.f.endswith('.minimal.apks'): |
| args.minimal_apks_file = args.f |
| elif args.f.endswith('.apk'): |
| args.apk_file = args.f |
| elif args.f.endswith('.so') or '.' not in os.path.basename(args.f): |
| args.elf_file = args.f |
| elif args.f.endswith('.map') or args.f.endswith('.map.gz'): |
| args.map_file = args.f |
| elif args.f.endswith('.pak'): |
| args.pak_files.append(args.f) |
| elif args.f.endswith('.ssargs'): |
| # Fails if trying to nest them, which should never happen. |
| args.ssargs_file = args.f |
| else: |
| on_config_error('Cannot identify file ' + args.f) |
| args.f = None |
| |
| ret = [ |
| args.apk_file, args.elf_file, args.minimal_apks_file, |
| args.__dict__.get('ssargs_file'), args.map_file |
| ] + (args.pak_files or []) |
| ret = [v for v in ret if v] |
| if not ret: |
| on_config_error( |
| 'Must pass at least one of --apk-file, --minimal-apks-file, ' |
| '--elf-file, --map-file, --pak-file, --ssargs-file') |
| return ret[0] |
| |
| |
| def ParseSsargs(lines): |
| """Parses .ssargs data. |
| |
| An .ssargs file is a text file to specify multiple containers as input to |
| SuperSize-archive. After '#'-based comments, start / end whitespaces, and |
| empty lines are stripped, each line specifies a distinct container. Format: |
| * Positional argument: |name| for the container. |
| * Main input file specified by -f, --apk-file, --elf-file, etc.: |
| * Can be an absolute path. |
| * Can be a relative path. In this case, it's up to the caller to supply the |
| base directory. |
| * -f switch must not specify another .ssargs file. |
| * For supported switches: See _AddContainerArguments(). |
| |
| Args: |
| lines: An iterator containing lines of .ssargs data. |
| Returns: |
| A list of arguments, one for each container. |
| Raises: |
| ValueError: Parse error, including input line number. |
| """ |
| sub_args_list = [] |
| parser = argparse.ArgumentParser(add_help=False) |
| parser.error = lambda msg: (_ for _ in ()).throw(ValueError(msg)) |
| parser.add_argument('name') |
| _AddContainerArguments(parser) |
| try: |
| for lineno, line in enumerate(lines, 1): |
| toks = shlex.split(line, comments=True) |
| if not toks: # Skip if line is empty after stripping comments. |
| continue |
| sub_args_list.append(parser.parse_args(toks)) |
| except ValueError as e: |
| e.args = ('Line %d: %s' % (lineno, e.args[0]), ) |
| raise e |
| return sub_args_list |
| |
| |
| def _MakeNativeSpec(json_config, **kwargs): |
| native_spec = NativeSpec(**kwargs) |
| if native_spec.elf_path or native_spec.map_path: |
| basename = os.path.basename(native_spec.elf_path or native_spec.map_path) |
| native_spec.component = json_config.ComponentForNativeFile(basename) |
| native_spec.gen_dir_regex = json_config.GenDirRegexForNativeFile(basename) |
| native_spec.source_path_prefix = json_config.SourcePathPrefixForNativeFile( |
| basename) |
| |
| if not native_spec.map_path: |
| # TODO(crbug.com/40757867): Implement string literal tracking without map |
| # files. nm emits some string literal symbols, but most are missing. |
| native_spec.track_string_literals = False |
| return native_spec |
| |
| return native_spec |
| |
| |
| def _ElfIsMainPartition(elf_path): |
| section_ranges = readelf.SectionInfoFromElf(elf_path) |
| return models.SECTION_PART_END in section_ranges.keys() |
| |
| |
| def _DeduceMapPath(elf_path): |
| if _ElfIsMainPartition(elf_path): |
| map_path = elf_path.replace('.so', '__combined.so') + '.map' |
| else: |
| map_path = elf_path + '.map' |
| if not os.path.exists(map_path): |
| map_path += '.gz' |
| if not os.path.exists(map_path): |
| map_path = None |
| |
| if map_path: |
| logging.debug('Detected map_path=%s', map_path) |
| return map_path |
| |
| |
| def _CreateNativeSpecs(*, tentative_output_dir, symbols_dir, apk_infolist, |
| elf_path, map_path, abi_filters, auto_abi_filters, |
| track_string_literals, ignore_linker_map, json_config, |
| on_config_error): |
| if ignore_linker_map: |
| map_path = None |
| elif (map_path and not map_path.endswith('.map') |
| and not map_path.endswith('.map.gz')): |
| on_config_error('Expected --map-file to end with .map or .map.gz') |
| elif elf_path and not map_path: |
| map_path = _DeduceMapPath(elf_path) |
| |
| ret = [] |
| # if --elf-path or --map-path (rather than --aux-elf-path, --aux-map-path): |
| if not apk_infolist: |
| if map_path or elf_path: |
| combined_elf_path = None |
| if map_path and '__combined.so' in map_path: |
| combined_elf_path = elf_path[:-3] + '__combined.so' |
| |
| ret.append( |
| _MakeNativeSpec(json_config, |
| apk_so_path=None, |
| map_path=map_path, |
| elf_path=elf_path, |
| combined_elf_path=combined_elf_path, |
| track_string_literals=track_string_literals)) |
| return abi_filters, ret |
| |
| lib_infos = [ |
| f for f in apk_infolist if f.filename.endswith('.so') and f.file_size > 0 |
| ] |
| |
| # Sort so elf_path/map_path applies largest non-filtered library. |
| matches_abi = lambda n: not abi_filters or any(f in n for f in abi_filters) |
| lib_infos.sort(key=lambda x: (not matches_abi(x.filename), -x.file_size)) |
| |
| for lib_info in lib_infos: |
| apk_so_path = lib_info.filename |
| cur_elf_path = None |
| cur_map_path = None |
| if not matches_abi(apk_so_path): |
| logging.debug('Not breaking down %s: secondary ABI', apk_so_path) |
| elif apk_so_path.endswith('_partition.so'): |
| # TODO(agrieve): Support symbol breakdowns for partitions (they exist in |
| # the __combined .map file. Debug information (nm output) is shared |
| # with base partition. |
| logging.debug('Not breaking down %s: partitioned library', apk_so_path) |
| else: |
| if elf_path: |
| # Consume --aux-elf-file for the largest matching binary. |
| cur_elf_path = elf_path |
| elf_path = None |
| elif tentative_output_dir: |
| # TODO(crbug.com/40229168): Remove handling the legacy library prefix |
| # 'crazy.' when there is no longer interest in size comparisons for |
| # these pre-N APKs. |
| cur_elf_path = os.path.join( |
| tentative_output_dir, symbols_dir, |
| posixpath.basename(apk_so_path.replace('crazy.', ''))) |
| if os.path.exists(cur_elf_path): |
| logging.debug('Detected elf_path=%s', cur_elf_path) |
| else: |
| # TODO(agrieve): Not able to find libcrashpad_handler_trampoline.so. |
| logging.debug('Not breaking down %s because file does not exist: %s', |
| apk_so_path, cur_elf_path) |
| cur_elf_path = None |
| |
| if map_path: |
| # Consume --aux-map-file for first non-skipped elf. |
| cur_map_path = map_path |
| map_path = None |
| elif cur_elf_path and not ignore_linker_map: |
| cur_map_path = _DeduceMapPath(cur_elf_path) |
| |
| if auto_abi_filters: |
| abi_filters = [posixpath.basename(posixpath.dirname(apk_so_path))] |
| logging.info('Detected --abi-filter %s', abi_filters[0]) |
| auto_abi_filters = False |
| |
| combined_elf_path = None |
| if cur_map_path and '__combined.so' in cur_map_path: |
| combined_elf_path = cur_elf_path[:-3] + '__combined.so' |
| |
| ret.append( |
| _MakeNativeSpec(json_config, |
| apk_so_path=apk_so_path, |
| map_path=cur_map_path, |
| elf_path=cur_elf_path, |
| combined_elf_path=combined_elf_path, |
| track_string_literals=track_string_literals)) |
| |
| return abi_filters, ret |
| |
| |
| # Cache to prevent excess log messages. |
| @functools.lru_cache |
| def _DeduceMappingPath(mapping_path, apk_prefix): |
| if apk_prefix: |
| if not mapping_path: |
| possible_mapping_path = apk_prefix + '.mapping' |
| if os.path.exists(possible_mapping_path): |
| mapping_path = possible_mapping_path |
| logging.debug('Detected --mapping-file=%s', mapping_path) |
| else: |
| logging.warning('Could not find proguard mapping file at %s', |
| possible_mapping_path) |
| return mapping_path |
| |
| |
| # Cache to prevent excess log messages. |
| @functools.lru_cache |
| def _DeducePathmapPath(resources_pathmap_path, apk_prefix): |
| if apk_prefix: |
| if not resources_pathmap_path: |
| possible_pathmap_path = apk_prefix + '.pathmap.txt' |
| # This could be pointing to a stale pathmap file if path shortening was |
| # previously enabled but is disabled for the current build. However, since |
| # current apk/aab will have unshortened paths, looking those paths up in |
| # the stale pathmap which is keyed by shortened paths would not find any |
| # mapping and thus should not cause any issues. |
| if os.path.exists(possible_pathmap_path): |
| resources_pathmap_path = possible_pathmap_path |
| logging.debug('Detected --resources-pathmap-file=%s', |
| resources_pathmap_path) |
| # Path shortening is optional, so do not warn for missing file. |
| return resources_pathmap_path |
| |
| |
| def _ReadMultipleArgsFromStream(lines, base_dir, err_prefix, on_config_error): |
| try: |
| ret = ParseSsargs(lines) |
| except ValueError as e: |
| on_config_error('%s: %s' % (err_prefix, e.args[0])) |
| for sub_args in ret: |
| for k, v in sub_args.__dict__.items(): |
| # Translate file arguments to be relative to |sub_dir|. |
| if (k.endswith('_file') or k == 'f') and isinstance(v, str): |
| sub_args.__dict__[k] = os.path.join(base_dir, v) |
| return ret |
| |
| |
| def _ReadMultipleArgsFromFile(ssargs_file, on_config_error): |
| with open(ssargs_file, 'r') as fh: |
| lines = list(fh) |
| err_prefix = 'In file ' + ssargs_file |
| # Supply |base_dir| as the directory containing the .ssargs file, to ensure |
| # consistent behavior wherever SuperSize-archive runs. |
| base_dir = os.path.dirname(os.path.abspath(ssargs_file)) |
| return _ReadMultipleArgsFromStream(lines, base_dir, err_prefix, |
| on_config_error) |
| |
| |
| # Both |top_args| and |sub_args| may be modified. |
| def _CreateContainerSpecs(apk_file_manager, |
| top_args, |
| sub_args, |
| json_config, |
| base_container_name, |
| on_config_error, |
| split_name=None): |
| sub_args.source_directory = (sub_args.source_directory |
| or top_args.source_directory) |
| sub_args.output_directory = (sub_args.output_directory |
| or top_args.output_directory) |
| analyze_native = not (sub_args.java_only or sub_args.no_native |
| or top_args.java_only or top_args.no_native) |
| analyze_dex = not (sub_args.native_only or sub_args.no_java |
| or top_args.native_only or top_args.no_java) |
| only_java_or_native = (sub_args.native_only or top_args.native_only |
| or sub_args.java_only or top_args.java_only) |
| analyze_pak = not only_java_or_native and bool(sub_args.output_directory) |
| analyze_res = not only_java_or_native and bool(sub_args.output_directory) |
| |
| if split_name: |
| apk_path = apk_file_manager.SplitPath(sub_args.minimal_apks_file, |
| split_name) |
| base_container_name = f'{base_container_name}/{split_name}.apk' |
| # Make on-demand a part of the name so that: |
| # * It's obvious from the name which DFMs are on-demand. |
| # * Diffs that change an on-demand status show as adds/removes. |
| if _IsOnDemand(apk_path): |
| base_container_name += '?' |
| else: |
| apk_path = sub_args.apk_file |
| |
| apk_prefix = sub_args.minimal_apks_file or sub_args.apk_file |
| if apk_prefix: |
| # Allow either .minimal.apks or just .apks. |
| apk_prefix = apk_prefix.replace('.minimal.apks', '.aab') |
| apk_prefix = apk_prefix.replace('.apks', '.aab') |
| |
| mapping_path = None |
| if analyze_dex: |
| mapping_path = _DeduceMappingPath(sub_args.mapping_file, apk_prefix) |
| resources_pathmap_path = _DeducePathmapPath(sub_args.resources_pathmap_file, |
| apk_prefix) |
| apk_spec = None |
| if apk_prefix: |
| apk_spec = ApkSpec(apk_path=apk_path, |
| minimal_apks_path=sub_args.minimal_apks_file, |
| mapping_path=mapping_path, |
| resources_pathmap_path=resources_pathmap_path, |
| split_name=split_name) |
| size_info_prefix = os.path.join(top_args.output_directory, 'size-info', |
| os.path.basename(apk_prefix)) |
| if analyze_res: |
| apk_spec.size_info_prefix = size_info_prefix |
| res_info_path = apk_spec.size_info_prefix + '.res.info' |
| if not os.path.exists(res_info_path): |
| on_config_error('File not found: ' + res_info_path) |
| apk_spec.analyze_dex = analyze_dex |
| apk_spec.track_string_literals = not (top_args.no_string_literals |
| or sub_args.no_string_literals) |
| apk_spec.default_component = json_config.DefaultComponentForSplit( |
| split_name) |
| apk_spec.path_defaults = json_config.ApkPathDefaults() |
| |
| pak_spec = None |
| apk_pak_paths = None |
| apk_infolist = None |
| if apk_spec: |
| apk_infolist = apk_file_manager.InfoList(apk_path) |
| apk_pak_paths = [ |
| f.filename for f in apk_infolist |
| if archive_util.RemoveAssetSuffix(f.filename).endswith('.pak') |
| ] |
| |
| if analyze_pak and (apk_pak_paths or sub_args.pak_files): |
| if apk_spec: |
| pak_info_path = size_info_prefix + '.pak.info' |
| else: |
| pak_info_path = sub_args.pak_info_file |
| if pak_info_path and not os.path.exists(pak_info_path): |
| on_config_error(f'File not found: {pak_info_file}. ' |
| 'Ensure is_official_build=true, or use --native-only') |
| |
| pak_spec = PakSpec(pak_paths=sub_args.pak_files, |
| pak_info_path=pak_info_path, |
| apk_pak_paths=apk_pak_paths) |
| |
| if analyze_native: |
| # Allow top-level --abi-filter to override values set in .ssargs. |
| abi_filters = top_args.abi_filters or sub_args.abi_filters |
| aux_elf_file = sub_args.aux_elf_file |
| aux_map_file = sub_args.aux_map_file |
| if split_name not in (None, 'base'): |
| aux_elf_file = None |
| aux_map_file = None |
| |
| auto_abi_filters = not abi_filters and split_name == 'base' |
| abi_filters, native_specs = _CreateNativeSpecs( |
| tentative_output_dir=top_args.output_directory, |
| symbols_dir=sub_args.symbols_dir, |
| apk_infolist=apk_infolist, |
| elf_path=sub_args.elf_file or aux_elf_file, |
| map_path=sub_args.map_file or aux_map_file, |
| abi_filters=abi_filters, |
| auto_abi_filters=auto_abi_filters, |
| track_string_literals=not (top_args.no_string_literals |
| or sub_args.no_string_literals), |
| ignore_linker_map=(top_args.ignore_linker_map |
| or sub_args.ignore_linker_map), |
| json_config=json_config, |
| on_config_error=on_config_error) |
| |
| # For app bundles, use a consistent ABI for all splits. |
| if auto_abi_filters: |
| top_args.abi_filters = abi_filters |
| else: |
| native_specs = [] |
| |
| ret = [ |
| ContainerSpec(container_name=base_container_name, |
| apk_spec=apk_spec, |
| pak_spec=pak_spec, |
| native_spec=None, |
| source_directory=sub_args.source_directory, |
| output_directory=sub_args.output_directory) |
| ] |
| if apk_spec is None: |
| # Special case for when pointed at a single ELF, use just one container. |
| assert len(native_specs) <= 1 |
| ret[0].native_spec = native_specs[0] if native_specs else None |
| else: |
| apk_spec.ignore_apk_paths.update(s.apk_so_path for s in native_specs) |
| if pak_spec and pak_spec.apk_pak_paths: |
| apk_spec.ignore_apk_paths.update(pak_spec.apk_pak_paths) |
| if apk_spec.analyze_dex: |
| apk_spec.ignore_apk_paths.update(i.filename for i in apk_infolist |
| if i.filename.endswith('.dex')) |
| apk_spec.ignore_apk_paths.add(apk.RESOURCES_ARSC_FILE) |
| |
| for native_spec in native_specs: |
| so_name = posixpath.basename(native_spec.apk_so_path) |
| abi = posixpath.basename(posixpath.dirname(native_spec.apk_so_path)) |
| container_name = f'{base_container_name}/{so_name} ({abi})' |
| # Use same apk_spec so that all containers for the apk_spec can be found. |
| ret.append( |
| ContainerSpec(container_name=container_name, |
| apk_spec=apk_spec, |
| pak_spec=None, |
| native_spec=native_spec, |
| source_directory=sub_args.source_directory, |
| output_directory=sub_args.output_directory)) |
| return ret |
| |
| |
| def _IsOnDemand(apk_path): |
| # Check if the manifest specifies whether or not to extract native libs. |
| output = subprocess.check_output([ |
| path_util.GetAapt2Path(), 'dump', 'xmltree', '--file', |
| 'AndroidManifest.xml', apk_path |
| ]).decode('ascii') |
| |
| def parse_attr(namespace, name): |
| # A: http://schemas.android.com/apk/res/android:isFeatureSplit(0x...)=true |
| # A: http://schemas.android.com/apk/distribution:onDemand=true |
| m = re.search(f'A: (?:.*?/{namespace}:)?{name}' + r'(?:\(.*?\))?=(\w+)', |
| output) |
| return m and m.group(1) == 'true' |
| |
| is_feature_split = parse_attr('android', 'isFeatureSplit') |
| # Can use <dist:on-demand>, or <module dist:onDemand="true">. |
| on_demand = parse_attr('distribution', 'onDemand') or 'on-demand' in output |
| on_demand = bool(on_demand and is_feature_split) |
| |
| return on_demand |
| |
| |
| def _CreateAllContainerSpecs(apk_file_manager, top_args, json_config, |
| on_config_error): |
| main_file = _IdentifyInputFile(top_args, on_config_error) |
| if top_args.no_output_directory: |
| top_args.output_directory = None |
| else: |
| output_directory_finder = path_util.OutputDirectoryFinder( |
| value=top_args.output_directory, |
| any_path_within_output_directory=main_file) |
| top_args.output_directory = output_directory_finder.Finalized() |
| |
| if not top_args.source_directory: |
| top_args.source_directory = path_util.GetSrcRootFromOutputDirectory( |
| top_args.output_directory) |
| assert top_args.source_directory |
| |
| if top_args.ssargs_file: |
| sub_args_list = _ReadMultipleArgsFromFile(top_args.ssargs_file, |
| on_config_error) |
| else: |
| sub_args_list = [top_args] |
| |
| # Do a quick first pass to ensure inputs have been built. |
| for sub_args in sub_args_list: |
| main_file = _IdentifyInputFile(sub_args, on_config_error) |
| if not os.path.exists(main_file): |
| raise Exception('Input does not exist: ' + main_file) |
| |
| # Each element in |sub_args_list| specifies a container. |
| ret = [] |
| for sub_args in sub_args_list: |
| main_file = _IdentifyInputFile(sub_args, on_config_error) |
| if hasattr(sub_args, 'name'): |
| container_name = sub_args.name |
| else: |
| container_name = os.path.basename(main_file) |
| if set(container_name) & set('<>?'): |
| parser.error('Container name cannot have characters in "<>?"') |
| |
| |
| if sub_args.minimal_apks_file: |
| split_names = apk_file_manager.ExtractSplits(sub_args.minimal_apks_file) |
| for split_name in split_names: |
| ret += _CreateContainerSpecs(apk_file_manager, |
| top_args, |
| sub_args, |
| json_config, |
| container_name, |
| on_config_error, |
| split_name=split_name) |
| else: |
| ret += _CreateContainerSpecs(apk_file_manager, top_args, sub_args, |
| json_config, container_name, on_config_error) |
| all_names = [c.container_name for c in ret] |
| assert len(set(all_names)) == len(all_names), \ |
| 'Found duplicate container names: ' + '\n'.join(sorted(all_names)) |
| |
| return ret |
| |
| |
| def _FilterContainerSpecs(container_specs, container_re=None): |
| ret = [] |
| seen_container_names = set() |
| for container_spec in container_specs: |
| container_name = container_spec.container_name |
| if container_name in seen_container_names: |
| raise ValueError('Duplicate container name: {}'.format(container_name)) |
| seen_container_names.add(container_name) |
| |
| if container_re and not container_re.search(container_name): |
| logging.info('Skipping filtered container %s', container_name) |
| continue |
| ret.append(container_spec) |
| return ret |
| |
| |
| def CreateSizeInfo(container_specs, build_config, json_config, |
| apk_file_manager): |
| def sort_key(container_spec): |
| # Native containers come first to ensure pak_id_map is populated before |
| # any pak_spec is encountered. |
| if container_spec.native_spec: |
| # Do the most complicated container first, since its most likely to fail. |
| if container_spec.native_spec.algorithm == 'linker_map': |
| native_key = 0 |
| elif container_spec.native_spec.algorithm == 'dwarf': |
| native_key = 1 |
| else: |
| native_key = 2 |
| else: |
| native_key = 3 |
| return (native_key, container_spec.container_name) |
| |
| container_specs.sort(key=sort_key) |
| |
| dex_containers = [ |
| c for c in container_specs |
| if not c.native_spec and c.apk_spec and c.apk_spec.analyze_dex |
| ] |
| # Running ApkAnalyzer concurrently saves ~30 seconds for Monochrome.apks. |
| apk_analyzer_results = {} |
| if dex_containers: |
| logging.info('Kicking of ApkAnalyzer for %d .apk files', |
| len(dex_containers)) |
| for container_spec in dex_containers: |
| apk_analyzer_results[container_spec.container_name] = ( |
| apkanalyzer.RunApkAnalyzerAsync(container_spec.apk_spec.apk_path, |
| container_spec.apk_spec.mapping_path)) |
| |
| ninja_containers = [ |
| c for c in container_specs |
| if c.native_spec and c.output_directory and c.native_spec.map_path |
| ] |
| ninja_source_mapper = None |
| if ninja_containers: |
| elf_paths_to_find_inputs_for = [ |
| c.native_spec.combined_elf_path or c.native_spec.elf_path |
| for c in ninja_containers |
| if c.native_spec.combined_elf_path or c.native_spec.elf_path |
| ] |
| ninja_source_mapper = native.ParseNinjaFiles( |
| ninja_containers[0].output_directory, elf_paths_to_find_inputs_for) |
| |
| raw_symbols_list = [] |
| pak_id_map = pakfile.PakIdMap() |
| dex_deobfuscator_cache = dex_deobfuscate.CachedDexDeobfuscators() |
| for container_spec in container_specs: |
| raw_symbols = _CreateContainerSymbols(container_spec, apk_file_manager, |
| apk_analyzer_results, |
| ninja_source_mapper, pak_id_map, |
| json_config.ComponentOverrides(), |
| dex_deobfuscator_cache) |
| assert raw_symbols, f'{container_spec.container_name} had no symbols.' |
| raw_symbols_list.append(raw_symbols) |
| |
| # Normalize names before sorting. |
| logging.info('Normalizing symbol names') |
| for raw_symbols in raw_symbols_list: |
| _NormalizeNames(raw_symbols) |
| |
| # Sorting must happen after normalization. |
| logging.info('Sorting symbols') |
| for raw_symbols in raw_symbols_list: |
| file_format.SortSymbols(raw_symbols) |
| |
| logging.debug('Accumulating symbols') |
| # Containers should always have at least one symbol. |
| container_list = [syms[0].container for syms in raw_symbols_list] |
| all_raw_symbols = [] |
| for raw_symbols in raw_symbols_list: |
| all_raw_symbols += raw_symbols |
| |
| file_format.CalculatePadding(all_raw_symbols) |
| |
| return models.SizeInfo(build_config, container_list, all_raw_symbols) |
| |
| |
| def Run(top_args, on_config_error): |
| path_util.CheckLlvmToolsAvailable() |
| |
| if not top_args.size_file.endswith('.size'): |
| on_config_error('size_file must end with .size') |
| if top_args.check_data_quality: |
| start_time = time.time() |
| |
| container_re = None |
| if top_args.container_filter: |
| try: |
| container_re = re.compile(top_args.container_filter) |
| except Exception as e: |
| on_config_error(f'Bad --container-filter input: {e}') |
| |
| json_config_path = top_args.json_config |
| if not json_config_path: |
| json_config_path = path_util.GetDefaultJsonConfigPath() |
| logging.info('Using --json-config=%s', json_config_path) |
| json_config = json_config_parser.Parse(json_config_path, on_config_error) |
| |
| with zip_util.ApkFileManager() as apk_file_manager: |
| container_specs = _CreateAllContainerSpecs(apk_file_manager, top_args, |
| json_config, on_config_error) |
| container_specs = _FilterContainerSpecs(container_specs, container_re) |
| |
| build_config = CreateBuildConfig(top_args.output_directory, |
| top_args.source_directory, |
| url=top_args.url, |
| title=top_args.title) |
| size_info = CreateSizeInfo(container_specs, build_config, json_config, |
| apk_file_manager) |
| |
| if logging.getLogger().isEnabledFor(logging.DEBUG): |
| for line in data_quality.DescribeSizeInfoCoverage(size_info): |
| logging.debug(line) |
| logging.info('Recorded info for %d symbols', len(size_info.raw_symbols)) |
| for container in size_info.containers: |
| logging.info('Recording metadata: \n %s', |
| '\n '.join(describe.DescribeDict(container.metadata))) |
| |
| logging.info('Saving result to %s', top_args.size_file) |
| file_format.SaveSizeInfo(size_info, top_args.size_file) |
| size_in_mb = os.path.getsize(top_args.size_file) / 1024.0 / 1024.0 |
| logging.info('Done. File size is %.2fMiB.', size_in_mb) |
| |
| if top_args.check_data_quality: |
| logging.info('Checking data quality') |
| data_quality.CheckDataQuality(size_info, not top_args.no_string_literals) |
| duration = (time.time() - start_time) / 60 |
| if duration > 10: |
| raise data_quality.QualityCheckError( |
| 'Command should not take longer than 10 minutes.' |
| ' Took {:.1f} minutes.'.format(duration)) |
