| # Copyright 2022 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Functions for creating native code symbols from ELF files.""" |
| |
| import calendar |
| import collections |
| import dataclasses |
| import datetime |
| import itertools |
| import logging |
| import os |
| import posixpath |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| |
| import ar |
| import archive_util |
| import demangle |
| import dwarfdump |
| import linker_map_parser |
| import models |
| import ninja_parser |
| import nm |
| import obj_analyzer |
| import parallel |
| import path_util |
| import readelf |
| import string_extract |
| import zip_util |
| |
| # When ensuring matching section sizes between .elf and .map files, these |
| # sections should be ignored. When lld creates a combined library with |
| # partitions, some sections (like .text) exist in each partition, but the ones |
| # below are common. At library splitting time, llvm-objcopy pulls what's needed |
| # from these sections into the new libraries. Hence, the ELF sections will end |
| # up smaller than the combined .map file sections. |
| _SECTION_SIZE_BLOCKLIST = ['.symtab', '.shstrtab', '.strtab'] |
| |
| # A limit on the number of symbols an address can have, before these symbols |
| # are compacted into shared symbols. Increasing this value causes more data |
| # to be stored .size files, but is also more expensive. |
| # Effect as of Oct 2017, with min_pss = max: |
| # 1: shared .text syms = 1772874 bytes, file size = 9.43MiB (645476 syms). |
| # 2: shared .text syms = 1065654 bytes, file size = 9.58MiB (669952 syms). |
| # 6: shared .text syms = 464058 bytes, file size = 10.11MiB (782693 syms). |
| # 10: shared .text syms = 365648 bytes, file size = 10.24MiB (813758 syms). |
| # 20: shared .text syms = 86202 bytes, file size = 10.38MiB (854548 syms). |
| # 40: shared .text syms = 48424 bytes, file size = 10.50MiB (890396 syms). |
| # 50: shared .text syms = 41860 bytes, file size = 10.54MiB (902304 syms). |
| # max: shared .text syms = 0 bytes, file size = 11.10MiB (1235449 syms). |
| _MAX_SAME_NAME_ALIAS_COUNT = 40 # 50kb is basically negligible. |
| |
| |
| # Holds computation state that is live only when an output directory exists. |
| @dataclasses.dataclass |
| class _OutputDirectoryContext: |
| elf_object_paths: list # Non-None only when elf_path is. |
| known_inputs: list # Non-None only when elf_path is. |
| output_directory: str |
| thin_archives: list |
| |
| |
| @dataclasses.dataclass |
| class ElfInfo: |
| architecture: str # Results of ArchFromElf(). |
| build_id: str # Result of BuildIdFromElf(). |
| section_ranges: dict # Results of SectionInfoFromElf(). |
| size: int # Result of os.path.getsize(). |
| |
| def OverheadSize(self): |
| section_sizes_total_without_bss = sum( |
| size for k, (_, size) in self.section_ranges.items() |
| if k not in models.BSS_SECTIONS) |
| ret = self.size - section_sizes_total_without_bss |
| assert ret >= 0, 'Negative ELF overhead {}'.format(ret) |
| return ret |
| |
| |
| def _CreateElfInfo(elf_path): |
| return ElfInfo(architecture=readelf.ArchFromElf(elf_path), |
| build_id=readelf.BuildIdFromElf(elf_path), |
| section_ranges=readelf.SectionInfoFromElf(elf_path), |
| size=os.path.getsize(elf_path)) |
| |
| |
| def _AddSourcePathsUsingObjectPaths(ninja_source_mapper, raw_symbols): |
| logging.info('Looking up source paths from ninja files') |
| for symbol in raw_symbols: |
| # Native symbols and pak symbols use object paths. |
| object_path = symbol.object_path |
| if not object_path: |
| continue |
| |
| # We don't have source info for prebuilt .a files. |
| if not os.path.isabs(object_path) and not object_path.startswith('..'): |
| symbol.source_path = ninja_source_mapper.FindSourceForPath(object_path) |
| assert ninja_source_mapper.unmatched_paths_count == 0, ( |
| 'One or more source file paths could not be found. Likely caused by ' |
| '.ninja files being generated at a different time than the .map file.') |
| |
| |
| def _AddSourcePathsUsingAddress(dwarf_source_mapper, raw_symbols): |
| logging.debug('Looking up source paths from dwarfdump') |
| query_count = 0 |
| match_count = 0 |
| for symbol in raw_symbols: |
| if symbol.section_name != models.SECTION_TEXT: |
| continue |
| query_count += 1 |
| source_path = dwarf_source_mapper.FindSourceForTextAddress(symbol.address) |
| if source_path: |
| match_count += 1 |
| symbol.source_path = source_path |
| logging.info('dwarfdump found paths for %d of %d .text symbols.', match_count, |
| query_count) |
| # Majority of unmatched queries are for assembly source files (ex libav1d) |
| # and v8 builtins. |
| if query_count > 0: |
| unmatched_ratio = (query_count - match_count) / query_count |
| assert unmatched_ratio < 0.2, ( |
| 'Percentage of failing |dwarf_source_mapper| queries ' + |
| '({}%) >= 20% '.format(unmatched_ratio * 100) + |
| 'FindSourceForTextAddress() likely has a bug.') |
| |
| |
| def _ConnectNmAliases(raw_symbols): |
| """Ensures |aliases| is set correctly for all symbols.""" |
| prev_sym = raw_symbols[0] |
| for sym in raw_symbols[1:]: |
| # Don't merge bss symbols. |
| if sym.address > 0 and prev_sym.address == sym.address: |
| # Don't merge padding-only symbols (** symbol gaps). |
| if prev_sym.size > 0: |
| # Don't merge if already merged. |
| if prev_sym.aliases is None or prev_sym.aliases is not sym.aliases: |
| if prev_sym.aliases: |
| prev_sym.aliases.append(sym) |
| else: |
| prev_sym.aliases = [prev_sym, sym] |
| sym.aliases = prev_sym.aliases |
| prev_sym = sym |
| |
| |
| def _AssignNmAliasPathsAndCreatePathAliases(raw_symbols, object_paths_by_name): |
| num_found_paths = 0 |
| num_unknown_names = 0 |
| num_path_mismatches = 0 |
| num_aliases_created = 0 |
| ret = [] |
| for symbol in raw_symbols: |
| ret.append(symbol) |
| full_name = symbol.full_name |
| # '__typeid_' symbols appear in linker .map only, and not nm output. |
| if full_name.startswith('__typeid_'): |
| if object_paths_by_name.get(full_name): |
| logging.warning('Found unexpected __typeid_ symbol in nm output: %s', |
| full_name) |
| continue |
| |
| # Don't skip if symbol.IsBss(). This is needed for LLD-LTO to work, since |
| # .bss object_path data are unavailable for linker_map_parser, and need to |
| # be extracted here. For regular LLD flow, incorrect aliased symbols can |
| # arise. But that's a lesser evil compared to having LLD-LTO .bss missing |
| # object_path and source_path. |
| # TODO(huangs): Fix aliased symbols for the LLD case. |
| if (symbol.IsStringLiteral() or not full_name or full_name[0] in '*.' |
| or # e.g. ** merge symbols, .Lswitch.table |
| full_name == 'startup'): |
| continue |
| |
| object_paths = object_paths_by_name.get(full_name) |
| if object_paths: |
| num_found_paths += 1 |
| else: |
| # Happens a lot with code that has LTO enabled (linker creates symbols). |
| num_unknown_names += 1 |
| continue |
| |
| if symbol.object_path and symbol.object_path not in object_paths: |
| if num_path_mismatches < 10: |
| logging.warning('Symbol path reported by .map not found by nm.') |
| logging.warning('sym=%r', symbol) |
| logging.warning('paths=%r', object_paths) |
| object_paths.append(symbol.object_path) |
| object_paths.sort() |
| num_path_mismatches += 1 |
| |
| symbol.object_path = object_paths[0] |
| |
| if len(object_paths) > 1: |
| # Create one symbol for each object_path. |
| aliases = symbol.aliases or [symbol] |
| symbol.aliases = aliases |
| num_aliases_created += len(object_paths) - 1 |
| for object_path in object_paths[1:]: |
| new_sym = models.Symbol(symbol.section_name, |
| symbol.size, |
| address=symbol.address, |
| full_name=full_name, |
| object_path=object_path, |
| aliases=aliases) |
| aliases.append(new_sym) |
| ret.append(new_sym) |
| |
| logging.debug( |
| 'Cross-referenced %d symbols with nm output. ' |
| 'num_unknown_names=%d num_path_mismatches=%d ' |
| 'num_aliases_created=%d', num_found_paths, num_unknown_names, |
| num_path_mismatches, num_aliases_created) |
| # Currently: num_unknown_names=1246 out of 591206 (0.2%). |
| if num_unknown_names > min(20, len(raw_symbols) * 0.01): |
| logging.warning( |
| 'Abnormal number of symbols not found in .o files (%d of %d)', |
| num_unknown_names, len(raw_symbols)) |
| return ret |
| |
| |
| def _DiscoverMissedObjectPaths(raw_symbols, known_inputs): |
| # Missing object paths are caused by .a files added by -l flags, which are not |
| # listed as explicit inputs within .ninja rules. |
| missed_inputs = set() |
| for symbol in raw_symbols: |
| path = symbol.object_path |
| if path.endswith(')'): |
| # Convert foo/bar.a(baz.o) -> foo/bar.a |
| path = path[:path.rindex('(')] |
| if path and path not in known_inputs: |
| missed_inputs.add(path) |
| return missed_inputs |
| |
| |
| def _CreateMergeStringsReplacements(merge_string_syms, |
| list_of_positions_by_object_path): |
| """Creates replacement symbols for |merge_syms|.""" |
| ret = [] |
| STRING_LITERAL_NAME = models.STRING_LITERAL_NAME |
| assert len(merge_string_syms) == len(list_of_positions_by_object_path) |
| tups = zip(merge_string_syms, list_of_positions_by_object_path) |
| for merge_sym, positions_by_object_path in tups: |
| merge_sym_address = merge_sym.address |
| new_symbols = [] |
| ret.append(new_symbols) |
| for object_path, positions in positions_by_object_path.items(): |
| for offset, size in positions: |
| address = merge_sym_address + offset |
| symbol = models.Symbol(models.SECTION_RODATA, |
| size, |
| address=address, |
| full_name=STRING_LITERAL_NAME, |
| object_path=object_path) |
| new_symbols.append(symbol) |
| |
| logging.debug('Created %d string literal symbols', sum(len(x) for x in ret)) |
| logging.debug('Sorting string literals') |
| for symbols in ret: |
| # For de-duping & alias creation, order by address & size. |
| # For alias symbol ordering, sort by object_path. |
| symbols.sort(key=lambda x: (x.address, -x.size, x.object_path)) |
| |
| logging.debug('Deduping string literals') |
| num_removed = 0 |
| size_removed = 0 |
| num_aliases = 0 |
| for i, symbols in enumerate(ret): |
| if not symbols: |
| continue |
| prev_symbol = symbols[0] |
| new_symbols = [prev_symbol] |
| for symbol in symbols[1:]: |
| padding = symbol.address - prev_symbol.end_address |
| if (prev_symbol.address == symbol.address |
| and prev_symbol.size == symbol.size): |
| # String is an alias. |
| num_aliases += 1 |
| aliases = prev_symbol.aliases |
| if aliases: |
| aliases.append(symbol) |
| symbol.aliases = aliases |
| else: |
| aliases = [prev_symbol, symbol] |
| prev_symbol.aliases = aliases |
| symbol.aliases = aliases |
| elif padding + symbol.size <= 0: |
| # String is a substring of prior one. |
| num_removed += 1 |
| size_removed += symbol.size |
| continue |
| elif padding < 0: |
| # String overlaps previous one. Adjust to not overlap. |
| symbol.address -= padding |
| symbol.size += padding |
| new_symbols.append(symbol) |
| prev_symbol = symbol |
| ret[i] = new_symbols |
| |
| logging.debug( |
| 'Removed %d overlapping string literals (%d bytes) & created %d aliases', |
| num_removed, size_removed, num_aliases) |
| return ret |
| |
| |
| def _AddOutlinedSymbolCountsFromNm(raw_symbols, names_by_address): |
| logging.debug('Update symbol names') |
| # linker_map_parser extracts '** outlined function' without knowing how many |
| # such symbols exist at each address. nm has this information, and stores the |
| # value as, e.g., '** outlined function * 5'. Copy the information over. |
| for s in raw_symbols: |
| if s.full_name.startswith('** outlined function'): |
| name_list = names_by_address.get(s.address) |
| if name_list: |
| for name in name_list: |
| if name.startswith('** outlined function'): |
| s.full_name = name |
| break |
| |
| |
| def _AddNmAliases(raw_symbols, names_by_address): |
| """Adds symbols that were removed by identical code folding.""" |
| # Step 1: Create list of (index_of_symbol, name_list). |
| logging.debug('Creating alias list') |
| replacements = [] |
| num_new_symbols = 0 |
| num_missing = 0 |
| missing_names = collections.defaultdict(list) |
| for i, s in enumerate(raw_symbols): |
| # Don't alias padding-only symbols (e.g. ** symbol gap) |
| if s.size_without_padding == 0: |
| continue |
| # Also skip artificial symbols that won't appear in nm output. |
| if s.full_name.startswith('** CFI jump table'): |
| continue |
| name_list = names_by_address.get(s.address) |
| if name_list: |
| if s.full_name not in name_list: |
| num_missing += 1 |
| missing_names[s.full_name].append(s.address) |
| # Sometimes happens for symbols from assembly files. |
| if num_missing < 10: |
| logging.debug('Name missing from aliases: %s %s (addr=%x)', |
| s.full_name, name_list, s.address) |
| continue |
| replacements.append((i, name_list)) |
| num_new_symbols += len(name_list) - 1 |
| |
| if missing_names and logging.getLogger().isEnabledFor(logging.INFO): |
| for address, names in names_by_address.items(): |
| for name in names: |
| if name in missing_names: |
| logging.info('Missing name %s is at address %x instead of [%s]' % |
| (name, address, ','.join('%x' % a |
| for a in missing_names[name]))) |
| |
| is_small_file = len(raw_symbols) < 1000 |
| if not is_small_file and num_new_symbols / len(raw_symbols) < .05: |
| logging.warning( |
| 'Number of aliases is oddly low (%.0f%%). It should ' |
| 'usually be around 25%%.', num_new_symbols / len(raw_symbols) * 100) |
| |
| # Step 2: Create new symbols as siblings to each existing one. |
| logging.debug('Creating %d new symbols from nm output', num_new_symbols) |
| expected_num_symbols = len(raw_symbols) + num_new_symbols |
| ret = [] |
| prev_src = 0 |
| for cur_src, name_list in replacements: |
| ret += raw_symbols[prev_src:cur_src] |
| prev_src = cur_src + 1 |
| sym = raw_symbols[cur_src] |
| # Create symbols (|sym| gets recreated and discarded). |
| new_syms = [] |
| for full_name in name_list: |
| # Do not set |aliases| in order to avoid being pruned by |
| # CompactLargeAliasesIntoSharedSymbols(), which assumes aliases differ |
| # only by path. The field will be set afterwards by _ConnectNmAliases(). |
| new_syms.append( |
| models.Symbol(sym.section_name, |
| sym.size, |
| address=sym.address, |
| full_name=full_name)) |
| ret += new_syms |
| ret += raw_symbols[prev_src:] |
| assert expected_num_symbols == len(ret) |
| return ret |
| |
| |
| def _ResolveThinArchivePaths(raw_symbols, thin_archives): |
| """Converts object_paths for thin archives to external .o paths.""" |
| for symbol in raw_symbols: |
| object_path = symbol.object_path |
| if object_path.endswith(')'): |
| start_idx = object_path.rindex('(') |
| archive_path = object_path[:start_idx] |
| if archive_path in thin_archives: |
| subpath = object_path[start_idx + 1:-1] |
| symbol.object_path = ar.CreateThinObjectPath(archive_path, subpath) |
| |
| |
| def _DeduceObjectPathForSwitchTables(raw_symbols, object_paths_by_name): |
| strip_num_suffix_regexp = re.compile(r'\s+\(\.\d+\)$') |
| num_switch_tables = 0 |
| num_unassigned = 0 |
| num_deduced = 0 |
| num_arbitrations = 0 |
| for s in raw_symbols: |
| if s.full_name.startswith('Switch table for '): |
| num_switch_tables += 1 |
| # Strip 'Switch table for ' prefix. |
| name = s.full_name[17:] |
| # Strip, e.g., ' (.123)' suffix. |
| name = re.sub(strip_num_suffix_regexp, '', name) |
| object_paths = object_paths_by_name.get(name, None) |
| if not s.object_path: |
| if object_paths is None: |
| num_unassigned += 1 |
| else: |
| num_deduced += 1 |
| # If ambiguity arises, arbitrate by taking the first. |
| s.object_path = object_paths[0] |
| if len(object_paths) > 1: |
| num_arbitrations += 1 |
| else: |
| assert object_paths and s.object_path in object_paths |
| if num_switch_tables > 0: |
| logging.info( |
| 'Found %d switch tables: Deduced %d object paths with ' + |
| '%d arbitrations. %d remain unassigned.', num_switch_tables, |
| num_deduced, num_arbitrations, num_unassigned) |
| |
| |
| def _ParseElfInfo(native_spec, outdir_context=None): |
| """Adds ELF section ranges and symbols.""" |
| assert native_spec.map_path or native_spec.elf_path, ( |
| 'Need a linker map or an ELF file.') |
| assert native_spec.map_path or not native_spec.track_string_literals, ( |
| 'track_string_literals not yet implemented without map file') |
| if native_spec.elf_path: |
| elf_section_ranges = readelf.SectionInfoFromElf(native_spec.elf_path) |
| |
| # Run nm on the elf file to retrieve the list of symbol names per-address. |
| # This list is required because the .map file contains only a single name |
| # for each address, yet multiple symbols are often coalesced when they are |
| # identical. This coalescing happens mainly for small symbols and for C++ |
| # templates. Such symbols make up ~500kb of libchrome.so on Android. |
| elf_nm_result = nm.CollectAliasesByAddressAsync(native_spec.elf_path) |
| |
| # Run nm on all .o/.a files to retrieve the symbol names within them. |
| # The list is used to detect when multiple .o files contain the same symbol |
| # (e.g. inline functions), and to update the object_path / source_path |
| # fields accordingly. |
| # Looking in object files is required because the .map file choses a |
| # single path for these symbols. |
| # Rather than record all paths for each symbol, set the paths to be the |
| # common ancestor of all paths. |
| if outdir_context and native_spec.map_path: |
| bulk_analyzer = obj_analyzer.BulkObjectFileAnalyzer( |
| outdir_context.output_directory, |
| track_string_literals=native_spec.track_string_literals) |
| bulk_analyzer.AnalyzePaths(outdir_context.elf_object_paths) |
| |
| if native_spec.map_path: |
| logging.info('Parsing Linker Map') |
| map_section_ranges, raw_symbols, linker_map_extras = ( |
| linker_map_parser.ParseFile(native_spec.map_path)) |
| |
| if outdir_context and outdir_context.thin_archives: |
| _ResolveThinArchivePaths(raw_symbols, outdir_context.thin_archives) |
| else: |
| logging.info('Collecting symbols from nm') |
| raw_symbols = nm.CreateUniqueSymbols(native_spec.elf_path, |
| elf_section_ranges) |
| |
| if native_spec.elf_path and native_spec.map_path: |
| logging.debug('Validating section sizes') |
| differing_elf_section_sizes = {} |
| differing_map_section_sizes = {} |
| for k, (_, elf_size) in elf_section_ranges.items(): |
| if k in _SECTION_SIZE_BLOCKLIST: |
| continue |
| _, map_size = map_section_ranges.get(k) |
| if map_size != elf_size: |
| differing_map_section_sizes[k] = map_size |
| differing_elf_section_sizes[k] = elf_size |
| if differing_map_section_sizes: |
| logging.error('ELF file and .map file do not agree on section sizes.') |
| logging.error('readelf: %r', differing_elf_section_sizes) |
| logging.error('.map file: %r', differing_map_section_sizes) |
| sys.exit(1) |
| |
| if native_spec.elf_path and native_spec.map_path and outdir_context: |
| missed_object_paths = _DiscoverMissedObjectPaths( |
| raw_symbols, outdir_context.known_inputs) |
| missed_object_paths = ar.ExpandThinArchives( |
| missed_object_paths, outdir_context.output_directory)[0] |
| bulk_analyzer.AnalyzePaths(missed_object_paths) |
| bulk_analyzer.SortPaths() |
| if native_spec.track_string_literals: |
| merge_string_syms = [ |
| s for s in raw_symbols if s.full_name == '** merge strings' |
| or s.full_name == '** lld merge strings' |
| ] |
| # More likely for there to be a bug in supersize than an ELF to not have a |
| # single string literal. |
| assert merge_string_syms |
| string_ranges = [(s.address, s.size) for s in merge_string_syms] |
| bulk_analyzer.AnalyzeStringLiterals(native_spec.elf_path, string_ranges) |
| |
| # Map file for some reason doesn't demangle all names. |
| # Demangle prints its own log statement. |
| demangle.DemangleRemainingSymbols(raw_symbols) |
| |
| object_paths_by_name = {} |
| if native_spec.elf_path: |
| logging.info( |
| 'Adding symbols removed by identical code folding (as reported by nm)') |
| # This normally does not block (it's finished by this time). |
| names_by_address = elf_nm_result.get() |
| if native_spec.map_path: |
| # This rewrites outlined symbols from |map_path|, and can be skipped if |
| # symbols already came from nm (e.g., for dwarf mode). |
| _AddOutlinedSymbolCountsFromNm(raw_symbols, names_by_address) |
| |
| raw_symbols = _AddNmAliases(raw_symbols, names_by_address) |
| |
| if native_spec.map_path and outdir_context: |
| object_paths_by_name = bulk_analyzer.GetSymbolNames() |
| logging.debug( |
| 'Fetched path information for %d symbols from %d files', |
| len(object_paths_by_name), |
| len(outdir_context.elf_object_paths) + len(missed_object_paths)) |
| _DeduceObjectPathForSwitchTables(raw_symbols, object_paths_by_name) |
| # For aliases, this provides path information where there wasn't any. |
| logging.info('Creating aliases for symbols shared by multiple paths') |
| raw_symbols = _AssignNmAliasPathsAndCreatePathAliases( |
| raw_symbols, object_paths_by_name) |
| |
| if native_spec.track_string_literals: |
| logging.info('Waiting for string literal extraction to complete.') |
| list_of_positions_by_object_path = bulk_analyzer.GetStringPositions() |
| bulk_analyzer.Close() |
| |
| if native_spec.track_string_literals: |
| logging.info('Deconstructing ** merge strings into literals') |
| replacements = _CreateMergeStringsReplacements( |
| merge_string_syms, list_of_positions_by_object_path) |
| for merge_sym, literal_syms in zip(merge_string_syms, replacements): |
| # Don't replace if no literals were found. |
| if literal_syms: |
| # Re-find the symbols since aliases cause their indices to change. |
| idx = raw_symbols.index(merge_sym) |
| # This assignment is a bit slow (causes array to be shifted), but |
| # is fast enough since len(merge_string_syms) < 10. |
| raw_symbols[idx:idx + 1] = literal_syms |
| |
| if native_spec.map_path: |
| linker_map_parser.DeduceObjectPathsFromThinMap(raw_symbols, |
| linker_map_extras) |
| |
| if native_spec.elf_path and native_spec.track_string_literals: |
| sym_and_string_literals = string_extract.ReadStringLiterals( |
| raw_symbols, native_spec.elf_path) |
| for sym, data in sym_and_string_literals: |
| sym.full_name = string_extract.GetNameOfStringLiteralBytes(data) |
| |
| # If we have an ELF file, use its ranges as the source of truth, since some |
| # sections can differ from the .map. |
| return (elf_section_ranges if native_spec.elf_path else map_section_ranges, |
| raw_symbols, object_paths_by_name) |
| |
| |
| def _AddUnattributedSectionSymbols(raw_symbols, section_ranges, source_path): |
| # Create symbols for ELF sections not covered by existing symbols. |
| logging.info('Searching for symbol gaps...') |
| new_syms_by_section = collections.defaultdict(list) |
| seen_sections = set() |
| |
| for section_name, group in itertools.groupby( |
| raw_symbols, lambda s: s.section_name): |
| seen_sections.add(section_name) |
| # Get last Last symbol in group. |
| sym = None # Needed for pylint. |
| for sym in group: |
| pass |
| end_address = sym.end_address # pylint: disable=undefined-loop-variable |
| size_from_syms = end_address - section_ranges[section_name][0] |
| overhead = section_ranges[section_name][1] - size_from_syms |
| assert overhead >= 0, ( |
| 'Last symbol (%s) ends %d bytes after section boundary (%x)' % |
| (sym, -overhead, sum(section_ranges[section_name]))) |
| if overhead > 0 and section_name not in models.BSS_SECTIONS: |
| new_syms_by_section[section_name].append( |
| models.Symbol(section_name, |
| overhead, |
| address=end_address, |
| full_name='** {} (unattributed)'.format(section_name), |
| source_path=source_path)) |
| logging.info('Last symbol in %s does not reach end of section, gap=%d', |
| section_name, overhead) |
| |
| # Sections that should not bundle into ".other". |
| unsummed_sections, summed_sections = models.ClassifySections( |
| section_ranges.keys()) |
| ret = [] |
| other_symbols = [] |
| # Sort keys to ensure consistent order (> 1 sections may have address = 0). |
| for section_name, (_, section_size) in list(section_ranges.items()): |
| if section_name in seen_sections: |
| continue |
| # Handle sections that don't appear in |raw_symbols|. |
| if (section_name not in unsummed_sections |
| and section_name not in summed_sections): |
| other_symbols.append( |
| models.Symbol(models.SECTION_OTHER, |
| section_size, |
| full_name='** ELF Section: {}'.format(section_name), |
| source_path=source_path)) |
| archive_util.ExtendSectionRange(section_ranges, models.SECTION_OTHER, |
| section_size) |
| else: |
| ret.append( |
| models.Symbol(section_name, |
| section_size, |
| full_name='** ELF Section: {}'.format(section_name), |
| source_path=source_path)) |
| other_symbols.sort(key=lambda s: (s.address, s.full_name)) |
| |
| # TODO(agrieve): It would probably simplify things to use a dict of |
| # section_name->raw_symbols while creating symbols. |
| # Merge |new_syms_by_section| into |raw_symbols| while maintaining ordering. |
| for section_name, group in itertools.groupby( |
| raw_symbols, lambda s: s.section_name): |
| ret.extend(group) |
| ret.extend(new_syms_by_section[section_name]) |
| return ret, other_symbols |
| |
| |
| def _ParseNinjaFiles(output_directory, elf_path=None): |
| linker_elf_path = elf_path |
| if elf_path: |
| # For partitioned libraries, the actual link command outputs __combined.so. |
| partitioned_elf_path = elf_path.replace('.so', '__combined.so') |
| if os.path.exists(partitioned_elf_path): |
| linker_elf_path = partitioned_elf_path |
| |
| logging.info('Parsing ninja files, looking for %s.', |
| (linker_elf_path or 'source mapping only (elf_path=None)')) |
| |
| source_mapper, ninja_elf_object_paths = ninja_parser.Parse( |
| output_directory, linker_elf_path) |
| |
| logging.debug('Parsed %d .ninja files. Linker inputs=%d', |
| source_mapper.parsed_file_count, |
| len(ninja_elf_object_paths or [])) |
| if elf_path: |
| assert ninja_elf_object_paths, ( |
| 'Failed to find link command in ninja files for ' + |
| os.path.relpath(linker_elf_path, output_directory)) |
| |
| return source_mapper, ninja_elf_object_paths |
| |
| |
| def _ElfInfoFromApk(apk_path, apk_so_path): |
| with zip_util.UnzipToTemp(apk_path, apk_so_path) as temp: |
| return _CreateElfInfo(temp) |
| |
| |
| def _CountRelocationsFromElf(elf_path): |
| args = [path_util.GetReadElfPath(), '-r', elf_path] |
| stdout = subprocess.check_output(args).decode('ascii') |
| relocations = re.findall( |
| 'Relocation section .* at offset .* contains (\d+) entries', stdout) |
| return sum([int(i) for i in relocations]) |
| |
| |
| def _FindToolchainSubdirs(output_directory): |
| return [ |
| n for n in os.listdir(output_directory) |
| if os.path.exists(os.path.join(output_directory, n, 'toolchain.ninja')) |
| ] |
| |
| |
| def CreateMetadata(*, native_spec, elf_info, shorten_path): |
| """Returns metadata for the given native_spec / elf_info.""" |
| logging.debug('Constructing native metadata') |
| native_metadata = {} |
| native_metadata[models.METADATA_ELF_ALGORITHM] = native_spec.algorithm |
| |
| if elf_info: |
| native_metadata[models.METADATA_ELF_ARCHITECTURE] = elf_info.architecture |
| native_metadata[models.METADATA_ELF_BUILD_ID] = elf_info.build_id |
| |
| if native_spec.apk_so_path: |
| native_metadata[models.METADATA_ELF_APK_PATH] = native_spec.apk_so_path |
| |
| if native_spec.elf_path: |
| native_metadata[models.METADATA_ELF_FILENAME] = shorten_path( |
| native_spec.elf_path) |
| timestamp_obj = datetime.datetime.utcfromtimestamp( |
| os.path.getmtime(native_spec.elf_path)) |
| timestamp = calendar.timegm(timestamp_obj.timetuple()) |
| native_metadata[models.METADATA_ELF_MTIME] = timestamp |
| |
| if native_spec.map_path: |
| native_metadata[models.METADATA_MAP_FILENAME] = shorten_path( |
| native_spec.map_path) |
| return native_metadata |
| |
| |
| def CreateSymbols(*, |
| apk_spec, |
| native_spec, |
| output_directory=None, |
| pak_id_map=None): |
| """Creates native symbols for the given native_spec. |
| |
| Args: |
| apk_spec: Instance of ApkSpec, or None. |
| native_spec: Instance of NativeSpec. |
| output_directory: Build output directory. If None, source_paths and symbol |
| alias information will not be recorded. |
| pak_id_map: Instance of PakIdMap. |
| |
| Returns: |
| A tuple of (section_ranges, raw_symbols, elf_info, metrics_by_file), where |
| metrics_by_file is a dict from file name to a dict of {metric_name: value}. |
| """ |
| apk_elf_info_result = None |
| if apk_spec and native_spec.apk_so_path: |
| # Extraction takes around 1 second, so do it in parallel. |
| apk_elf_info_result = parallel.ForkAndCall( |
| _ElfInfoFromApk, (apk_spec.apk_path, native_spec.apk_so_path)) |
| |
| raw_symbols = [] |
| ninja_source_mapper = None |
| dwarf_source_mapper = None |
| section_ranges = {} |
| ninja_elf_object_paths = None |
| metrics_by_file = {} |
| if output_directory and native_spec.map_path: |
| # Finds all objects passed to the linker and creates a map of .o -> .cc. |
| ninja_source_mapper, ninja_elf_object_paths = _ParseNinjaFiles( |
| output_directory, native_spec.elf_path) |
| elif native_spec.elf_path: |
| logging.info('Parsing source path info via dwarfdump') |
| dwarf_source_mapper = dwarfdump.CreateAddressSourceMapper( |
| native_spec.elf_path) |
| logging.info('Found %d source paths across %s ranges', |
| dwarf_source_mapper.NumberOfPaths(), |
| dwarf_source_mapper.num_ranges) |
| |
| # Start by finding elf_object_paths so that nm can run on them while the |
| # linker .map is being parsed. |
| if ninja_elf_object_paths: |
| elf_object_paths, thin_archives = ar.ExpandThinArchives( |
| ninja_elf_object_paths, output_directory) |
| known_inputs = set(elf_object_paths) |
| known_inputs.update(ninja_elf_object_paths) |
| else: |
| elf_object_paths = [] |
| known_inputs = None |
| # When we don't know which elf file is used, just search all paths. |
| # TODO(agrieve): Seems to be used only for tests. Remove? |
| if ninja_source_mapper: |
| thin_archives = set( |
| p for p in ninja_source_mapper.IterAllPaths() if p.endswith('.a') |
| and ar.IsThinArchive(os.path.join(output_directory, p))) |
| else: |
| thin_archives = None |
| |
| if output_directory: |
| toolchain_subdirs = _FindToolchainSubdirs(output_directory) |
| outdir_context = _OutputDirectoryContext(elf_object_paths=elf_object_paths, |
| known_inputs=known_inputs, |
| output_directory=output_directory, |
| thin_archives=thin_archives) |
| else: |
| toolchain_subdirs = None |
| outdir_context = None |
| |
| object_paths_by_name = None |
| if native_spec.elf_path or native_spec.map_path: |
| section_ranges, raw_symbols, object_paths_by_name = _ParseElfInfo( |
| native_spec, outdir_context=outdir_context) |
| if pak_id_map and native_spec.map_path: |
| # For trichrome, pak files are in different apks than native library, |
| # so need to pass along pak_id_map separately and ensure |
| # TrichromeLibrary appears first in .ssargs file. |
| logging.debug('Extracting pak IDs from symbol names') |
| pak_id_map.Update(object_paths_by_name, ninja_source_mapper) |
| |
| elf_info = None |
| if apk_elf_info_result: |
| logging.debug('Extracting section sizes from .so within .apk') |
| elf_info = apk_elf_info_result.get() |
| if native_spec.elf_path: |
| expected_build_id = readelf.BuildIdFromElf(native_spec.elf_path) |
| assert elf_info.build_id == expected_build_id, ( |
| 'BuildID of {} != $APK/{}: {} != {}'.format(native_spec.elf_path, |
| native_spec.apk_so_path, |
| expected_build_id, |
| elf_info.build_id)) |
| elif native_spec.elf_path: |
| # Strip ELF before capturing section information to avoid recording |
| # debug sections. |
| with tempfile.NamedTemporaryFile( |
| suffix=os.path.basename(native_spec.elf_path)) as f: |
| strip_path = path_util.GetStripPath() |
| subprocess.run([strip_path, '-o', f.name, native_spec.elf_path], |
| check=True) |
| elf_info = _CreateElfInfo(f.name) |
| |
| if elf_info: |
| section_ranges = elf_info.section_ranges.copy() |
| if native_spec.elf_path: |
| key = posixpath.basename(native_spec.elf_path) |
| metrics_by_file[key] = { |
| f'{models.METRICS_SIZE}/{k}': size |
| for (k, (offset, size)) in section_ranges.items() |
| } |
| relocations_count = _CountRelocationsFromElf(native_spec.elf_path) |
| metrics_by_file[key][ |
| f'{models.METRICS_COUNT}/{models.METRICS_COUNT_RELOCATIONS}'] = ( |
| relocations_count) |
| |
| source_path = '' |
| if native_spec.apk_so_path: |
| # Put section symbols under $NATIVE/libfoo.so (abi)/... |
| source_path = '{}/{} ({})'.format( |
| models.NATIVE_PREFIX_PATH, posixpath.basename(native_spec.apk_so_path), |
| elf_info.architecture) |
| |
| raw_symbols, other_symbols = _AddUnattributedSectionSymbols( |
| raw_symbols, section_ranges, source_path) |
| |
| if elf_info: |
| elf_overhead_size = elf_info.OverheadSize() |
| elf_overhead_symbol = models.Symbol(models.SECTION_OTHER, |
| elf_overhead_size, |
| full_name='Overhead: ELF file', |
| source_path=source_path) |
| archive_util.ExtendSectionRange(section_ranges, models.SECTION_OTHER, |
| elf_overhead_size) |
| other_symbols.append(elf_overhead_symbol) |
| |
| # Always have .other come last. |
| other_symbols.sort(key=lambda s: (s.IsOverhead(), s.full_name.startswith( |
| '**'), s.address, s.full_name)) |
| |
| if ninja_source_mapper: |
| _AddSourcePathsUsingObjectPaths(ninja_source_mapper, raw_symbols) |
| elif dwarf_source_mapper: |
| _AddSourcePathsUsingAddress(dwarf_source_mapper, raw_symbols) |
| |
| raw_symbols.extend(other_symbols) |
| |
| # Path normalization must come before compacting aliases so that |
| # ancestor paths do not mix generated and non-generated paths. |
| archive_util.NormalizePaths(raw_symbols, |
| gen_dir_regex=native_spec.gen_dir_regex, |
| toolchain_subdirs=toolchain_subdirs) |
| |
| if native_spec.elf_path or native_spec.map_path: |
| logging.info('Converting excessive aliases into shared-path symbols') |
| archive_util.CompactLargeAliasesIntoSharedSymbols( |
| raw_symbols, _MAX_SAME_NAME_ALIAS_COUNT) |
| |
| logging.debug('Connecting nm aliases') |
| _ConnectNmAliases(raw_symbols) |
| |
| return section_ranges, raw_symbols, elf_info, metrics_by_file |
