| #!/usr/bin/env python3 |
| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Runs nm on specified .a and .o file, plus some analysis. |
| |
| CollectAliasesByAddress(): |
| Runs nm on the elf to collect all symbol names. This reveals symbol names of |
| identical-code-folded functions. |
| |
| CollectAliasesByAddressAsync(): |
| Runs CollectAliasesByAddress in a subprocess and returns a promise. |
| |
| RunNmOnIntermediates(): |
| BulkForkAndCall() target: Runs nm on a .a file or a list of .o files, parses |
| the output, extracts symbol information, and (if available) extracts string |
| offset information. |
| |
| CreateUniqueSymbols(): |
| Creates Symbol objects from nm output. |
| """ |
| |
| import argparse |
| import collections |
| import logging |
| import os |
| import subprocess |
| |
| import demangle |
| import models |
| import parallel |
| import path_util |
| import readelf |
| import sys |
| |
| |
| def _IsRelevantNmName(name): |
| # Skip lines like: |
| # 00000000 t $t |
| # 00000000 r $d.23 |
| # 00000344 N |
| return name and not name.startswith('$') |
| |
| |
| def _IsRelevantObjectFileName(name): |
| # Prevent marking compiler-generated symbols as candidates for shared paths. |
| # E.g., multiple files might have "CSWTCH.12", but they are different symbols. |
| # |
| # Find these via: |
| # size_info.symbols.GroupedByFullName(min_count=-2).Filter( |
| # lambda s: s.WhereObjectPathMatches('{')).SortedByCount() |
| # and then search for {shared}. |
| # List of names this applies to: |
| # startup |
| # __tcf_0 <-- Generated for global destructors. |
| # ._79 |
| # .Lswitch.table, .Lswitch.table.12 |
| # CSWTCH.12 |
| # lock.12 |
| # table.12 |
| # __compound_literal.12 |
| # .L.ref.tmp.1 |
| # .L.str, .L.str.3 |
| # .L__func__.main: (when using __func__) |
| # .L__FUNCTION__._ZN6webrtc17AudioDeviceBuffer11StopPlayoutEv |
| # .L__PRETTY_FUNCTION__._Unwind_Resume |
| # .L_ZZ24ScaleARGBFilterCols_NEONE9dx_offset (an array literal) |
| if name in ('__tcf_0', 'startup'): |
| return False |
| if name.startswith('._') and name[2:].isdigit(): |
| return False |
| if name.startswith('.L') and name.find('.', 2) != -1: |
| return False |
| |
| dot_idx = name.find('.') |
| if dot_idx == -1: |
| return True |
| name = name[:dot_idx] |
| |
| return name not in ('CSWTCH', 'lock', '__compound_literal', 'table') |
| |
| |
| def CollectAliasesByAddress(elf_path): |
| """Runs nm on |elf_path| and returns a dict of address->[names]""" |
| # Constructors often show up twice, so use sets to ensure no duplicates. |
| names_by_address = collections.defaultdict(set) |
| |
| # Many OUTLINED_FUNCTION_* entries can coexist on a single address, possibly |
| # mixed with regular symbols. However, naively keeping these is bad because: |
| # * OUTLINED_FUNCTION_* can have many duplicates. Keeping them would cause |
| # false associations downstream, when looking up object_paths from names. |
| # * For addresses with multiple OUTLINED_FUNCTION_* entries, we can't get the |
| # associated object_path (exception: the one entry in the .map file, for LLD |
| # without ThinLTO). So keeping copies around is rather useless. |
| # Our solution is to merge OUTLINED_FUNCTION_* entries at the same address |
| # into a single symbol. We'd also like to keep track of the number of copies |
| # (although it will not be used to compute PSS computation). This is done by |
| # writing the count in the name, e.g., '** outlined function * 5'. |
| num_outlined_functions_at_address = collections.Counter() |
| |
| # About 60mb of output, but piping takes ~30s, and loading it into RAM |
| # directly takes 3s. |
| args = [path_util.GetNmPath(), '--no-sort', '--defined-only', elf_path] |
| # pylint: disable=unexpected-keyword-arg |
| proc = subprocess.Popen(args, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.DEVNULL, |
| encoding='utf-8') |
| # llvm-nm may write to stderr. Discard to denoise. |
| stdout, _ = proc.communicate() |
| assert proc.returncode == 0 |
| for line in stdout.splitlines(): |
| space_idx = line.find(' ') |
| address_str = line[:space_idx] |
| section = line[space_idx + 1] |
| mangled_name = line[space_idx + 3:] |
| |
| # To verify that rodata does not have aliases: |
| # nm --no-sort --defined-only libchrome.so > nm.out |
| # grep -v '\$' nm.out | grep ' r ' | sort | cut -d' ' -f1 > addrs |
| # wc -l < addrs; uniq < addrs | wc -l |
| if section not in 'tTW' or not _IsRelevantNmName(mangled_name): |
| continue |
| |
| address = int(address_str, 16) |
| if not address: |
| continue |
| if mangled_name.startswith('OUTLINED_FUNCTION_'): |
| num_outlined_functions_at_address[address] += 1 |
| else: |
| names_by_address[address].add(mangled_name) |
| |
| # Need to add before demangling because |names_by_address| changes type. |
| for address, count in num_outlined_functions_at_address.items(): |
| name = '** outlined function' + (' * %d' % count if count > 1 else '') |
| names_by_address[address].add(name) |
| |
| # Demangle all names. |
| demangle.DemangleSetsInDictsInPlace(names_by_address) |
| |
| # Since this is run in a separate process, minimize data passing by returning |
| # only aliased symbols. |
| # Also: Sort to ensure stable ordering. |
| return { |
| addr: sorted(names, key=lambda n: (n.startswith('**'), n)) |
| for addr, names in names_by_address.items() |
| if len(names) > 1 or num_outlined_functions_at_address.get(addr, 0) > 1 |
| } |
| |
| |
| def CreateUniqueSymbols(elf_path, section_ranges): |
| """Creates symbols from nm --print-size output. |
| |
| Creates only one symbol for each address (does not create symbol aliases). |
| """ |
| # Filter to sections we care about and sort by (address, size). |
| section_ranges = [ |
| x for x in section_ranges.items() if x[0] in models.NATIVE_SECTIONS |
| ] |
| section_ranges.sort(key=lambda x: x[1]) |
| min_address = section_ranges[0][1][0] |
| max_address = sum(section_ranges[-1][1]) |
| |
| args = [ |
| path_util.GetNmPath(), '--no-sort', '--defined-only', '--print-size', |
| elf_path |
| ] |
| # pylint: disable=unexpected-keyword-arg |
| stdout = subprocess.check_output(args, |
| stderr=subprocess.DEVNULL, |
| encoding='utf-8') |
| lines = stdout.splitlines() |
| logging.debug('Parsing %d lines of output', len(lines)) |
| symbols_by_address = {} |
| # Example 32-bit output: |
| # 00857f94 00000004 t __on_dlclose_late |
| # 000001ec r ndk_build_number |
| for line in lines: |
| tokens = line.split(' ', 3) |
| num_tokens = len(tokens) |
| if num_tokens < 3: |
| # Address with no size and no name. |
| continue |
| address_str = tokens[0] |
| # Check if size is omitted (can happen with binutils but not llvm). |
| if num_tokens == 3: |
| size_str = '0' |
| section = tokens[1] |
| mangled_name = tokens[2] |
| else: |
| size_str = tokens[1] |
| section = tokens[2] |
| mangled_name = tokens[3] |
| |
| if section not in 'BbDdTtRrWw' or not _IsRelevantNmName(mangled_name): |
| continue |
| |
| address = int(address_str, 16) |
| |
| # Ignore symbols outside of sections that we care about. |
| # Symbols can still exist in sections that we do not care about if those |
| # sections are interleaved. We discard such symbols in the next loop. |
| if not min_address <= address < max_address: |
| continue |
| |
| # Pick the alias that defines a size. |
| existing_alias = symbols_by_address.get(address) |
| if existing_alias and existing_alias.size > 0: |
| continue |
| |
| size = int(size_str, 16) |
| |
| # E.g.: .str.2.llvm.12282370934750212 |
| if mangled_name.startswith('.str.'): |
| mangled_name = models.STRING_LITERAL_NAME |
| elif mangled_name.startswith('__ARMV7PILongThunk_'): |
| # Convert thunks from prefix to suffix so that name is demangleable. |
| mangled_name = mangled_name[len('__ARMV7PILongThunk_'):] + '.LongThunk' |
| elif mangled_name.startswith('__ThumbV7PILongThunk_'): |
| mangled_name = mangled_name[len('__ThumbV7PILongThunk_'):] + '.LongThunk' |
| |
| # Use address (next loop) to determine between .data and .data.rel.ro. |
| section_name = None |
| if section in 'Tt': |
| section_name = models.SECTION_TEXT |
| elif section in 'Rr': |
| section_name = models.SECTION_RODATA |
| elif section in 'Bb': |
| section_name = models.SECTION_BSS |
| |
| # No need to demangle names since they will be demangled by |
| # DemangleRemainingSymbols(). |
| symbols_by_address[address] = models.Symbol(section_name, |
| size, |
| address=address, |
| full_name=mangled_name) |
| |
| logging.debug('Sorting %d NM symbols', len(symbols_by_address)) |
| # Sort symbols by address. |
| sorted_symbols = sorted(symbols_by_address.values(), key=lambda s: s.address) |
| |
| # Assign section to symbols based on address, and size where unspecified. |
| # Use address rather than nm's section character to distinguish between |
| # .data.rel.ro and .data. |
| logging.debug('Assigning section_name and filling in missing sizes') |
| section_range_iter = iter(section_ranges) |
| section_end = -1 |
| raw_symbols = [] |
| active_assembly_sym = None |
| for i, sym in enumerate(sorted_symbols): |
| # Move to next section if applicable. |
| while sym.address >= section_end: |
| section_range = next(section_range_iter) |
| section_name, (section_start, section_size) = section_range |
| section_end = section_start + section_size |
| |
| # Skip symbols that don't fall into a section that we care about |
| # (e.g. GCC_except_table533 from .eh_frame). |
| if sym.address < section_start: |
| continue |
| |
| if sym.section_name and sym.section_name != section_name: |
| logging.warning('Re-assigning section for %r to %s', sym, section_name) |
| sym.section_name = section_name |
| |
| if i + 1 < len(sorted_symbols): |
| next_addr = min(section_end, sorted_symbols[i + 1].address) |
| else: |
| next_addr = section_end |
| |
| # Heuristic: Discard subsequent assembly symbols (no size) that are ALL_CAPS |
| # or .-prefixed, since they are likely labels within a function. |
| if (active_assembly_sym and sym.size == 0 |
| and sym.section_name == models.SECTION_TEXT): |
| if sym.full_name.startswith('.') or sym.full_name.isupper(): |
| active_assembly_sym.size += next_addr - sym.address |
| # Triggers ~30 times for all of libchrome.so. |
| logging.debug('Discarding assembly label: %s', sym.full_name) |
| continue |
| |
| active_assembly_sym = sym if sym.size == 0 else None |
| |
| # For assembly symbols: |
| # Add in a size when absent and guard against size overlapping next symbol. |
| if active_assembly_sym or sym.end_address > next_addr: |
| sym.size = next_addr - sym.address |
| |
| raw_symbols.append(sym) |
| |
| return raw_symbols |
| |
| |
| def _CollectAliasesByAddressAsyncHelper(elf_path): |
| result = CollectAliasesByAddress(elf_path) |
| return parallel.EncodeDictOfLists(result, key_transform=str) |
| |
| |
| def CollectAliasesByAddressAsync(elf_path): |
| """Calls CollectAliasesByAddress in a helper process. Returns a Result.""" |
| def decode(encoded): |
| return parallel.DecodeDictOfLists(encoded, key_transform=int) |
| |
| return parallel.ForkAndCall(_CollectAliasesByAddressAsyncHelper, (elf_path, ), |
| decode_func=decode) |
| |
| |
| def _ParseOneObjectFileNmOutput(lines): |
| # Constructors are often repeated because they have the same unmangled |
| # name, but multiple mangled names. See: |
| # https://stackoverflow.com/questions/6921295/dual-emission-of-constructor-symbols |
| symbol_names = set() |
| string_addresses = [] |
| for line in lines: |
| if not line: |
| break |
| space_idx = line.find(' ') # Skip over address. |
| section = line[space_idx + 1] |
| mangled_name = line[space_idx + 3:] |
| if _IsRelevantNmName(mangled_name): |
| # Refer to _IsRelevantObjectFileName() for examples of names. |
| if section == 'r' and ( |
| mangled_name.startswith('.L.str') or |
| mangled_name.startswith('.L__') and mangled_name.find('.', 3) != -1): |
| # Leave as a string for easier marshalling. |
| string_addresses.append(line[:space_idx].lstrip('0') or '0') |
| elif _IsRelevantObjectFileName(mangled_name): |
| symbol_names.add(mangled_name) |
| return symbol_names, string_addresses |
| |
| |
| # This is a target for BulkForkAndCall(). |
| def RunNmOnIntermediates(target, output_directory): |
| """Returns encoded_symbol_names_by_path, encoded_string_addresses_by_path. |
| |
| Args: |
| target: Either a single path to a .a (as a string), or a list of .o paths. |
| """ |
| is_archive = isinstance(target, str) |
| args = [path_util.GetNmPath(), '--no-sort', '--defined-only'] |
| if is_archive: |
| args.append(target) |
| else: |
| args.extend(target) |
| proc = subprocess.Popen( |
| args, |
| cwd=output_directory, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| encoding='utf-8') |
| # llvm-nm can print 'no symbols' to stderr. Capture and count the number of |
| # lines, to be returned to the caller. |
| stdout, stderr = proc.communicate() |
| assert proc.returncode == 0, 'NM failed: ' + ' '.join(args) |
| num_no_symbols = len(stderr.splitlines()) |
| lines = stdout.splitlines() |
| # Empty .a file has no output. |
| if not lines: |
| return parallel.EMPTY_ENCODED_DICT, parallel.EMPTY_ENCODED_DICT |
| is_multi_file = not lines[0] |
| lines = iter(lines) |
| if is_multi_file: |
| next(lines) |
| path = next(lines)[:-1] # Path ends with a colon. |
| else: |
| assert not is_archive |
| path = target[0] |
| |
| symbol_names_by_path = {} |
| string_addresses_by_path = {} |
| while path: |
| if is_archive: |
| # E.g. foo/bar.a(baz.o) |
| path = '%s(%s)' % (target, path) |
| |
| mangled_symbol_names, string_addresses = _ParseOneObjectFileNmOutput(lines) |
| symbol_names_by_path[path] = mangled_symbol_names |
| if string_addresses: |
| string_addresses_by_path[path] = string_addresses |
| path = next(lines, ':')[:-1] |
| |
| # The multiprocess API uses pickle, which is ridiculously slow. More than 2x |
| # faster to use join & split. |
| # TODO(agrieve): We could use path indices as keys rather than paths to cut |
| # down on marshalling overhead. |
| return (parallel.EncodeDictOfLists(symbol_names_by_path), |
| parallel.EncodeDictOfLists(string_addresses_by_path), num_no_symbols) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--output-directory', required=True) |
| parser.add_argument('elf_path', type=os.path.realpath) |
| |
| args = parser.parse_args() |
| logging.basicConfig(level=logging.DEBUG, |
| format='%(levelname).1s %(relativeCreated)6d %(message)s') |
| |
| # Other functions in this file have test entrypoints in object_analyzer.py. |
| section_ranges = readelf.SectionInfoFromElf(args.elf_path) |
| symbols = CreateUniqueSymbols(args.elf_path, section_ranges) |
| for s in symbols: |
| print(s) |
| logging.warning('Printed %d symbols', len(symbols)) |
| |
| |
| if __name__ == '__main__': |
| main() |