blob: 40e59bf18c46d64933a34834410ba6812a6ac45b [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Runs nm on specified .a and .o file, plus some analysis.
Runs nm on the elf to collect all symbol names. This reveals symbol names of
identical-code-folded functions.
Runs CollectAliasesByAddress in a subprocess and returns a promise.
BulkForkAndCall() target: Runs nm on a .a file or a list of .o files, parses
the output, extracts symbol information, and (if available) extracts string
offset information.
Creates Symbol objects from nm output.
import argparse
import collections
import logging
import os
import subprocess
import demangle
import models
import parallel
import path_util
import readelf
import sys
def _IsRelevantNmName(name):
# Skip lines like:
# 00000000 t $t
# 00000000 r $d.23
# 00000344 N
return name and not name.startswith('$')
def _IsRelevantObjectFileName(name):
# Prevent marking compiler-generated symbols as candidates for shared paths.
# E.g., multiple files might have "CSWTCH.12", but they are different symbols.
# Find these via:
# size_info.symbols.GroupedByFullName(min_count=-2).Filter(
# lambda s: s.WhereObjectPathMatches('{')).SortedByCount()
# and then search for {shared}.
# List of names this applies to:
# startup
# __tcf_0 <-- Generated for global destructors.
# ._79
# .Lswitch.table, .Lswitch.table.12
# lock.12
# table.12
# __compound_literal.12
# .L.ref.tmp.1
# .L.str, .L.str.3
# .L__func__.main: (when using __func__)
# .L__FUNCTION__._ZN6webrtc17AudioDeviceBuffer11StopPlayoutEv
# .L__PRETTY_FUNCTION__._Unwind_Resume
# .L_ZZ24ScaleARGBFilterCols_NEONE9dx_offset (an array literal)
if name in ('__tcf_0', 'startup'):
return False
if name.startswith('._') and name[2:].isdigit():
return False
if name.startswith('.L') and name.find('.', 2) != -1:
return False
dot_idx = name.find('.')
if dot_idx == -1:
return True
name = name[:dot_idx]
return name not in ('CSWTCH', 'lock', '__compound_literal', 'table')
def CollectAliasesByAddress(elf_path):
"""Runs nm on |elf_path| and returns a dict of address->[names]"""
# Constructors often show up twice, so use sets to ensure no duplicates.
names_by_address = collections.defaultdict(set)
# Many OUTLINED_FUNCTION_* entries can coexist on a single address, possibly
# mixed with regular symbols. However, naively keeping these is bad because:
# * OUTLINED_FUNCTION_* can have many duplicates. Keeping them would cause
# false associations downstream, when looking up object_paths from names.
# * For addresses with multiple OUTLINED_FUNCTION_* entries, we can't get the
# associated object_path (exception: the one entry in the .map file, for LLD
# without ThinLTO). So keeping copies around is rather useless.
# Our solution is to merge OUTLINED_FUNCTION_* entries at the same address
# into a single symbol. We'd also like to keep track of the number of copies
# (although it will not be used to compute PSS computation). This is done by
# writing the count in the name, e.g., '** outlined function * 5'.
num_outlined_functions_at_address = collections.Counter()
# About 60mb of output, but piping takes ~30s, and loading it into RAM
# directly takes 3s.
args = [path_util.GetNmPath(), '--no-sort', '--defined-only', elf_path]
# pylint: disable=unexpected-keyword-arg
proc = subprocess.Popen(args,
# llvm-nm may write to stderr. Discard to denoise.
stdout, _ = proc.communicate()
assert proc.returncode == 0
for line in stdout.splitlines():
space_idx = line.find(' ')
address_str = line[:space_idx]
section = line[space_idx + 1]
mangled_name = line[space_idx + 3:]
# To verify that rodata does not have aliases:
# nm --no-sort --defined-only > nm.out
# grep -v '\$' nm.out | grep ' r ' | sort | cut -d' ' -f1 > addrs
# wc -l < addrs; uniq < addrs | wc -l
if section not in 'tTW' or not _IsRelevantNmName(mangled_name):
address = int(address_str, 16)
if not address:
if mangled_name.startswith('OUTLINED_FUNCTION_'):
num_outlined_functions_at_address[address] += 1
# Need to add before demangling because |names_by_address| changes type.
for address, count in num_outlined_functions_at_address.items():
name = '** outlined function' + (' * %d' % count if count > 1 else '')
# Demangle all names.
# Since this is run in a separate process, minimize data passing by returning
# only aliased symbols.
# Also: Sort to ensure stable ordering.
return {
addr: sorted(names, key=lambda n: (n.startswith('**'), n))
for addr, names in names_by_address.items()
if len(names) > 1 or num_outlined_functions_at_address.get(addr, 0) > 1
def CreateUniqueSymbols(elf_path, section_ranges):
"""Creates symbols from nm --print-size output.
Creates only one symbol for each address (does not create symbol aliases).
# Filter to sections we care about and sort by (address, size).
section_ranges = [
x for x in section_ranges.items() if x[0] in models.NATIVE_SECTIONS
section_ranges.sort(key=lambda x: x[1])
min_address = section_ranges[0][1][0]
max_address = sum(section_ranges[-1][1])
args = [
path_util.GetNmPath(), '--no-sort', '--defined-only', '--print-size',
# pylint: disable=unexpected-keyword-arg
stdout = subprocess.check_output(args,
lines = stdout.splitlines()
logging.debug('Parsing %d lines of output', len(lines))
symbols_by_address = {}
# Example 32-bit output:
# 00857f94 00000004 t __on_dlclose_late
# 000001ec r ndk_build_number
for line in lines:
tokens = line.split(' ', 3)
num_tokens = len(tokens)
if num_tokens < 3:
# Address with no size and no name.
address_str = tokens[0]
# Check if size is omitted (can happen with binutils but not llvm).
if num_tokens == 3:
size_str = '0'
section = tokens[1]
mangled_name = tokens[2]
size_str = tokens[1]
section = tokens[2]
mangled_name = tokens[3]
if section not in 'BbDdTtRrWw' or not _IsRelevantNmName(mangled_name):
address = int(address_str, 16)
# Ignore symbols outside of sections that we care about.
# Symbols can still exist in sections that we do not care about if those
# sections are interleaved. We discard such symbols in the next loop.
if not min_address <= address < max_address:
# Pick the alias that defines a size.
existing_alias = symbols_by_address.get(address)
if existing_alias and existing_alias.size > 0:
size = int(size_str, 16)
# E.g.: .str.2.llvm.12282370934750212
if mangled_name.startswith('.str.'):
mangled_name = models.STRING_LITERAL_NAME
elif mangled_name.startswith('__ARMV7PILongThunk_'):
# Convert thunks from prefix to suffix so that name is demangleable.
mangled_name = mangled_name[len('__ARMV7PILongThunk_'):] + '.LongThunk'
elif mangled_name.startswith('__ThumbV7PILongThunk_'):
mangled_name = mangled_name[len('__ThumbV7PILongThunk_'):] + '.LongThunk'
# Use address (next loop) to determine between .data and
section_name = None
if section in 'Tt':
section_name = models.SECTION_TEXT
elif section in 'Rr':
section_name = models.SECTION_RODATA
elif section in 'Bb':
section_name = models.SECTION_BSS
# No need to demangle names since they will be demangled by
# DemangleRemainingSymbols().
symbols_by_address[address] = models.Symbol(section_name,
logging.debug('Sorting %d NM symbols', len(symbols_by_address))
# Sort symbols by address.
sorted_symbols = sorted(symbols_by_address.values(), key=lambda s: s.address)
# Assign section to symbols based on address, and size where unspecified.
# Use address rather than nm's section character to distinguish between
# and .data.
logging.debug('Assigning section_name and filling in missing sizes')
section_range_iter = iter(section_ranges)
section_end = -1
raw_symbols = []
active_assembly_sym = None
for i, sym in enumerate(sorted_symbols):
# Move to next section if applicable.
while sym.address >= section_end:
section_range = next(section_range_iter)
section_name, (section_start, section_size) = section_range
section_end = section_start + section_size
# Skip symbols that don't fall into a section that we care about
# (e.g. GCC_except_table533 from .eh_frame).
if sym.address < section_start:
if sym.section_name and sym.section_name != section_name:
logging.warning('Re-assigning section for %r to %s', sym, section_name)
sym.section_name = section_name
if i + 1 < len(sorted_symbols):
next_addr = min(section_end, sorted_symbols[i + 1].address)
next_addr = section_end
# Heuristic: Discard subsequent assembly symbols (no size) that are ALL_CAPS
# or .-prefixed, since they are likely labels within a function.
if (active_assembly_sym and sym.size == 0
and sym.section_name == models.SECTION_TEXT):
if sym.full_name.startswith('.') or sym.full_name.isupper():
active_assembly_sym.size += next_addr - sym.address
# Triggers ~30 times for all of
logging.debug('Discarding assembly label: %s', sym.full_name)
active_assembly_sym = sym if sym.size == 0 else None
# For assembly symbols:
# Add in a size when absent and guard against size overlapping next symbol.
if active_assembly_sym or sym.end_address > next_addr:
sym.size = next_addr - sym.address
return raw_symbols
def _CollectAliasesByAddressAsyncHelper(elf_path):
result = CollectAliasesByAddress(elf_path)
return parallel.EncodeDictOfLists(result, key_transform=str)
def CollectAliasesByAddressAsync(elf_path):
"""Calls CollectAliasesByAddress in a helper process. Returns a Result."""
def decode(encoded):
return parallel.DecodeDictOfLists(encoded, key_transform=int)
return parallel.ForkAndCall(_CollectAliasesByAddressAsyncHelper, (elf_path, ),
def _ParseOneObjectFileNmOutput(lines):
# Constructors are often repeated because they have the same unmangled
# name, but multiple mangled names. See:
symbol_names = set()
string_addresses = []
for line in lines:
if not line:
space_idx = line.find(' ') # Skip over address.
section = line[space_idx + 1]
mangled_name = line[space_idx + 3:]
if _IsRelevantNmName(mangled_name):
# Refer to _IsRelevantObjectFileName() for examples of names.
if section == 'r' and (
mangled_name.startswith('.L.str') or
mangled_name.startswith('.L__') and mangled_name.find('.', 3) != -1):
# Leave as a string for easier marshalling.
string_addresses.append(line[:space_idx].lstrip('0') or '0')
elif _IsRelevantObjectFileName(mangled_name):
return symbol_names, string_addresses
# This is a target for BulkForkAndCall().
def RunNmOnIntermediates(target, output_directory):
"""Returns encoded_symbol_names_by_path, encoded_string_addresses_by_path.
target: Either a single path to a .a (as a string), or a list of .o paths.
is_archive = isinstance(target, str)
args = [path_util.GetNmPath(), '--no-sort', '--defined-only']
if is_archive:
proc = subprocess.Popen(
# llvm-nm can print 'no symbols' to stderr. Capture and count the number of
# lines, to be returned to the caller.
stdout, stderr = proc.communicate()
assert proc.returncode == 0, 'NM failed: ' + ' '.join(args)
num_no_symbols = len(stderr.splitlines())
lines = stdout.splitlines()
# Empty .a file has no output.
if not lines:
is_multi_file = not lines[0]
lines = iter(lines)
if is_multi_file:
path = next(lines)[:-1] # Path ends with a colon.
assert not is_archive
path = target[0]
symbol_names_by_path = {}
string_addresses_by_path = {}
while path:
if is_archive:
# E.g. foo/bar.a(baz.o)
path = '%s(%s)' % (target, path)
mangled_symbol_names, string_addresses = _ParseOneObjectFileNmOutput(lines)
symbol_names_by_path[path] = mangled_symbol_names
if string_addresses:
string_addresses_by_path[path] = string_addresses
path = next(lines, ':')[:-1]
# The multiprocess API uses pickle, which is ridiculously slow. More than 2x
# faster to use join & split.
# TODO(agrieve): We could use path indices as keys rather than paths to cut
# down on marshalling overhead.
return (parallel.EncodeDictOfLists(symbol_names_by_path),
parallel.EncodeDictOfLists(string_addresses_by_path), num_no_symbols)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output-directory', required=True)
parser.add_argument('elf_path', type=os.path.realpath)
args = parser.parse_args()
format='%(levelname).1s %(relativeCreated)6d %(message)s')
# Other functions in this file have test entrypoints in
section_ranges = readelf.SectionInfoFromElf(args.elf_path)
symbols = CreateUniqueSymbols(args.elf_path, section_ranges)
for s in symbols:
logging.warning('Printed %d symbols', len(symbols))
if __name__ == '__main__':