| #!/usr/bin/env python |
| # |
| # Copyright (C) 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """annotate.py: annotate source files based on perf.data. |
| """ |
| |
| |
| import argparse |
| import os |
| import os.path |
| import shutil |
| import subprocess |
| import sys |
| |
| from simpleperf_report_lib import * |
| from utils import * |
| |
| # TODO: using addr2line can't convert from function_start_address to |
| # source_file:line very well for java code. Because in .debug_line section, |
| # there is some distance between function_start_address and the address |
| # of the first instruction which can be mapped to source line. |
| class Addr2Line(object): |
| """collect information of how to map [dso_name,vaddr] to [source_file:line]. |
| """ |
| def __init__(self, annotator, addr2line_path): |
| self.dso_dict = dict() |
| self.annotator = annotator |
| self.addr2line_path = addr2line_path |
| |
| |
| def add_addr(self, dso_name, addr): |
| dso = self.dso_dict.get(dso_name) |
| if dso is None: |
| self.dso_dict[dso_name] = dso = dict() |
| if not dso.has_key(addr): |
| dso[addr] = None |
| |
| |
| def convert_addrs_to_lines(self): |
| # store a list of source files |
| self.file_list = [] |
| # map from file to id with file_list[id] == file |
| self.file_dict = {} |
| |
| for dso_name in self.dso_dict.keys(): |
| self._convert_addrs_to_lines(dso_name, self.dso_dict[dso_name]) |
| self._combine_source_files() |
| |
| |
| def _convert_addrs_to_lines(self, dso_name, dso): |
| dso_path = self.annotator.find_dso_path(dso_name) |
| if dso_path is None: |
| log_warning("can't find dso '%s'" % dso_name) |
| dso.clear() |
| return |
| addrs = sorted(dso.keys()); |
| addr_str = [] |
| for addr in addrs: |
| addr_str.append('0x%x' % addr) |
| addr_str = '\n'.join(addr_str) |
| subproc = subprocess.Popen([self.addr2line_path, '-e', dso_path], |
| stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
| (stdoutdata, _) = subproc.communicate(addr_str) |
| stdoutdata = stdoutdata.split('\n') |
| if len(stdoutdata) < len(addrs): |
| log_fatal("addr2line didn't output enough lines") |
| for i in range(len(addrs)): |
| strs = stdoutdata[i].split(':') |
| if len(strs) == 2 and strs[0].find('?') == -1: |
| file = strs[0].strip() |
| if strs[1].find('?') == -1: |
| line = 0 |
| for c in strs[1]: |
| if c.isdigit(): |
| line = line * 10 + ord(c) - ord('0') |
| else: |
| break |
| else: |
| line = None |
| id = self.file_dict.get(file) |
| if id is None: |
| id = len(self.file_list) |
| self.file_list.append(file) |
| self.file_dict[file] = id |
| dso[addrs[i]] = (id, line) |
| |
| |
| def _combine_source_files(self): |
| """It is possible that addr2line gives us different names for the same |
| file, like: |
| /usr/local/.../src/main/jni/sudo-game-jni.cpp |
| sudo-game-jni.cpp |
| We'd better combine these two files. We can do it by combining |
| source files with no conflicts in path. |
| """ |
| # Collect files having the same filename. |
| filename_dict = dict() |
| for file in self.file_list: |
| index = max(file.rfind('/'), file.rfind(os.sep)) |
| filename = file[index+1:] |
| entry = filename_dict.get(filename) |
| if entry is None: |
| filename_dict[filename] = entry = [] |
| entry.append(file) |
| |
| # Combine files having the same filename and having no conflicts in path. |
| for filename in filename_dict.keys(): |
| files = filename_dict[filename] |
| if len(files) == 1: |
| continue |
| for file in files: |
| to_file = file |
| # Test if we can merge files[i] with another file having longer |
| # path. |
| for f in files: |
| if len(f) > len(to_file) and f.find(file) != -1: |
| to_file = f |
| if to_file != file: |
| from_id = self.file_dict[file] |
| to_id = self.file_dict[to_file] |
| self.file_list[from_id] = self.file_list[to_id] |
| |
| |
| def get_source(self, dso_name, addr): |
| dso = self.dso_dict.get(dso_name) |
| if dso is None: |
| return (None, None) |
| item = dso.get(addr) |
| if item is None: |
| return (None, None) |
| return (self.file_list[item[0]], item[1]) |
| |
| |
| class Period(object): |
| """event count information. It can be used to represent event count |
| of a line, a function, a source file, or a binary. It contains two |
| parts: period and acc_period. |
| When used for a line, period is the event count occurred when running |
| that line, acc_period is the accumulated event count occurred when |
| running that line and functions called by that line. Same thing applies |
| when it is used for a function, a source file, or a binary. |
| """ |
| def __init__(self, period=0, acc_period=0): |
| self.period = period |
| self.acc_period = acc_period |
| |
| |
| def __iadd__(self, other): |
| self.period += other.period |
| self.acc_period += other.acc_period |
| return self |
| |
| |
| class DsoPeriod(object): |
| """Period for each shared library""" |
| def __init__(self, dso_name): |
| self.dso_name = dso_name |
| self.period = Period() |
| |
| |
| def add_period(self, period): |
| self.period += period |
| |
| |
| class FilePeriod(object): |
| """Period for each source file""" |
| def __init__(self, file): |
| self.file = file |
| self.period = Period() |
| # Period for each line in the file. |
| self.line_dict = {} |
| # Period for each function in the source file. |
| self.function_dict = {} |
| |
| |
| def add_period(self, period): |
| self.period += period |
| |
| |
| def add_line_period(self, line, period): |
| a = self.line_dict.get(line) |
| if a is None: |
| self.line_dict[line] = a = Period() |
| a += period |
| |
| |
| def add_function_period(self, function_name, function_start_line, period): |
| a = self.function_dict.get(function_name) |
| if a is None: |
| if function_start_line is None: |
| function_start_line = -1 |
| self.function_dict[function_name] = a = [function_start_line, Period()] |
| a[1] += period |
| |
| |
| class SourceFileAnnotator(object): |
| """group code for annotating source files""" |
| def __init__(self, config): |
| # check config variables |
| config_names = ['perf_data_list', 'symfs_dir', 'source_dirs', |
| 'annotate_dest_dir', 'comm_filters', 'pid_filters', |
| 'tid_filters', 'dso_filters', 'addr2line_path'] |
| for name in config_names: |
| if not config.has_key(name): |
| log_fatal('config [%s] is missing' % name) |
| symfs_dir = config['symfs_dir'] |
| if symfs_dir and not os.path.isdir(symfs_dir): |
| log_fatal('[symfs_dir] "%s" is not a dir' % symfs_dir) |
| kallsyms = config['kallsyms'] |
| if kallsyms and not os.path.isfile(kallsyms): |
| log_fatal('[kallsyms] "%s" is not a file' % kallsyms) |
| source_dirs = config['source_dirs'] |
| for dir in source_dirs: |
| if not os.path.isdir(dir): |
| log_fatal('[source_dirs] "%s" is not a dir' % dir) |
| |
| # init member variables |
| self.config = config |
| self.symfs_dir = None |
| self.kallsyms = None |
| self.comm_filter = None |
| self.pid_filter = None |
| self.tid_filter = None |
| self.dso_filter = None |
| symfs_dir = config['symfs_dir'] |
| if symfs_dir: |
| self.symfs_dir = symfs_dir |
| kallsyms = config['kallsyms'] |
| if kallsyms: |
| self.kallsyms = kallsyms |
| comm_filter = config['comm_filters'] |
| if comm_filter: |
| self.comm_filter = set(comm_filter) |
| pid_filter = config['pid_filters'] |
| if pid_filter: |
| self.pid_filter = set() |
| for pid in pid_filter: |
| self.pid_filter.add(int(pid)) |
| tid_filter = config['tid_filters'] |
| if tid_filter: |
| self.tid_filter = set() |
| for tid in tid_filter: |
| self.tid_filter.add(int(tid)) |
| dso_filter = config['dso_filters'] |
| if dso_filter: |
| self.dso_filter = set(dso_filter) |
| |
| output_dir = config['annotate_dest_dir'] |
| if os.path.isdir(output_dir): |
| shutil.rmtree(output_dir) |
| os.makedirs(output_dir) |
| |
| self.addr2line = Addr2Line(self, self.config['addr2line_path']) |
| |
| |
| def annotate(self): |
| self._collect_addrs() |
| self._convert_addrs_to_lines() |
| self._generate_periods() |
| self._write_summary() |
| self._collect_source_files() |
| self._annotate_files() |
| |
| |
| def _collect_addrs(self): |
| """Read perf.data, collect all addresses we need to convert to |
| source file:line. |
| """ |
| for perf_data in self.config['perf_data_list']: |
| lib = ReportLib() |
| if self.symfs_dir is not None: |
| lib.SetSymfs(self.symfs_dir) |
| if self.kallsyms is not None: |
| lib.SetKallsymsFile(self.kallsyms) |
| while True: |
| sample = lib.GetNextSample() |
| if sample is None: |
| lib.Close() |
| break |
| if not self._filter_sample(sample): |
| continue |
| symbols = [] |
| symbols.append(lib.GetSymbolOfCurrentSample()) |
| callchain = lib.GetCallChainOfCurrentSample() |
| for i in range(callchain.nr): |
| symbols.append(callchain.entries[i].symbol) |
| for symbol in symbols: |
| if self._filter_symbol(symbol): |
| self.addr2line.add_addr(symbol.dso_name, symbol.vaddr_in_file) |
| self.addr2line.add_addr(symbol.dso_name, symbol.symbol_addr) |
| |
| |
| def _filter_sample(self, sample): |
| """Return true if the sample can be used.""" |
| if self.comm_filter is not None: |
| if sample.thread_comm not in self.comm_filter: |
| return False |
| if self.pid_filter is not None: |
| if sample.pid not in self.pid_filter: |
| return False |
| if self.tid_filter is not None: |
| if sample.tid not in self.tid_filter: |
| return False |
| return True |
| |
| |
| def _filter_symbol(self, symbol): |
| if self.dso_filter is None or symbol.dso_name in self.dso_filter: |
| return True |
| return False |
| |
| |
| def _convert_addrs_to_lines(self): |
| self.addr2line.convert_addrs_to_lines() |
| |
| |
| def find_dso_path(self, dso): |
| if dso[0] != '/' or dso == '//anon': |
| return None |
| if self.symfs_dir is not None: |
| dso_path = os.path.join(self.symfs_dir, dso[1:]) |
| if os.path.isfile(dso_path): |
| return dso_path |
| if os.path.isfile(dso): |
| return dso |
| return None |
| |
| |
| def _generate_periods(self): |
| """read perf.data, collect Period for all types: |
| binaries, source files, functions, lines. |
| """ |
| self.period = 0 |
| self.dso_periods = dict() |
| self.file_periods = dict() |
| for perf_data in self.config['perf_data_list']: |
| lib = ReportLib() |
| if self.symfs_dir is not None: |
| lib.SetSymfs(self.symfs_dir) |
| if self.kallsyms is not None: |
| lib.SetKallsymsFile(self.kallsyms) |
| while True: |
| sample = lib.GetNextSample() |
| if sample is None: |
| lib.Close() |
| break |
| if not self._filter_sample(sample): |
| continue |
| symbols = [] |
| symbols.append(lib.GetSymbolOfCurrentSample()) |
| callchain = lib.GetCallChainOfCurrentSample() |
| for i in range(callchain.nr): |
| symbols.append(callchain.entries[i].symbol) |
| # Each sample has a callchain, but its period is only used once |
| # to add period for each function/source_line/source_file/binary. |
| # For example, if more than one entry in the callchain hits a |
| # function, the event count of that function is only increased once. |
| # Otherwise, we may get periods > 100%. |
| is_sample_used = False |
| used_dso_dict = dict() |
| used_file_dict = dict() |
| used_function_dict = dict() |
| used_line_dict = dict() |
| period = Period(sample.period, sample.period) |
| for i in range(len(symbols)): |
| symbol = symbols[i] |
| if i == 1: |
| period = Period(0, sample.period) |
| if not self._filter_symbol(symbol): |
| continue |
| is_sample_used = True |
| # Add period to dso. |
| self._add_dso_period(symbol.dso_name, period, used_dso_dict) |
| # Add period to source file. |
| source = self.addr2line.get_source(symbol.dso_name, symbol.vaddr_in_file) |
| if source[0] is not None: |
| self._add_file_period(source[0], period, used_file_dict) |
| # Add period to line. |
| if source[1] is not None: |
| self._add_line_period(source, period, used_line_dict) |
| # Add period to function. |
| source = self.addr2line.get_source(symbol.dso_name, symbol.symbol_addr) |
| if source[0] is not None: |
| self._add_file_period(source[0], period, used_file_dict) |
| self._add_function_period(source, symbol.symbol_name, period, |
| used_function_dict) |
| |
| if is_sample_used: |
| self.period += sample.period |
| |
| |
| def _add_dso_period(self, dso_name, period, used_dso_dict): |
| if not used_dso_dict.has_key(dso_name): |
| used_dso_dict[dso_name] = True |
| dso_period = self.dso_periods.get(dso_name) |
| if dso_period is None: |
| dso_period = self.dso_periods[dso_name] = DsoPeriod(dso_name) |
| dso_period.add_period(period) |
| |
| |
| def _add_file_period(self, file, period, used_file_dict): |
| if not used_file_dict.has_key(file): |
| used_file_dict[file] = True |
| file_period = self.file_periods.get(file) |
| if file_period is None: |
| file_period = self.file_periods[file] = FilePeriod(file) |
| file_period.add_period(period) |
| |
| |
| def _add_line_period(self, source, period, used_line_dict): |
| if not used_line_dict.has_key(source): |
| used_line_dict[source] = True |
| file_period = self.file_periods[source[0]] |
| file_period.add_line_period(source[1], period) |
| |
| |
| def _add_function_period(self, source, function_name, period, used_function_dict): |
| if not used_function_dict.has_key((source[0], function_name)): |
| used_function_dict[(source[0], function_name)] = True |
| file_period = self.file_periods[source[0]] |
| file_period.add_function_period(function_name, source[1], period) |
| |
| |
| def _write_summary(self): |
| summary = os.path.join(self.config['annotate_dest_dir'], 'summary') |
| with open(summary, 'w') as f: |
| f.write('total period: %d\n\n' % self.period) |
| dso_periods = sorted(self.dso_periods.values(), |
| cmp=lambda x, y: cmp(y.period.acc_period, x.period.acc_period)) |
| for dso_period in dso_periods: |
| f.write('dso %s: %s\n' % (dso_period.dso_name, |
| self._get_percentage_str(dso_period.period))) |
| f.write('\n') |
| |
| file_periods = sorted(self.file_periods.values(), |
| cmp=lambda x, y: cmp(y.period.acc_period, x.period.acc_period)) |
| for file_period in file_periods: |
| f.write('file %s: %s\n' % (file_period.file, |
| self._get_percentage_str(file_period.period))) |
| for file_period in file_periods: |
| f.write('\n\n%s: %s\n' % (file_period.file, |
| self._get_percentage_str(file_period.period))) |
| values = [] |
| for func_name in file_period.function_dict.keys(): |
| func_start_line, period = file_period.function_dict[func_name] |
| values.append((func_name, func_start_line, period)) |
| values = sorted(values, |
| cmp=lambda x, y: cmp(y[2].acc_period, x[2].acc_period)) |
| for value in values: |
| func = file_period.function_dict[func_name] |
| f.write('\tfunction (%s): line %d, %s\n' % ( |
| value[0], value[1], self._get_percentage_str(value[2]))) |
| f.write('\n') |
| for line in sorted(file_period.line_dict.keys()): |
| f.write('\tline %d: %s\n' % ( |
| line, self._get_percentage_str(file_period.line_dict[line]))) |
| |
| |
| def _get_percentage_str(self, period, short=False): |
| s = 'acc_p: %f%%, p: %f%%' if short else 'accumulated_period: %f%%, period: %f%%' |
| return s % self._get_percentage(period) |
| |
| |
| def _get_percentage(self, period): |
| if self.period == 0: |
| return (0, 0) |
| acc_p = 100.0 * period.acc_period / self.period |
| p = 100.0 * period.period / self.period |
| return (acc_p, p) |
| |
| |
| def _collect_source_files(self): |
| self.source_file_dict = dict() |
| source_file_suffix = ['h', 'c', 'cpp', 'cc', 'java'] |
| for source_dir in self.config['source_dirs']: |
| for root, _, files in os.walk(source_dir): |
| for file in files: |
| if file[file.rfind('.')+1:] in source_file_suffix: |
| entry = self.source_file_dict.get(file) |
| if entry is None: |
| entry = self.source_file_dict[file] = [] |
| entry.append(os.path.join(root, file)) |
| |
| |
| def _find_source_file(self, file): |
| filename = file[file.rfind(os.sep)+1:] |
| source_files = self.source_file_dict.get(filename) |
| if source_files is None: |
| return None |
| match_count = 0 |
| result = None |
| for path in source_files: |
| if path.find(file) != -1: |
| match_count += 1 |
| result = path |
| if match_count > 1: |
| log_warning('multiple source for %s, select %s' % (file, result)) |
| return result |
| |
| |
| def _annotate_files(self): |
| """Annotate Source files: add acc_period/period for each source file. |
| 1. Annotate java source files, which have $JAVA_SRC_ROOT prefix. |
| 2. Annotate c++ source files. |
| """ |
| dest_dir = self.config['annotate_dest_dir'] |
| for key in self.file_periods.keys(): |
| is_java = False |
| if key.startswith('$JAVA_SRC_ROOT/'): |
| path = key[len('$JAVA_SRC_ROOT/'):] |
| items = path.split('/') |
| path = os.sep.join(items) |
| from_path = self._find_source_file(path) |
| to_path = os.path.join(dest_dir, 'java', path) |
| is_java = True |
| elif key.startswith('/') and os.path.isfile(key): |
| path = key |
| from_path = path |
| to_path = os.path.join(dest_dir, path[1:]) |
| else: |
| path = key[1:] if key.startswith('/') else key |
| # Change path on device to path on host |
| path = os.sep.join(path.split('/')) |
| from_path = self._find_source_file(path) |
| to_path = os.path.join(dest_dir, path) |
| if from_path is None: |
| log_warning("can't find source file for path %s" % key) |
| continue |
| self._annotate_file(from_path, to_path, self.file_periods[key], is_java) |
| |
| |
| def _annotate_file(self, from_path, to_path, file_period, is_java): |
| """Annotate a source file. |
| |
| Annotate a source file in three steps: |
| 1. In the first line, show periods of this file. |
| 2. For each function, show periods of this function. |
| 3. For each line not hitting the same line as functions, show |
| line periods. |
| """ |
| log_info('annotate file %s' % from_path) |
| with open(from_path, 'r') as rf: |
| lines = rf.readlines() |
| |
| annotates = dict() |
| for line in file_period.line_dict.keys(): |
| annotates[line] = self._get_percentage_str(file_period.line_dict[line], True) |
| for func_name in file_period.function_dict.keys(): |
| func_start_line, period = file_period.function_dict[func_name] |
| if func_start_line == -1: |
| continue |
| line = func_start_line - 1 if is_java else func_start_line |
| annotates[line] = '[func] ' + self._get_percentage_str(period, True) |
| annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True) |
| |
| max_annotate_cols = 0 |
| for key in annotates.keys(): |
| max_annotate_cols = max(max_annotate_cols, len(annotates[key])) |
| |
| empty_annotate = ' ' * (max_annotate_cols + 6) |
| |
| dirname = os.path.dirname(to_path) |
| if not os.path.isdir(dirname): |
| os.makedirs(dirname) |
| with open(to_path, 'w') as wf: |
| for line in range(1, len(lines) + 1): |
| annotate = annotates.get(line) |
| if annotate is None: |
| annotate = empty_annotate |
| else: |
| annotate = '/* ' + annotate + ( |
| ' ' * (max_annotate_cols - len(annotate))) + ' */' |
| wf.write(annotate) |
| wf.write(lines[line-1]) |
| |
| |
| if __name__ == '__main__': |
| parser = argparse.ArgumentParser( |
| description='Annotate based on perf.data. See configurations in annotate.config.') |
| parser.add_argument('--config', default='annotate.config', |
| help='Set configuration file. Default is annotate.config.') |
| args = parser.parse_args() |
| config = load_config(args.config) |
| annotator = SourceFileAnnotator(config) |
| annotator.annotate() |
