| #!/usr/bin/env vpython3 |
| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from __future__ import absolute_import |
| from __future__ import print_function |
| import argparse |
| import gzip |
| import json |
| import os |
| import shutil |
| import six |
| from six.moves import zip |
| |
| _OUTPUT_DIR = 'output' |
| _OUTPUT_GRAPH_DIR = os.path.join(_OUTPUT_DIR, 'graph') |
| |
| |
| class Process(object): |
| |
| def __init__(self): |
| self.pid = None |
| self.name = None |
| self.labels = None |
| self.types = {} |
| self.strings = {} |
| self.stackframes = {} |
| self.allocators = None |
| self.version = None |
| |
| |
| class Entry(object): |
| |
| def __init__(self): |
| self.count = None |
| self.size = None |
| self.type = None |
| self.stackframe = None |
| |
| |
| class GraphDump(object): |
| |
| def __init__(self): |
| self.pid = None |
| self.name = None |
| self.labels = None |
| self.heap = None |
| self.root = '' |
| self.leaks = '' |
| self.leak_stackframes = 0 |
| self.leak_objects = 0 |
| |
| def OpenTraceFile(file_path, mode): |
| if file_path.endswith('.gz'): |
| return gzip.open(file_path, mode + 'b') |
| return open(file_path, mode + 't') |
| |
| def FindMemoryDumps(filename): |
| processes = {} |
| |
| with OpenTraceFile(filename, 'r') as f: |
| data = json.loads(f.read()) |
| |
| for event in data['traceEvents']: |
| pid = event['pid'] |
| if pid not in processes: |
| processes[pid] = Process() |
| processes[pid].pid = pid |
| process = processes[pid] |
| |
| # Retrieve process informations. |
| if event['ph'] == 'M': |
| if event['name'] == 'process_name' and 'name' in event['args']: |
| process.name = event['args']['name'] |
| if event['name'] == 'process_labels' and 'labels' in event['args']: |
| process.labels = event['args']['labels'] |
| |
| if event['name'] == 'typeNames': |
| process.types = {} |
| for type_id, t in six.iteritems(event['args']['typeNames']): |
| process.types[int(type_id)] = t |
| |
| if event['name'] == 'stackFrames': |
| process.stackframes = {} |
| for stack_id, s in six.iteritems(event['args']['stackFrames']): |
| new_stackframe = {} |
| new_stackframe['name'] = s['name'] |
| if 'parent' in s: |
| new_stackframe['parent'] = int(s['parent']) |
| process.stackframes[int(stack_id)] = new_stackframe |
| |
| # Look for a detailed memory dump event. |
| if not ((event['name'] == 'periodic_interval' or |
| event['name'] == 'explicitly_triggered') and |
| event['args']['dumps']['level_of_detail'] == 'detailed'): |
| continue |
| |
| # Check for a memory dump V1. |
| if u'heaps' in event['args']['dumps']: |
| # Get the first memory dump. |
| if not process.allocators: |
| process.version = 1 |
| process.allocators = event['args']['dumps']['heaps'] |
| |
| # Check for a memory dump V2. |
| # See format: [chromium] src/base/trace_event/heap_profiler_event_writer.h |
| if u'heaps_v2' in event['args']['dumps']: |
| # Memory dump format V2 is dumping information incrementally. Update |
| # the cumulated indexes. |
| maps = event['args']['dumps']['heaps_v2']['maps'] |
| for string in maps['strings']: |
| process.strings[string['id']] = string['string'] |
| |
| for node in maps['nodes']: |
| node_v1 = {} |
| node_v1['name'] = process.strings[node['name_sid']] |
| if 'parent' in node: |
| node_v1['parent'] = node['parent'] |
| process.stackframes[node['id']] = node_v1 |
| |
| for t in maps['types']: |
| process.types[t['id']] = process.strings[t['name_sid']] |
| |
| # Get the first memory dump. |
| if not process.allocators: |
| dump = event['args']['dumps'] |
| process.version = 2 |
| process.allocators = dump['heaps_v2']['allocators'] |
| |
| # Remove processes with incomplete memory dump. |
| # Note: Calling list() otherwise we can't modify list while iterating. |
| for pid, process in list(processes.items()): |
| if not (process.allocators and process.stackframes and process.types): |
| del processes[pid] |
| |
| return processes |
| |
| |
| def ResolveMemoryDumpFields(entries, stackframes, types): |
| def ResolveStackTrace(stack_id, stackframes): |
| stackframe = stackframes[stack_id] |
| tail = () |
| if 'parent' in stackframe: |
| tail = ResolveStackTrace(stackframe['parent'], stackframes) |
| name = stackframe['name'].replace('\r', '').replace('\n', '') |
| return (name,) + tail |
| |
| def ResolveType(type_id, types): |
| return types[type_id] |
| |
| for entry in entries: |
| # Stackframe may be -1 (18446744073709551615L) when not stackframe are |
| # available. |
| if entry.stackframe not in stackframes: |
| entry.stackframe = [] |
| else: |
| entry.stackframe = ResolveStackTrace(entry.stackframe, stackframes) |
| entry.type = ResolveType(entry.type, types) |
| |
| |
| def IncrementHeapEntry(stack, count, size, typename, root): |
| if not stack: |
| root['count'] += count |
| root['size'] += size |
| if typename not in root['count_by_type']: |
| root['count_by_type'][typename] = 0 |
| root['count_by_type'][typename] += count |
| else: |
| top = stack[-1] |
| tail = stack[:-1] |
| |
| if top not in root['children']: |
| new_node = {} |
| new_node['count'] = 0 |
| new_node['size'] = 0 |
| new_node['children'] = {} |
| new_node['count_by_type'] = {} |
| root['children'][top] = new_node |
| |
| IncrementHeapEntry(tail, count, size, typename, root['children'][top]) |
| |
| |
| def CanonicalHeapEntries(root): |
| total_count = 0 |
| total_size = 0 |
| for child in six.itervalues(root['children']): |
| total_count += child['count'] |
| total_size += child['size'] |
| root['count'] -= total_count |
| root['size'] -= total_size |
| |
| for typename in root['count_by_type']: |
| total_count_for_type = 0 |
| for child in six.itervalues(root['children']): |
| if typename in child['count_by_type']: |
| total_count_for_type += child['count_by_type'][typename] |
| root['count_by_type'][typename] -= total_count_for_type |
| |
| for child in six.itervalues(root['children']): |
| CanonicalHeapEntries(child) |
| |
| |
| def FindLeaks(root, stack, leaks, threshold, size_threshold): |
| for frame in root['children']: |
| FindLeaks(root['children'][frame], [frame] + stack, leaks, threshold, |
| size_threshold) |
| |
| if root['count'] > threshold and root['size'] > size_threshold: |
| leaks.append({'count': root['count'], |
| 'size': root['size'], |
| 'count_by_type': root['count_by_type'], |
| 'stackframes': stack}) |
| |
| def DumpTree(root, frame, output, threshold, size_threshold): |
| output.write('\n{ \"name\": \"%s\",' % frame) |
| if root['count'] > threshold and root['count'] > size_threshold: |
| output.write(' \"size\": \"%s\",' % root['size']) |
| output.write(' \"count\": \"%s\",' % root['count']) |
| output.write(' \"children\": [') |
| is_first = True |
| for child_frame, child in root['children'].items(): |
| if is_first: |
| is_first = False |
| else: |
| output.write(',') |
| |
| DumpTree(child, child_frame, output, threshold, size_threshold) |
| output.write(']') |
| output.write('}') |
| |
| |
| def GetEntries(heap, process): |
| """ |
| Returns all entries in a heap, after filtering out unknown entries, and doing |
| some post processing to extract the relevant fields. |
| """ |
| if not process: |
| return [] |
| |
| entries = [] |
| if process.version == 1: |
| for raw_entry in process.allocators[heap]['entries']: |
| # Cumulative sizes and types are skipped. see: |
| # https://chromium.googlesource.com/chromium/src/+/a990af190304be5bf38b120799c594df5a293518/base/trace_event/heap_profiler_heap_dump_writer.cc#294 |
| if 'type' not in raw_entry or not raw_entry['bt']: |
| continue |
| |
| entry = Entry() |
| entry.count = int(raw_entry['count'], 16) |
| entry.size = int(raw_entry['size'], 16) |
| entry.type = int(raw_entry['type']) |
| entry.stackframe = int(raw_entry['bt']) |
| entries.append(entry) |
| |
| elif process.version == 2: |
| raw_entries = list(zip(process.allocators[heap]['counts'], |
| process.allocators[heap]['sizes'], |
| process.allocators[heap]['types'], |
| process.allocators[heap]['nodes'])) |
| for (raw_count, raw_size, raw_type, raw_stackframe) in raw_entries: |
| entry = Entry() |
| entry.count = raw_count |
| entry.size = raw_size |
| entry.type = raw_type |
| entry.stackframe = raw_stackframe |
| entries.append(entry) |
| |
| # Resolve fields by looking into indexes |
| ResolveMemoryDumpFields(entries, process.stackframes, process.types) |
| |
| return entries |
| |
| |
| def FilterProcesses(processes, filter_by_name, filter_by_labels): |
| remaining_processes = {} |
| for pid, process in six.iteritems(processes): |
| if filter_by_name and process.name != filter_by_name: |
| continue |
| if (filter_by_labels and |
| (not process.labels or filter_by_labels not in process.labels)): |
| continue |
| remaining_processes[pid] = process |
| |
| return remaining_processes |
| |
| |
| def FindRelevantProcesses(start_trace, end_trace, |
| filter_by_name, |
| filter_by_labels, |
| match_by_labels): |
| # Retrieve the processes and the associated memory dump. |
| end_processes = FindMemoryDumps(end_trace) |
| end_processes = FilterProcesses(end_processes, filter_by_name, |
| filter_by_labels) |
| |
| start_processes = None |
| if start_trace: |
| start_processes = FindMemoryDumps(start_trace) |
| start_processes = FilterProcesses(start_processes, filter_by_name, |
| filter_by_labels) |
| |
| # Build a sequence of pair of processes to be compared. |
| processes = [] |
| if not start_processes: |
| # Only keep end-processes. |
| for _, end_process in six.iteritems(end_processes): |
| processes.append((None, end_process)) |
| elif match_by_labels: |
| # Processes are paired based on name/labels. |
| for _, end_process in six.iteritems(end_processes): |
| matching_start_process = None |
| for _, start_process in six.iteritems(start_processes): |
| if (start_process.name == end_process.name and |
| (start_process.name in ['Browser', 'GPU'] or |
| start_process.labels == end_process.labels)): |
| matching_start_process = start_process |
| |
| if matching_start_process: |
| processes.append((matching_start_process, end_process)) |
| else: |
| # Processes are paired based on their PID. |
| relevant_pids = set(end_processes.keys()) & set(start_processes.keys()) |
| for pid in relevant_pids: |
| start_process = start_processes[pid] |
| end_process = end_processes[pid] |
| processes.append((start_process, end_process)) |
| |
| return processes |
| |
| |
| def BuildGraphDumps(processes, threshold, size_threshold): |
| """ |
| Build graph for a sequence of pair of processes. |
| If start_process is None, counts objects in end_trace. |
| Otherwise, counts objects present in end_trace, but not in start_process. |
| """ |
| |
| graph_dumps = [] |
| |
| for (start_process, end_process) in processes: |
| pid = end_process.pid |
| name = end_process.name if end_process.name else '' |
| labels = end_process.labels if end_process.labels else '' |
| print('Process[%d] %s: %s' % (pid, name, labels)) |
| |
| for heap in end_process.allocators: |
| start_entries = GetEntries(heap, start_process) |
| end_entries = GetEntries(heap, end_process) |
| |
| graph = GraphDump() |
| graph.pid = pid |
| graph.name = name |
| graph.labels = labels |
| graph.heap = heap |
| graph_dumps.append(graph) |
| |
| # Do the math: diffing start and end memory dumps. |
| root = {} |
| root['count'] = 0 |
| root['size'] = 0 |
| root['children'] = {} |
| root['count_by_type'] = {} |
| |
| for entry in start_entries: |
| if entry.type: |
| IncrementHeapEntry(entry.stackframe, - entry.count, - entry.size, |
| entry.type, root) |
| for entry in end_entries: |
| if entry.type: |
| IncrementHeapEntry(entry.stackframe, entry.count, entry.size, |
| entry.type, root) |
| |
| CanonicalHeapEntries(root) |
| |
| graph.root = root |
| |
| # Find leaks |
| leaks = [] |
| FindLeaks(root, [], leaks, threshold, size_threshold) |
| leaks.sort(reverse=True, key=lambda k: k['size']) |
| |
| if leaks: |
| print(' %s: %d potential leaks found.' % (heap, len(leaks))) |
| graph.leaks = leaks |
| graph.leak_stackframes = len(leaks) |
| for leak in leaks: |
| graph.leak_objects += leak['count'] |
| |
| return graph_dumps |
| |
| |
| def WritePotentialLeaks(graph_dumps): |
| for graph in graph_dumps: |
| if graph.leaks: |
| filename = 'process_%d_%s-leaks.json' % (graph.pid, graph.heap) |
| output_filename = os.path.join(_OUTPUT_DIR, filename) |
| with open(output_filename, 'w') as output: |
| json.dump(graph.leaks, output) |
| |
| |
| def WriteGrahDumps(graph_dumps, threshold, size_threshold): |
| for graph in graph_dumps: |
| # Dump the remaining allocated objects tree. |
| filename = 'process_%d_%s-objects.json' % (graph.pid, graph.heap) |
| output_filename = os.path.join(_OUTPUT_GRAPH_DIR, filename) |
| if graph.root: |
| with open(output_filename, 'w') as output: |
| DumpTree(graph.root, '.', output, threshold, size_threshold) |
| graph.root = filename |
| |
| |
| def WriteIndex(graph_dumps): |
| output_filename = os.path.join(_OUTPUT_GRAPH_DIR, 'index.json') |
| with open(output_filename, 'w') as output: |
| json.dump([ |
| {'pid': graph.pid, |
| 'heap': graph.heap, |
| 'name': graph.name, |
| 'labels': graph.labels, |
| 'objects': graph.root, |
| 'potential leaks': graph.leak_stackframes, |
| 'objects leaked': graph.leak_objects, |
| } |
| for graph in graph_dumps], output) |
| |
| |
| def WriteHTML(): |
| # Copy the HTML page. |
| source = os.path.join(os.path.dirname(os.path.abspath(__file__)), |
| 'diff_heap_profiler.html') |
| destination = os.path.join(_OUTPUT_GRAPH_DIR, 'index.html') |
| shutil.copyfile(source, destination) |
| |
| # Copy the D3 library file. |
| source = os.path.join(os.path.dirname(os.path.abspath(__file__)), |
| os.path.pardir, |
| os.path.pardir, |
| os.path.pardir, |
| 'tracing', |
| 'third_party', |
| 'd3', |
| 'd3.min.js') |
| destination = os.path.join(_OUTPUT_GRAPH_DIR, 'd3.min.js') |
| shutil.copyfile(source, destination) |
| |
| |
| def Main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| '--flame-graph', |
| action='store_true', |
| help='Output a flame graph based on stackframe allocations') |
| parser.add_argument( |
| '--threshold', |
| type=int, |
| default=0, |
| help='Objects threshold for being a potential memory leak') |
| parser.add_argument( |
| '--size-threshold', |
| type=int, |
| default=0, |
| help='Size threshold for being a potential memory leak') |
| parser.add_argument( |
| '--filter-by-name', |
| type=str, |
| help='Only keep processes with name (i.e. Browser, Renderer, ...)') |
| parser.add_argument( |
| '--filter-by-labels', |
| type=str, |
| help='Only keep processes with matching labels') |
| parser.add_argument( |
| '--match-by-labels', |
| action='store_true', |
| help='Match processes between runs by labels') |
| parser.add_argument( |
| 'trace', |
| nargs='+', |
| help='Trace files to be processed') |
| options = parser.parse_args() |
| |
| if options.threshold == 0 and options.size_threshold == 0: |
| options.threshold = 1000 |
| |
| if len(options.trace) == 1: |
| end_trace = options.trace[0] |
| start_trace = None |
| else: |
| start_trace = options.trace[0] |
| end_trace = options.trace[1] |
| |
| if not os.path.exists(_OUTPUT_DIR): |
| os.makedirs(_OUTPUT_DIR) |
| |
| # Find relevant processes to be processed. |
| processes = FindRelevantProcesses(start_trace, end_trace, |
| options.filter_by_name, |
| options.filter_by_labels, |
| options.match_by_labels) |
| |
| graph_dumps = BuildGraphDumps(processes, options.threshold, |
| options.size_threshold) |
| |
| WritePotentialLeaks(graph_dumps) |
| |
| if options.flame_graph: |
| if not os.path.exists(_OUTPUT_GRAPH_DIR): |
| os.makedirs(_OUTPUT_GRAPH_DIR) |
| WriteGrahDumps(graph_dumps, options.threshold, options.size_threshold) |
| WriteIndex(graph_dumps) |
| WriteHTML() |
| |
| if __name__ == '__main__': |
| Main() |
