| // Copyright 2017 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "chrome/profiling/json_exporter.h" |
| |
| #include <map> |
| |
| #include "base/containers/adapters.h" |
| #include "base/format_macros.h" |
| #include "base/json/json_writer.h" |
| #include "base/json/string_escape.h" |
| #include "base/macros.h" |
| #include "base/strings/string_number_conversions.h" |
| #include "base/strings/stringprintf.h" |
| #include "services/resource_coordinator/public/cpp/memory_instrumentation/tracing_observer.h" |
| |
| namespace profiling { |
| |
| namespace { |
| |
| // Maps strings to integers for the JSON string table. |
| using StringTable = std::map<std::string, size_t>; |
| |
| constexpr uint32_t kAllocatorCount = |
| static_cast<uint32_t>(AllocatorType::kCount); |
| |
| struct BacktraceNode { |
| BacktraceNode(size_t string_id, size_t parent) |
| : string_id_(string_id), parent_(parent) {} |
| |
| static constexpr size_t kNoParent = static_cast<size_t>(-1); |
| |
| size_t string_id() const { return string_id_; } |
| size_t parent() const { return parent_; } |
| |
| bool operator<(const BacktraceNode& other) const { |
| return std::tie(string_id_, parent_) < |
| std::tie(other.string_id_, other.parent_); |
| } |
| |
| private: |
| const size_t string_id_; |
| const size_t parent_; // kNoParent indicates no parent. |
| }; |
| |
| using BacktraceTable = std::map<BacktraceNode, size_t>; |
| |
| // Used as a temporary map key to uniquify an allocation with a given context |
| // and stack. A lock is held when dumping bactraces that guarantees that no |
| // Backtraces will be created or destroyed during the lifetime of that |
| // structure. Therefore it's safe to use a raw pointer Since backtraces are |
| // uniquified, this does pointer comparisons on the backtrace to give a stable |
| // ordering, even if that ordering has no intrinsic meaning. |
| struct UniqueAlloc { |
| public: |
| UniqueAlloc(const Backtrace* bt, int ctx_id) |
| : backtrace(bt), context_id(ctx_id) {} |
| |
| bool operator<(const UniqueAlloc& other) const { |
| return std::tie(backtrace, context_id) < |
| std::tie(other.backtrace, other.context_id); |
| } |
| |
| const Backtrace* backtrace = nullptr; |
| const int context_id = 0; |
| }; |
| |
| struct UniqueAllocMetrics { |
| size_t size = 0; |
| size_t count = 0; |
| }; |
| |
| using UniqueAllocationMap = std::map<UniqueAlloc, UniqueAllocMetrics>; |
| |
| // The hardcoded ID for having no context for an allocation. |
| constexpr int kUnknownTypeId = 0; |
| |
| const char* StringForAllocatorType(uint32_t type) { |
| switch (static_cast<AllocatorType>(type)) { |
| case AllocatorType::kMalloc: |
| return "malloc"; |
| case AllocatorType::kPartitionAlloc: |
| return "partition_alloc"; |
| case AllocatorType::kOilpan: |
| return "blink_gc"; |
| default: |
| NOTREACHED(); |
| return "unknown"; |
| } |
| } |
| |
| // Writes a dummy process name entry given a PID. When we have more information |
| // on a process it can be filled in here. But for now the tracing tools expect |
| // this entry since everything is associated with a PID. |
| void WriteProcessName(int pid, std::ostream& out) { |
| out << "{ \"pid\":" << pid << ", \"ph\":\"M\", \"name\":\"process_name\", " |
| << "\"args\":{\"name\":\"Browser\"}},"; |
| |
| // Catapult needs a thread named "CrBrowserMain" to recognize Chrome browser. |
| out << "{ \"pid\":" << pid << ", \"ph\":\"M\", \"name\":\"thread_name\", " |
| << "\"tid\": 1," |
| << "\"args\":{\"name\":\"CrBrowserMain\"}},"; |
| |
| // At least, one event must be present on the thread to avoid being pruned. |
| out << "{ \"name\": \"MemlogTraceEvent\", \"cat\": \"memlog\", " |
| << "\"ph\": \"B\", \"ts\": 1, \"pid\": " << pid << ", " |
| << "\"tid\": 1, \"args\": {}}"; |
| } |
| |
| // Writes the top-level allocators section. This section is used by the tracing |
| // UI to show a small summary for each allocator. It's necessary as a |
| // placeholder to allow the stack-viewing UI to be shown. |
| // |
| // Each array should be the number of allocators long. |
| void WriteAllocatorsSummary(size_t total_size[], |
| size_t total_count[], |
| std::ostream& out) { |
| out << "\"allocators\":{\n"; |
| for (uint32_t i = 0; i < kAllocatorCount; i++) { |
| const char* alloc_type = StringForAllocatorType(i); |
| |
| // Overall sizes. |
| static constexpr char kAttrsSizeBody[] = R"( |
| "%s": { |
| "attrs": { |
| "virtual_size": { |
| "type": "scalar", |
| "units": "bytes", |
| "value": "%zx" |
| }, |
| "size": { |
| "type": "scalar", |
| "units": "bytes", |
| "value": "%zx" |
| } |
| } |
| },)"; |
| out << base::StringPrintf(kAttrsSizeBody, alloc_type, total_size[i], |
| total_size[i]); |
| |
| // Allocated objects. |
| static constexpr char kAttrsObjectsBody[] = R"( |
| "%s/allocated_objects": { |
| "attrs": { |
| "shim_allocated_objects_count": { |
| "type": "scalar", |
| "units": "objects", |
| "value": "%zx" |
| }, |
| "shim_allocated_objects_size": { |
| "type": "scalar", |
| "units": "bytes", |
| "value": "%zx" |
| } |
| } |
| })"; |
| out << base::StringPrintf(kAttrsObjectsBody, alloc_type, total_count[i], |
| total_size[i]); |
| |
| // Comma except for the last time. |
| if (i < kAllocatorCount - 1) |
| out << ','; |
| out << "\n"; |
| } |
| out << "},\n"; |
| } |
| |
| // Writes the dictionary keys to preceed a "dumps" trace argument. |
| void WriteDumpsHeader(int pid, std::ostream& out) { |
| out << "{ \"pid\":" << pid << ","; |
| out << "\"ph\":\"v\","; |
| out << "\"name\":\"periodic_interval\","; |
| out << "\"ts\": 1,"; |
| out << "\"id\": \"1\","; |
| out << "\"args\":{"; |
| out << "\"dumps\":"; |
| } |
| |
| void WriteDumpsFooter(std::ostream& out) { |
| out << "}}"; // args, event |
| } |
| |
| // Writes the dictionary keys to preceed a "heaps_v2" trace argument inside a |
| // "dumps". This is "v2" heap dump format. |
| void WriteHeapsV2Header(std::ostream& out) { |
| out << "\"heaps_v2\": {\n"; |
| } |
| |
| // Closes the dictionaries from the WriteHeapsV2Header function above. |
| void WriteHeapsV2Footer(std::ostream& out) { |
| out << "}"; // heaps_v2 |
| } |
| |
| void WriteMemoryMaps( |
| const std::vector<memory_instrumentation::mojom::VmRegionPtr>& maps, |
| std::ostream& out) { |
| base::trace_event::TracedValue traced_value; |
| memory_instrumentation::TracingObserver::MemoryMapsAsValueInto(maps, |
| &traced_value); |
| out << "\"process_mmaps\":" << traced_value.ToString(); |
| } |
| |
| // Inserts or retrieves the ID for a string in the string table. |
| size_t AddOrGetString(std::string str, StringTable* string_table) { |
| auto result = string_table->emplace(std::move(str), string_table->size()); |
| // "result.first" is an iterator into the map. |
| return result.first->second; |
| } |
| |
| // Processes the context information needed for the give set of allocations. |
| // Strings are added for each referenced context and a mapping between |
| // context IDs and string IDs is filled in for each. |
| void FillContextStrings(const UniqueAllocationMap& allocations, |
| const std::map<std::string, int>& context_map, |
| StringTable* string_table, |
| std::map<int, size_t>* context_to_string_map) { |
| std::set<int> used_context; |
| for (const auto& alloc : allocations) |
| used_context.insert(alloc.first.context_id); |
| |
| if (used_context.find(kUnknownTypeId) != used_context.end()) { |
| // Hard code a string for the unknown context type. |
| context_to_string_map->emplace(kUnknownTypeId, |
| AddOrGetString("[unknown]", string_table)); |
| } |
| |
| // The context map is backwards from what we need, so iterate through the |
| // whole thing and see which ones are used. |
| for (const auto& context : context_map) { |
| if (used_context.find(context.second) != used_context.end()) { |
| size_t string_id = AddOrGetString(context.first, string_table); |
| context_to_string_map->emplace(context.second, string_id); |
| } |
| } |
| } |
| |
| size_t AddOrGetBacktraceNode(BacktraceNode node, |
| BacktraceTable* backtrace_table) { |
| auto result = |
| backtrace_table->emplace(std::move(node), backtrace_table->size()); |
| // "result.first" is an iterator into the map. |
| return result.first->second; |
| } |
| |
| // Returns the index into nodes of the node to reference for this stack. That |
| // node will reference its parent node, etc. to allow the full stack to |
| // be represented. |
| size_t AppendBacktraceStrings(const Backtrace& backtrace, |
| BacktraceTable* backtrace_table, |
| StringTable* string_table) { |
| int parent = -1; |
| // Addresses must be outputted in reverse order. |
| for (const Address& addr : base::Reversed(backtrace.addrs())) { |
| static constexpr char kPcPrefix[] = "pc:"; |
| // std::numeric_limits<>::digits gives the number of bits in the value. |
| // Dividing by 4 gives the number of hex digits needed to store the value. |
| // Adding to sizeof(kPcPrefix) yields the buffer size needed including the |
| // null terminator. |
| static constexpr int kBufSize = |
| sizeof(kPcPrefix) + |
| (std::numeric_limits<decltype(addr.value)>::digits / 4); |
| char buf[kBufSize]; |
| snprintf(buf, kBufSize, "%s%" PRIx64, kPcPrefix, addr.value); |
| size_t sid = AddOrGetString(buf, string_table); |
| parent = AddOrGetBacktraceNode(BacktraceNode(sid, parent), backtrace_table); |
| } |
| return parent; // Last item is the end of this stack. |
| } |
| |
| // Writes the string table which looks like: |
| // "strings":[ |
| // {"id":123,string:"This is the string"}, |
| // ... |
| // ] |
| void WriteStrings(const StringTable& string_table, std::ostream& out) { |
| out << "\"strings\":["; |
| bool first_time = true; |
| for (const auto& string_pair : string_table) { |
| if (!first_time) |
| out << ",\n"; |
| else |
| first_time = false; |
| |
| out << "{\"id\":" << string_pair.second; |
| // TODO(brettw) when we have real symbols this will need escaping. |
| out << ",\"string\":\"" << string_pair.first << "\"}"; |
| } |
| out << "]"; |
| } |
| |
| // Writes the nodes array in the maps section. These are all the backtrace |
| // entries and are indexed by the allocator nodes array. |
| // "nodes":[ |
| // {"id":1, "name_sid":123, "parent":17}, |
| // ... |
| // ] |
| void WriteMapNodes(const BacktraceTable& nodes, std::ostream& out) { |
| out << "\"nodes\":["; |
| |
| bool first_time = true; |
| for (const auto& node : nodes) { |
| if (!first_time) |
| out << ",\n"; |
| else |
| first_time = false; |
| |
| out << "{\"id\":" << node.second; |
| out << ",\"name_sid\":" << node.first.string_id(); |
| if (node.first.parent() != BacktraceNode::kNoParent) |
| out << ",\"parent\":" << node.first.parent(); |
| out << "}"; |
| } |
| out << "]"; |
| } |
| |
| // Write the types array in the maps section. These are the context for each |
| // allocation and just maps IDs to string IDs. |
| // "types":[ |
| // {"id":1, "name_sid":123}, |
| // ] |
| void WriteTypeNodes(const std::map<int, size_t>& type_to_string, |
| std::ostream& out) { |
| out << "\"types\":["; |
| |
| bool first_time = true; |
| for (const auto& type : type_to_string) { |
| if (!first_time) |
| out << ",\n"; |
| else |
| first_time = false; |
| |
| out << "{\"id\":" << type.first << ",\"name_sid\":" << type.second << "}"; |
| } |
| out << "]"; |
| } |
| |
| // Writes the number of matching allocations array which looks like: |
| // "counts":[1, 1, 2] |
| void WriteCounts(const UniqueAllocationMap& allocations, std::ostream& out) { |
| out << "\"counts\":["; |
| bool first_time = true; |
| for (const auto& cur : allocations) { |
| if (!first_time) |
| out << ","; |
| else |
| first_time = false; |
| out << cur.second.count; |
| } |
| out << "]"; |
| } |
| |
| // Writes the total sizes of each allocation which looks like: |
| // "sizes":[32, 64, 12] |
| void WriteSizes(const UniqueAllocationMap& allocations, std::ostream& out) { |
| out << "\"sizes\":["; |
| bool first_time = true; |
| for (const auto& cur : allocations) { |
| if (!first_time) |
| out << ","; |
| else |
| first_time = false; |
| out << cur.second.size; |
| } |
| out << "]"; |
| } |
| |
| // Writes the types array of integers which looks like: |
| // "types":[0, 0, 1] |
| void WriteTypes(const UniqueAllocationMap& allocations, std::ostream& out) { |
| out << "\"types\":["; |
| bool first_time = true; |
| for (const auto& cur : allocations) { |
| if (!first_time) |
| out << ","; |
| else |
| first_time = false; |
| out << cur.first.context_id; |
| } |
| out << "]"; |
| } |
| |
| // Writes the nodes array which indexes for each allocation into the maps nodes |
| // array written above. It looks like: |
| // "nodes":[1, 5, 10] |
| void WriteAllocatorNodes(const UniqueAllocationMap& allocations, |
| const std::map<const Backtrace*, size_t>& backtraces, |
| std::ostream& out) { |
| out << "\"nodes\":["; |
| bool first_time = true; |
| for (const auto& cur : allocations) { |
| if (!first_time) |
| out << ","; |
| else |
| first_time = false; |
| auto found = backtraces.find(cur.first.backtrace); |
| out << found->second; |
| } |
| out << "]"; |
| } |
| |
| } // namespace |
| |
| ExportParams::ExportParams() = default; |
| ExportParams::~ExportParams() = default; |
| |
| void ExportAllocationEventSetToJSON( |
| int pid, |
| const ExportParams& params, |
| std::unique_ptr<base::DictionaryValue> metadata_dict, |
| std::ostream& out) { |
| out << "{ \"traceEvents\": ["; |
| WriteProcessName(pid, out); |
| out << ",\n"; |
| WriteDumpsHeader(pid, out); |
| ExportMemoryMapsAndV2StackTraceToJSON(params, out); |
| WriteDumpsFooter(out); |
| out << "]"; |
| |
| // Append metadata. |
| if (metadata_dict) { |
| std::string metadata; |
| base::JSONWriter::Write(*metadata_dict, &metadata); |
| out << ",\"metadata\": " << metadata; |
| } |
| |
| out << "}\n"; |
| } |
| |
| void ExportMemoryMapsAndV2StackTraceToJSON(const ExportParams& params, |
| std::ostream& out) { |
| // Start dictionary. |
| out << "{\n"; |
| |
| WriteMemoryMaps(params.maps, out); |
| out << ",\n"; |
| |
| // Write level of detail. |
| out << R"("level_of_detail": "detailed")" |
| << ",\n"; |
| |
| // Aggregate stats for each allocator type. |
| size_t total_size[kAllocatorCount] = {0}; |
| size_t total_count[kAllocatorCount] = {0}; |
| UniqueAllocationMap filtered_allocations[kAllocatorCount]; |
| for (const auto& alloc_pair : params.allocs) { |
| uint32_t allocator_index = |
| static_cast<uint32_t>(alloc_pair.first.allocator()); |
| size_t alloc_count = alloc_pair.second; |
| size_t alloc_size = alloc_pair.first.size(); |
| |
| size_t alloc_total_size = alloc_size * alloc_count; |
| total_size[allocator_index] += alloc_total_size; |
| total_count[allocator_index] += alloc_count; |
| |
| UniqueAlloc unique_alloc(alloc_pair.first.backtrace(), |
| alloc_pair.first.context_id()); |
| UniqueAllocMetrics& unique_alloc_metrics = |
| filtered_allocations[allocator_index][unique_alloc]; |
| unique_alloc_metrics.size += alloc_total_size; |
| unique_alloc_metrics.count += alloc_count; |
| } |
| |
| // Filter irrelevant allocations. |
| // TODO(crbug:763595): Leave placeholders for pruned allocations. |
| for (uint32_t i = 0; i < kAllocatorCount; i++) { |
| for (auto alloc = filtered_allocations[i].begin(); |
| alloc != filtered_allocations[i].end();) { |
| if (alloc->second.size < params.min_size_threshold && |
| alloc->second.count < params.min_count_threshold) { |
| alloc = filtered_allocations[i].erase(alloc); |
| } else { |
| ++alloc; |
| } |
| } |
| } |
| |
| WriteAllocatorsSummary(total_size, total_count, out); |
| WriteHeapsV2Header(out); |
| |
| // Output Heaps_V2 format version. Currently "1" is the only valid value. |
| out << "\"version\": 1,\n"; |
| |
| StringTable string_table; |
| |
| // Put all required context strings in the string table and generate a |
| // mapping from allocation context_id to string ID. |
| std::map<int, size_t> context_to_string_map; |
| for (uint32_t i = 0; i < kAllocatorCount; i++) { |
| FillContextStrings(filtered_allocations[i], params.context_map, |
| &string_table, &context_to_string_map); |
| } |
| |
| // Find all backtraces referenced by the set and not filtered. The backtrace |
| // storage will contain more stacks than we want to write out (it will refer |
| // to all processes, while we're only writing one). So do those only on |
| // demand. |
| // |
| // The map maps backtrace keys to node IDs (computed below). |
| std::map<const Backtrace*, size_t> backtraces; |
| for (size_t i = 0; i < kAllocatorCount; i++) { |
| for (const auto& alloc : filtered_allocations[i]) |
| backtraces.emplace(alloc.first.backtrace, 0); |
| } |
| |
| // Write each backtrace, converting the string for the stack entry to string |
| // IDs. The backtrace -> node ID will be filled in at this time. |
| BacktraceTable nodes; |
| VLOG(1) << "Number of backtraces " << backtraces.size(); |
| for (auto& bt : backtraces) |
| bt.second = AppendBacktraceStrings(*bt.first, &nodes, &string_table); |
| |
| // Maps section. |
| out << "\"maps\": {\n"; |
| WriteStrings(string_table, out); |
| out << ",\n"; |
| WriteMapNodes(nodes, out); |
| out << ",\n"; |
| WriteTypeNodes(context_to_string_map, out); |
| out << "},\n"; // End of maps section. |
| |
| // Allocators section. |
| out << "\"allocators\":{\n"; |
| for (uint32_t i = 0; i < kAllocatorCount; i++) { |
| out << " \"" << StringForAllocatorType(i) << "\":{\n "; |
| |
| WriteCounts(filtered_allocations[i], out); |
| out << ",\n "; |
| WriteSizes(filtered_allocations[i], out); |
| out << ",\n "; |
| WriteTypes(filtered_allocations[i], out); |
| out << ",\n "; |
| WriteAllocatorNodes(filtered_allocations[i], backtraces, out); |
| out << "\n }"; |
| |
| // Comma every time but the last. |
| if (i < kAllocatorCount - 1) |
| out << ','; |
| out << "\n"; |
| } |
| out << "}\n"; // End of allocators section. |
| |
| WriteHeapsV2Footer(out); |
| |
| // End dictionary. |
| out << "}\n"; |
| } |
| |
| } // namespace profiling |