Android: Log native code's resident pages.
Log native code's resident pages.
Use the data for visualization and analysis. This work will help in
assessing code ordering and will give insight on how to improve it.
Bug: 933377
Change-Id: If93f7c07f74111e5a5bb50567453cc9b202219df
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1482976
Commit-Queue: Monica Salama <msalama@google.com>
Reviewed-by: Hector Dearman <hjd@chromium.org>
Reviewed-by: Benoit L <lizeb@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#638583}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: f099bbfcbac681b397641afc0ae42881e23ee97ediff --git a/native_lib_memory/extract_resident_pages.py b/native_lib_memory/extract_resident_pages.py
new file mode 100755
index 0000000..ce04733
--- /dev/null
+++ b/native_lib_memory/extract_resident_pages.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python
+#
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Generates Json file for native library's resident pages."""
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+
+_SRC_PATH = os.path.abspath(os.path.join(
+ os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
+sys.path.insert(0, os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
+from devil.android import device_utils
+from devil.android import device_errors
+
+
+def _CreateArgumentParser():
+ parser = argparse.ArgumentParser(
+ description='Create JSON file for residency pages')
+ parser.add_argument('--device-serial', type=str, required=True)
+ parser.add_argument('--on-device-file-path', type=str,
+ help='Path to residency.txt', required=True)
+ parser.add_argument('--output-directory', type=str, help='Output directory',
+ required=False)
+ return parser
+
+
+def _ReadFileFromDevice(device_serial, file_path):
+ """Reads the file from the device, and returns its content.
+
+ Args:
+ device_serial: (str) Device identifier
+ file_path: (str) On-device path to the residency file.
+
+ Returns:
+ (str or None) The file content.
+ """
+ content = None
+ try:
+ device = device_utils.DeviceUtils(device_serial)
+ content = device.ReadFile(file_path, True)
+ except device_errors.CommandFailedError:
+ logging.exception(
+ 'Possible failure reaching the device or reading the file')
+
+ return content
+
+
+def ParseResidentPages(resident_pages):
+ """Parses and converts the residency data into a list where
+ the index corresponds to the page number and the value 1 if resident
+ and 0 otherwise.
+
+ |resident_pages| contains a string of resident pages:
+ 0
+ 1
+ ...
+ ...
+ N
+
+ Args:
+ resident_pages: (str) As returned by ReadFileFromDevice()
+
+ Returns:
+ (list) Pages list.
+ """
+ pages_list = []
+ expected = 0
+ for page in resident_pages.splitlines():
+ while expected < int(page):
+ pages_list.append(0)
+ expected += 1
+
+ pages_list.append(1)
+ expected += 1;
+ return pages_list
+
+
+def _GetResidentPagesJSON(pages_list):
+ """Transforms the pages list to JSON object.
+
+ Args:
+ pages_list: (list) As returned by ParseResidentPages()
+
+ Returns:
+ (JSON object) Pages JSON object.
+ """
+ json_data = []
+ for i in range(len(pages_list)):
+ json_data.append({'page_num': i, 'resident': pages_list[i]})
+ return json_data
+
+
+def _WriteJSONToFile(json_data, output_file_path):
+ """Dumps JSON data to file.
+
+ Args:
+ json_data: (JSON object) Data to be dumped in the file.
+ output_file_path: (str) Output file path
+ """
+ with open(output_file_path, 'w') as f:
+ json.dump(json_data, f)
+
+
+def main():
+ parser = _CreateArgumentParser()
+ args = parser.parse_args()
+ logging.basicConfig(level=logging.INFO)
+
+ content = _ReadFileFromDevice(args.device_serial,
+ args.on_device_file_path)
+ if not content:
+ logging.error('Error reading file from device')
+ return 1
+
+ pages_list = ParseResidentPages(content)
+ pages_json = _GetResidentPagesJSON(pages_list)
+ _WriteJSONToFile(pages_json, os.path.join(args.output_directory,
+ 'residency.json'))
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/native_lib_memory/extract_resident_pages_unittest.py b/native_lib_memory/extract_resident_pages_unittest.py
new file mode 100755
index 0000000..3c8a73a
--- /dev/null
+++ b/native_lib_memory/extract_resident_pages_unittest.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit test for extracting resident pages."""
+
+import random
+import unittest
+
+import extract_resident_pages
+
+class ExtractResidentPagesUnittest(unittest.TestCase):
+
+ def testParseResidentPages(self):
+ max_pages = 10600
+ pages = []
+ resident_pages = ""
+
+ for i in range(max_pages):
+ is_resident = random.randint(0,1)
+ pages.append(is_resident)
+ if is_resident:
+ resident_pages += str(i) + '\n'
+
+ pages_list = extract_resident_pages.ParseResidentPages(resident_pages)
+
+ for i in range(len(pages_list)):
+ self.assertEqual(pages[i], pages_list[i])
+
+ # As ParseResidentPages is only aware of the maximum page number that is
+ # resident, check that all others are not resident.
+ for i in range(len(pages_list), len(pages)):
+ self.assertFalse(pages[i])
+
+
+if __name__ == '__main__':
+
+ unittest.main()
diff --git a/native_lib_memory/extract_symbols.py b/native_lib_memory/extract_symbols.py
index 6051366..a0bb277 100755
--- a/native_lib_memory/extract_symbols.py
+++ b/native_lib_memory/extract_symbols.py
@@ -224,8 +224,8 @@
'by tools/cygprofile/process_profiles.py',
required=False)
parser.add_argument('--residency', type=str,
- help='Path to a JSON file with residency data, as written'
- ' by process_resdency.py', required=False)
+ help='Path to JSON file with residency pages, as written'
+ ' by extract_resident_pages.py', required=False)
parser.add_argument('--build-directory', type=str, help='Build directory',
required=True)
parser.add_argument('--output-directory', type=str, help='Output directory',
@@ -254,9 +254,12 @@
offset = 0
if args.residency:
- with open(args.residency) as f:
- residency = json.load(f)
- offset = residency['offset']
+ if not os.path.exists(args.residency):
+ logging.error('Residency file not found')
+ return 1
+ residency_path = os.path.join(args.output_directory, 'residency.json')
+ if residency_path != args.residency:
+ shutil.copy(args.residency, residency_path)
logging.info('Extracting symbols from %s', native_lib_filename)
native_lib_symbols = symbol_extractor.SymbolInfosFromBinary(
@@ -283,9 +286,6 @@
directory = os.path.dirname(__file__)
for filename in ['visualize.html', 'visualize.js', 'visualize.css']:
- if args.residency:
- shutil.copy(args.residency,
- os.path.join(args.output_directory, 'residency.json'))
shutil.copy(os.path.join(directory, filename),
os.path.join(args.output_directory, filename))
diff --git a/native_lib_memory/visualize.js b/native_lib_memory/visualize.js
index 67ea517..53a22e6 100644
--- a/native_lib_memory/visualize.js
+++ b/native_lib_memory/visualize.js
@@ -223,7 +223,10 @@
function createGraph(codePages, reachedPerPage, residency) {
const PAGE_SIZE = 4096;
+ // Offset is relative to the start of libmonochrome.so not to .text
+ // All offsets are aligned down to page size.
let offsets = codePages.map((x) => x.offset).sort((a, b) => a - b);
+ // |minOffset| is equal to start of text aligned down to page size.
let minOffset = +offsets[0];
let maxOffset = +offsets[offsets.length - 1] + PAGE_SIZE;
let startEndOfOrderedText = getStartAndEndOfOrderedText(codePages);
@@ -239,15 +242,20 @@
}
if (residency) {
- let timestamps = Object.keys(
- residency).map((x) => +x).sort((a, b) => a - b);
- let lastTimestamp = +timestamps[timestamps.length - 1];
- let residencyData = residency[lastTimestamp];
- // Other offsets are relative to start of the native library.
- let typedResidencyData = residencyData.map(
+ let typedResidencyData = residency.map(
(page) => (
- {"type": RESIDENCY, "data": {"offset": page.offset + minOffset,
+ {"type": RESIDENCY, "data": {"offset": page.page_num * PAGE_SIZE
+ + minOffset,
"resident": page.resident}}));
+ nextResidencyOffset
+ = (residency[residency.length - 1].page_num + 1) * PAGE_SIZE
+ + minOffset;
+
+ for(let i = nextResidencyOffset; i < maxOffset; i+=PAGE_SIZE) {
+ typedResidencyData.push(
+ {"type": RESIDENCY, "data": {"offset": i, "resident": 0}});
+ }
+
flatData = flatData.concat(typedResidencyData);
}
