| #!/usr/bin/env vpython3 |
| # Copyright 2011 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Reports binary size metrics for an APK. |
| |
| More information at //docs/speed/binary_size/metrics.md. |
| """ |
| |
| |
| import argparse |
| import collections |
| from contextlib import contextmanager |
| import json |
| import logging |
| import os |
| import posixpath |
| import re |
| import struct |
| import sys |
| import tempfile |
| import zipfile |
| import zlib |
| |
| import devil_chromium |
| from devil.android.sdk import build_tools |
| from devil.utils import cmd_helper |
| from devil.utils import lazy |
| import method_count |
| from pylib import constants |
| from pylib.constants import host_paths |
| |
| _AAPT_PATH = lazy.WeakConstant(lambda: build_tools.GetPath('aapt')) |
| _ANDROID_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', |
| 'android', 'gyp') |
| _BUILD_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', 'util') |
| _READOBJ_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party', |
| 'llvm-build', 'Release+Asserts', 'bin', |
| 'llvm-readobj') |
| |
| with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): |
| import perf_tests_results_helper # pylint: disable=import-error |
| |
| with host_paths.SysPath(host_paths.TRACING_PATH): |
| from tracing.value import convert_chart_json # pylint: disable=import-error |
| |
| with host_paths.SysPath(_ANDROID_UTILS_PATH, 0): |
| from util import build_utils # pylint: disable=import-error |
| |
| with host_paths.SysPath(_BUILD_UTILS_PATH, 0): |
| from lib.results import result_sink # pylint: disable=import-error |
| from lib.results import result_types # pylint: disable=import-error |
| |
| # Captures an entire config from aapt output. |
| _AAPT_CONFIG_PATTERN = r'config %s:(.*?)config [a-zA-Z-]+:' |
| # Matches string resource entries from aapt output. |
| _AAPT_ENTRY_RE = re.compile( |
| r'resource (?P<id>\w{10}) [\w\.]+:string/.*?"(?P<val>.+?)"', re.DOTALL) |
| _BASE_CHART = { |
| 'format_version': '0.1', |
| 'benchmark_name': 'resource_sizes', |
| 'benchmark_description': 'APK resource size information.', |
| 'trace_rerun_options': [], |
| 'charts': {} |
| } |
| # Macro definitions look like (something, 123) when |
| # enable_resource_allowlist_generation=true. |
| _RC_HEADER_RE = re.compile(r'^#define (?P<name>\w+).* (?P<id>\d+)\)?$') |
| _RE_NON_LANGUAGE_PAK = re.compile(r'^assets/.*(resources|percent)\.pak$') |
| _READELF_SIZES_METRICS = { |
| 'text': ['.text'], |
| 'data': ['.data', '.rodata', '.data.rel.ro', '.data.rel.ro.local'], |
| 'relocations': |
| ['.rel.dyn', '.rel.plt', '.rela.dyn', '.rela.plt', '.relr.dyn'], |
| 'unwind': [ |
| '.ARM.extab', '.ARM.exidx', '.eh_frame', '.eh_frame_hdr', |
| '.ARM.exidxsentinel_section_after_text' |
| ], |
| 'symbols': [ |
| '.dynsym', '.dynstr', '.dynamic', '.shstrtab', '.got', '.plt', '.iplt', |
| '.got.plt', '.hash', '.gnu.hash' |
| ], |
| 'other': [ |
| '.init_array', '.preinit_array', '.ctors', '.fini_array', '.comment', |
| '.note.gnu.gold-version', '.note.crashpad.info', '.note.android.ident', |
| '.ARM.attributes', '.note.gnu.build-id', '.gnu.version', |
| '.gnu.version_d', '.gnu.version_r', '.interp', '.gcc_except_table', |
| '.note.gnu.property' |
| ] |
| } |
| |
| |
| class _AccumulatingReporter: |
| def __init__(self): |
| self._combined_metrics = collections.defaultdict(int) |
| |
| def __call__(self, graph_title, trace_title, value, units): |
| self._combined_metrics[(graph_title, trace_title, units)] += value |
| |
| def DumpReports(self, report_func): |
| for (graph_title, trace_title, |
| units), value in sorted(self._combined_metrics.items()): |
| report_func(graph_title, trace_title, value, units) |
| |
| |
| class _ChartJsonReporter(_AccumulatingReporter): |
| def __init__(self, chartjson): |
| super().__init__() |
| self._chartjson = chartjson |
| self.trace_title_prefix = '' |
| |
| def __call__(self, graph_title, trace_title, value, units): |
| super().__call__(graph_title, trace_title, value, units) |
| |
| perf_tests_results_helper.ReportPerfResult( |
| self._chartjson, graph_title, self.trace_title_prefix + trace_title, |
| value, units) |
| |
| def SynthesizeTotals(self, unique_method_count): |
| for tup, value in sorted(self._combined_metrics.items()): |
| graph_title, trace_title, units = tup |
| if trace_title == 'unique methods': |
| value = unique_method_count |
| perf_tests_results_helper.ReportPerfResult(self._chartjson, graph_title, |
| 'Combined_' + trace_title, |
| value, units) |
| |
| |
| def _PercentageDifference(a, b): |
| if a == 0: |
| return 0 |
| return float(b - a) / a |
| |
| |
| def _ReadZipInfoExtraFieldLength(zip_file, zip_info): |
| """Reads the value of |extraLength| from |zip_info|'s local file header. |
| |
| |zip_info| has an |extra| field, but it's read from the central directory. |
| Android's zipalign tool sets the extra field only in local file headers. |
| """ |
| # Refer to https://en.wikipedia.org/wiki/Zip_(file_format)#File_headers |
| zip_file.fp.seek(zip_info.header_offset + 28) |
| return struct.unpack('<H', zip_file.fp.read(2))[0] |
| |
| |
| def _MeasureApkSignatureBlock(zip_file): |
| """Measures the size of the v2 / v3 signing block. |
| |
| Refer to: https://source.android.com/security/apksigning/v2 |
| """ |
| # Seek to "end of central directory" struct. |
| eocd_offset_from_end = -22 - len(zip_file.comment) |
| zip_file.fp.seek(eocd_offset_from_end, os.SEEK_END) |
| assert zip_file.fp.read(4) == b'PK\005\006', ( |
| 'failed to find end-of-central-directory') |
| |
| # Read out the "start of central directory" offset. |
| zip_file.fp.seek(eocd_offset_from_end + 16, os.SEEK_END) |
| start_of_central_directory = struct.unpack('<I', zip_file.fp.read(4))[0] |
| |
| # Compute the offset after the last zip entry. |
| last_info = max(zip_file.infolist(), key=lambda i: i.header_offset) |
| last_header_size = (30 + len(last_info.filename) + |
| _ReadZipInfoExtraFieldLength(zip_file, last_info)) |
| end_of_last_file = (last_info.header_offset + last_header_size + |
| last_info.compress_size) |
| return start_of_central_directory - end_of_last_file |
| |
| |
| def _RunReadobj(so_path, options): |
| return cmd_helper.GetCmdOutput([_READOBJ_PATH, '--elf-output-style=GNU'] + |
| options + [so_path]) |
| |
| |
| def _ExtractLibSectionSizesFromApk(apk_path, lib_path): |
| with Unzip(apk_path, filename=lib_path) as extracted_lib_path: |
| grouped_section_sizes = collections.defaultdict(int) |
| no_bits_section_sizes, section_sizes = _CreateSectionNameSizeMap( |
| extracted_lib_path) |
| for group_name, section_names in _READELF_SIZES_METRICS.items(): |
| for section_name in section_names: |
| if section_name in section_sizes: |
| grouped_section_sizes[group_name] += section_sizes.pop(section_name) |
| |
| # Consider all NOBITS sections as .bss. |
| grouped_section_sizes['bss'] = sum(no_bits_section_sizes.values()) |
| |
| # Group any unknown section headers into the "other" group. |
| for section_header, section_size in section_sizes.items(): |
| sys.stderr.write('Unknown elf section header: %s\n' % section_header) |
| grouped_section_sizes['other'] += section_size |
| |
| return grouped_section_sizes |
| |
| |
| def _CreateSectionNameSizeMap(so_path): |
| stdout = _RunReadobj(so_path, ['-S', '--wide']) |
| section_sizes = {} |
| no_bits_section_sizes = {} |
| # Matches [ 2] .hash HASH 00000000006681f0 0001f0 003154 04 A 3 0 8 |
| for match in re.finditer(r'\[[\s\d]+\] (\..*)$', stdout, re.MULTILINE): |
| items = match.group(1).split() |
| target = no_bits_section_sizes if items[1] == 'NOBITS' else section_sizes |
| target[items[0]] = int(items[4], 16) |
| |
| return no_bits_section_sizes, section_sizes |
| |
| |
| def _ParseManifestAttributes(apk_path): |
| # Check if the manifest specifies whether or not to extract native libs. |
| output = cmd_helper.GetCmdOutput([ |
| _AAPT_PATH.read(), 'd', 'xmltree', apk_path, 'AndroidManifest.xml']) |
| |
| def parse_attr(namespace, name): |
| # android:extractNativeLibs(0x010104ea)=(type 0x12)0x0 |
| # android:extractNativeLibs(0x010104ea)=(type 0x12)0xffffffff |
| # dist:onDemand=(type 0x12)0xffffffff |
| m = re.search( |
| f'(?:{namespace}:)?{name}' + r'(?:\(.*?\))?=\(type .*?\)(\w+)', output) |
| return m and int(m.group(1), 16) |
| |
| skip_extract_lib = bool(parse_attr('android', 'extractNativeLibs')) |
| sdk_version = parse_attr('android', 'minSdkVersion') |
| is_feature_split = parse_attr('android', 'isFeatureSplit') |
| # Can use <dist:on-demand>, or <module dist:onDemand="true">. |
| on_demand = parse_attr('dist', 'onDemand') or 'on-demand' in output |
| on_demand = bool(on_demand and is_feature_split) |
| |
| return sdk_version, skip_extract_lib, on_demand |
| |
| |
| def _NormalizeLanguagePaks(translations, factor): |
| english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak') |
| num_translations = translations.GetNumEntries() |
| ret = 0 |
| if english_pak: |
| ret -= translations.ComputeZippedSize() |
| ret += int(english_pak.compress_size * num_translations * factor) |
| return ret |
| |
| |
| def _NormalizeResourcesArsc(apk_path, num_arsc_files, num_translations, |
| out_dir): |
| """Estimates the expected overhead of untranslated strings in resources.arsc. |
| |
| See http://crbug.com/677966 for why this is necessary. |
| """ |
| # If there are multiple .arsc files, use the resource packaged APK instead. |
| if num_arsc_files > 1: |
| if not out_dir: |
| return -float('inf') |
| ap_name = os.path.basename(apk_path).replace('.apk', '.ap_') |
| ap_path = os.path.join(out_dir, 'arsc/apks', ap_name) |
| if not os.path.exists(ap_path): |
| raise Exception('Missing expected file: %s, try rebuilding.' % ap_path) |
| apk_path = ap_path |
| |
| aapt_output = _RunAaptDumpResources(apk_path) |
| # en-rUS is in the default config and may be cluttered with non-translatable |
| # strings, so en-rGB is a better baseline for finding missing translations. |
| en_strings = _CreateResourceIdValueMap(aapt_output, 'en-rGB') |
| fr_strings = _CreateResourceIdValueMap(aapt_output, 'fr') |
| |
| # en-US and en-GB will never be translated. |
| config_count = num_translations - 2 |
| |
| size = 0 |
| for res_id, string_val in en_strings.items(): |
| if string_val == fr_strings[res_id]: |
| string_size = len(string_val) |
| # 7 bytes is the per-entry overhead (not specific to any string). See |
| # https://android.googlesource.com/platform/frameworks/base.git/+/android-4.2.2_r1/tools/aapt/StringPool.cpp#414. |
| # The 1.5 factor was determined experimentally and is meant to account for |
| # other languages generally having longer strings than english. |
| size += config_count * (7 + string_size * 1.5) |
| |
| return int(size) |
| |
| |
| def _CreateResourceIdValueMap(aapt_output, lang): |
| """Return a map of resource ids to string values for the given |lang|.""" |
| config_re = _AAPT_CONFIG_PATTERN % lang |
| return {entry.group('id'): entry.group('val') |
| for config_section in re.finditer(config_re, aapt_output, re.DOTALL) |
| for entry in re.finditer(_AAPT_ENTRY_RE, config_section.group(0))} |
| |
| |
| def _RunAaptDumpResources(apk_path): |
| cmd = [_AAPT_PATH.read(), 'dump', '--values', 'resources', apk_path] |
| status, output = cmd_helper.GetCmdStatusAndOutput(cmd) |
| if status != 0: |
| raise Exception('Failed running aapt command: "%s" with output "%s".' % |
| (' '.join(cmd), output)) |
| return output |
| |
| |
| class _FileGroup: |
| """Represents a category that apk files can fall into.""" |
| |
| def __init__(self, name): |
| self.name = name |
| self._zip_infos = [] |
| self._extracted_multipliers = [] |
| |
| def AddZipInfo(self, zip_info, extracted_multiplier=0): |
| self._zip_infos.append(zip_info) |
| self._extracted_multipliers.append(extracted_multiplier) |
| |
| def AllEntries(self): |
| return iter(self._zip_infos) |
| |
| def GetNumEntries(self): |
| return len(self._zip_infos) |
| |
| def FindByPattern(self, pattern): |
| return next((i for i in self._zip_infos if re.match(pattern, i.filename)), |
| None) |
| |
| def FindLargest(self): |
| if not self._zip_infos: |
| return None |
| return max(self._zip_infos, key=lambda i: i.file_size) |
| |
| def ComputeZippedSize(self): |
| return sum(i.compress_size for i in self._zip_infos) |
| |
| def ComputeUncompressedSize(self): |
| return sum(i.file_size for i in self._zip_infos) |
| |
| def ComputeExtractedSize(self): |
| ret = 0 |
| for zi, multiplier in zip(self._zip_infos, self._extracted_multipliers): |
| ret += zi.file_size * multiplier |
| return ret |
| |
| def ComputeInstallSize(self): |
| return self.ComputeExtractedSize() + self.ComputeZippedSize() |
| |
| |
| def _AnalyzeInternal(apk_path, |
| sdk_version, |
| report_func, |
| dex_stats_collector, |
| out_dir, |
| apks_path=None, |
| split_name=None): |
| """Analyse APK to determine size contributions of different file classes. |
| |
| Returns: Normalized APK size. |
| """ |
| dex_stats_collector.CollectFromZip(split_name or '', apk_path) |
| file_groups = [] |
| |
| def make_group(name): |
| group = _FileGroup(name) |
| file_groups.append(group) |
| return group |
| |
| def has_no_extension(filename): |
| return os.path.splitext(filename)[1] == '' |
| |
| native_code = make_group('Native code') |
| java_code = make_group('Java code') |
| native_resources_no_translations = make_group('Native resources (no l10n)') |
| translations = make_group('Native resources (l10n)') |
| stored_translations = make_group('Native resources stored (l10n)') |
| icu_data = make_group('ICU (i18n library) data') |
| v8_snapshots = make_group('V8 Snapshots') |
| png_drawables = make_group('PNG drawables') |
| res_directory = make_group('Non-compiled Android resources') |
| arsc = make_group('Compiled Android resources') |
| metadata = make_group('Package metadata') |
| unknown = make_group('Unknown files') |
| notices = make_group('licenses.notice file') |
| unwind_cfi = make_group('unwind_cfi (dev and canary only)') |
| |
| with zipfile.ZipFile(apk_path, 'r') as apk: |
| apk_contents = apk.infolist() |
| # Account for zipalign overhead that exists in local file header. |
| zipalign_overhead = sum( |
| _ReadZipInfoExtraFieldLength(apk, i) for i in apk_contents) |
| # Account for zipalign overhead that exists in central directory header. |
| # Happens when python aligns entries in apkbuilder.py, but does not |
| # exist when using Android's zipalign. E.g. for bundle .apks files. |
| zipalign_overhead += sum(len(i.extra) for i in apk_contents) |
| signing_block_size = _MeasureApkSignatureBlock(apk) |
| |
| _, skip_extract_lib, _ = _ParseManifestAttributes(apk_path) |
| |
| # Pre-L: Dalvik - .odex file is simply decompressed/optimized dex file (~1x). |
| # L, M: ART - .odex file is compiled version of the dex file (~4x). |
| # N: ART - Uses Dalvik-like JIT for normal apps (~1x), full compilation for |
| # shared apps (~4x). |
| # Actual multipliers calculated using "apk_operations.py disk-usage". |
| # Will need to update multipliers once apk obfuscation is enabled. |
| # E.g. with obfuscation, the 4.04 changes to 4.46. |
| speed_profile_dex_multiplier = 1.17 |
| orig_filename = apks_path or apk_path |
| is_webview = 'WebView' in orig_filename |
| is_monochrome = 'Monochrome' in orig_filename |
| is_library = 'Library' in orig_filename |
| is_trichrome = 'TrichromeChrome' in orig_filename |
| # WebView is always a shared APK since other apps load it. |
| # Library is always shared since it's used by chrome and webview |
| # Chrome is always shared since renderers can't access dex otherwise |
| # (see DexFixer). |
| is_shared_apk = sdk_version >= 24 and (is_monochrome or is_webview |
| or is_library or is_trichrome) |
| # Dex decompression overhead varies by Android version. |
| if sdk_version < 21: |
| # JellyBean & KitKat |
| dex_multiplier = 1.16 |
| elif sdk_version < 24: |
| # Lollipop & Marshmallow |
| dex_multiplier = 4.04 |
| elif is_shared_apk: |
| # Oreo and above, compilation_filter=speed |
| dex_multiplier = 4.04 |
| else: |
| # Oreo and above, compilation_filter=speed-profile |
| dex_multiplier = speed_profile_dex_multiplier |
| |
| total_apk_size = os.path.getsize(apk_path) |
| for member in apk_contents: |
| filename = member.filename |
| if filename.endswith('/'): |
| continue |
| if filename.endswith('.so'): |
| basename = posixpath.basename(filename) |
| should_extract_lib = not skip_extract_lib and basename.startswith('lib') |
| native_code.AddZipInfo( |
| member, extracted_multiplier=int(should_extract_lib)) |
| elif filename.startswith('classes') and filename.endswith('.dex'): |
| # Android P+, uncompressed dex does not need to be extracted. |
| compressed = member.compress_type != zipfile.ZIP_STORED |
| multiplier = dex_multiplier |
| if not compressed and sdk_version >= 28: |
| multiplier -= 1 |
| |
| java_code.AddZipInfo(member, extracted_multiplier=multiplier) |
| elif re.search(_RE_NON_LANGUAGE_PAK, filename): |
| native_resources_no_translations.AddZipInfo(member) |
| elif filename.endswith('.pak') or filename.endswith('.lpak'): |
| compressed = member.compress_type != zipfile.ZIP_STORED |
| bucket = translations if compressed else stored_translations |
| extracted_multiplier = 0 |
| if compressed: |
| extracted_multiplier = int('en_' in filename or 'en-' in filename) |
| bucket.AddZipInfo(member, extracted_multiplier=extracted_multiplier) |
| elif 'icu' in filename and filename.endswith('.dat'): |
| icu_data.AddZipInfo(member) |
| elif filename.endswith('.bin'): |
| v8_snapshots.AddZipInfo(member) |
| elif filename.startswith('res/'): |
| if (filename.endswith('.png') or filename.endswith('.webp') |
| or has_no_extension(filename)): |
| png_drawables.AddZipInfo(member) |
| else: |
| res_directory.AddZipInfo(member) |
| elif filename.endswith('.arsc'): |
| arsc.AddZipInfo(member) |
| elif filename.startswith('META-INF') or filename in ( |
| 'AndroidManifest.xml', 'assets/webapk_dex_version.txt', |
| 'stamp-cert-sha256'): |
| metadata.AddZipInfo(member) |
| elif filename.endswith('.notice'): |
| notices.AddZipInfo(member) |
| elif filename.startswith('assets/unwind_cfi'): |
| unwind_cfi.AddZipInfo(member) |
| else: |
| unknown.AddZipInfo(member) |
| |
| if apks_path: |
| # We're mostly focused on size of Chrome for non-English locales, so assume |
| # Hindi (arbitrarily chosen) locale split is installed. |
| with zipfile.ZipFile(apks_path) as z: |
| subpath = 'splits/{}-hi.apk'.format(split_name) |
| if subpath in z.namelist(): |
| hindi_apk_info = z.getinfo(subpath) |
| total_apk_size += hindi_apk_info.file_size |
| else: |
| assert split_name != 'base', 'splits/base-hi.apk should always exist' |
| |
| total_install_size = total_apk_size |
| total_install_size_android_go = total_apk_size |
| zip_overhead = total_apk_size |
| |
| for group in file_groups: |
| actual_size = group.ComputeZippedSize() |
| install_size = group.ComputeInstallSize() |
| uncompressed_size = group.ComputeUncompressedSize() |
| extracted_size = group.ComputeExtractedSize() |
| total_install_size += extracted_size |
| zip_overhead -= actual_size |
| |
| report_func('Breakdown', group.name + ' size', actual_size, 'bytes') |
| report_func('InstallBreakdown', group.name + ' size', int(install_size), |
| 'bytes') |
| # Only a few metrics are compressed in the first place. |
| # To avoid over-reporting, track uncompressed size only for compressed |
| # entries. |
| if uncompressed_size != actual_size: |
| report_func('Uncompressed', group.name + ' size', uncompressed_size, |
| 'bytes') |
| |
| if group is java_code: |
| # Updates are compiled using quicken, but system image uses speed-profile. |
| multiplier = speed_profile_dex_multiplier |
| |
| # Android P+, uncompressed dex does not need to be extracted. |
| compressed = uncompressed_size != actual_size |
| if not compressed and sdk_version >= 28: |
| multiplier -= 1 |
| extracted_size = int(uncompressed_size * multiplier) |
| total_install_size_android_go += extracted_size |
| report_func('InstallBreakdownGo', group.name + ' size', |
| actual_size + extracted_size, 'bytes') |
| elif group is translations and apks_path: |
| # Assume Hindi rather than English (accounted for above in total_apk_size) |
| total_install_size_android_go += actual_size |
| else: |
| total_install_size_android_go += extracted_size |
| |
| # Per-file zip overhead is caused by: |
| # * 30 byte entry header + len(file name) |
| # * 46 byte central directory entry + len(file name) |
| # * 0-3 bytes for zipalign. |
| report_func('Breakdown', 'Zip Overhead', zip_overhead, 'bytes') |
| report_func('InstallSize', 'APK size', total_apk_size, 'bytes') |
| report_func('InstallSize', 'Estimated installed size', |
| int(total_install_size), 'bytes') |
| report_func('InstallSize', 'Estimated installed size (Android Go)', |
| int(total_install_size_android_go), 'bytes') |
| transfer_size = _CalculateCompressedSize(apk_path) |
| report_func('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes') |
| |
| # Size of main dex vs remaining. |
| main_dex_info = java_code.FindByPattern('classes.dex') |
| if main_dex_info: |
| main_dex_size = main_dex_info.file_size |
| report_func('Specifics', 'main dex size', main_dex_size, 'bytes') |
| secondary_size = java_code.ComputeUncompressedSize() - main_dex_size |
| report_func('Specifics', 'secondary dex size', secondary_size, 'bytes') |
| |
| main_lib_info = native_code.FindLargest() |
| native_code_unaligned_size = 0 |
| for lib_info in native_code.AllEntries(): |
| # Skip placeholders. |
| if lib_info.file_size == 0: |
| continue |
| section_sizes = _ExtractLibSectionSizesFromApk(apk_path, lib_info.filename) |
| native_code_unaligned_size += sum(v for k, v in section_sizes.items() |
| if k != 'bss') |
| # Size of main .so vs remaining. |
| if lib_info == main_lib_info: |
| main_lib_size = lib_info.file_size |
| report_func('Specifics', 'main lib size', main_lib_size, 'bytes') |
| secondary_size = native_code.ComputeUncompressedSize() - main_lib_size |
| report_func('Specifics', 'other lib size', secondary_size, 'bytes') |
| |
| for metric_name, size in section_sizes.items(): |
| report_func('MainLibInfo', metric_name, size, 'bytes') |
| |
| # Main metric that we want to monitor for jumps. |
| normalized_apk_size = total_apk_size |
| # unwind_cfi exists only in dev, canary, and non-channel builds. |
| normalized_apk_size -= unwind_cfi.ComputeZippedSize() |
| # Sections within .so files get 4kb aligned, so use section sizes rather than |
| # file size. Also gets rid of compression. |
| normalized_apk_size -= native_code.ComputeZippedSize() |
| normalized_apk_size += native_code_unaligned_size |
| # Normalized dex size: Size within the zip + size on disk for Android Go |
| # devices running Android O (which ~= uncompressed dex size). |
| # Use a constant compression factor to account for fluctuations. |
| normalized_apk_size -= java_code.ComputeZippedSize() |
| normalized_apk_size += java_code.ComputeUncompressedSize() |
| # Don't include zipalign overhead in normalized size, since it effectively |
| # causes size changes files that proceed aligned files to be rounded. |
| # For APKs where classes.dex directly proceeds libchrome.so (the normal case), |
| # this causes small dex size changes to disappear into libchrome.so alignment. |
| normalized_apk_size -= zipalign_overhead |
| # Don't include the size of the apk's signing block because it can fluctuate |
| # by up to 4kb (from my non-scientific observations), presumably based on hash |
| # sizes. |
| normalized_apk_size -= signing_block_size |
| |
| # Unaligned size should be ~= uncompressed size or something is wrong. |
| # As of now, padding_fraction ~= .007 |
| padding_fraction = -_PercentageDifference( |
| native_code.ComputeUncompressedSize(), native_code_unaligned_size) |
| # Ignore this check for small / no native code |
| if native_code.ComputeUncompressedSize() > 1000000: |
| assert 0 <= padding_fraction < .02, ( |
| 'Padding was: {} (file_size={}, sections_sum={})'.format( |
| padding_fraction, native_code.ComputeUncompressedSize(), |
| native_code_unaligned_size)) |
| |
| if apks_path: |
| # Locale normalization not needed when measuring only one locale. |
| # E.g. a change that adds 300 chars of unstranslated strings would cause the |
| # metric to be off by only 390 bytes (assuming a multiplier of 2.3 for |
| # Hindi). |
| pass |
| else: |
| # Avoid noise caused when strings change and translations haven't yet been |
| # updated. |
| num_translations = translations.GetNumEntries() |
| num_stored_translations = stored_translations.GetNumEntries() |
| |
| if num_translations > 1: |
| # Multipliers found by looking at MonochromePublic.apk and seeing how much |
| # smaller en-US.pak is relative to the average locale.pak. |
| normalized_apk_size += _NormalizeLanguagePaks(translations, 1.17) |
| if num_stored_translations > 1: |
| normalized_apk_size += _NormalizeLanguagePaks(stored_translations, 1.43) |
| if num_translations + num_stored_translations > 1: |
| if num_translations == 0: |
| # WebView stores all locale paks uncompressed. |
| num_arsc_translations = num_stored_translations |
| else: |
| # Monochrome has more configurations than Chrome since it includes |
| # WebView (which supports more locales), but these should mostly be |
| # empty so ignore them here. |
| num_arsc_translations = num_translations |
| normalized_apk_size += _NormalizeResourcesArsc(apk_path, |
| arsc.GetNumEntries(), |
| num_arsc_translations, |
| out_dir) |
| |
| # It will be -Inf for .apk files with multiple .arsc files and no out_dir set. |
| if normalized_apk_size < 0: |
| sys.stderr.write('Skipping normalized_apk_size (no output directory set)\n') |
| else: |
| report_func('Specifics', 'normalized apk size', normalized_apk_size, |
| 'bytes') |
| # The "file count" metric cannot be grouped with any other metrics when the |
| # end result is going to be uploaded to the perf dashboard in the HistogramSet |
| # format due to mixed units (bytes vs. zip entries) causing malformed |
| # summaries to be generated. |
| # TODO(crbug.com/41425646): Remove this workaround if unit mixing is |
| # ever supported. |
| report_func('FileCount', 'file count', len(apk_contents), 'zip entries') |
| |
| for info in unknown.AllEntries(): |
| sys.stderr.write( |
| 'Unknown entry: %s %d\n' % (info.filename, info.compress_size)) |
| return normalized_apk_size |
| |
| |
| def _CalculateCompressedSize(file_path): |
| CHUNK_SIZE = 256 * 1024 |
| compressor = zlib.compressobj() |
| total_size = 0 |
| with open(file_path, 'rb') as f: |
| for chunk in iter(lambda: f.read(CHUNK_SIZE), b''): |
| total_size += len(compressor.compress(chunk)) |
| total_size += len(compressor.flush()) |
| return total_size |
| |
| |
| @contextmanager |
| def Unzip(zip_file, filename=None): |
| """Utility for temporary use of a single file in a zip archive.""" |
| with build_utils.TempDir() as unzipped_dir: |
| unzipped_files = build_utils.ExtractAll( |
| zip_file, unzipped_dir, True, pattern=filename) |
| if len(unzipped_files) == 0: |
| raise Exception( |
| '%s not found in %s' % (filename, zip_file)) |
| yield unzipped_files[0] |
| |
| |
| def _ConfigOutDir(out_dir): |
| if out_dir: |
| constants.SetOutputDirectory(out_dir) |
| else: |
| try: |
| # Triggers auto-detection when CWD == output directory. |
| constants.CheckOutputDirectory() |
| out_dir = constants.GetOutDirectory() |
| except Exception: # pylint: disable=broad-except |
| pass |
| return out_dir |
| |
| |
| def _IterSplits(namelist): |
| for subpath in namelist: |
| # Looks for paths like splits/vr-master.apk, splits/vr-hi.apk. |
| name_parts = subpath.split('/') |
| if name_parts[0] == 'splits' and len(name_parts) == 2: |
| name_parts = name_parts[1].split('-') |
| if len(name_parts) == 2: |
| split_name, config_name = name_parts |
| if config_name == 'master.apk': |
| yield subpath, split_name |
| |
| |
| def _ExtractToTempFile(zip_obj, subpath, temp_file): |
| temp_file.seek(0) |
| temp_file.truncate() |
| temp_file.write(zip_obj.read(subpath)) |
| temp_file.flush() |
| |
| |
| def _AnalyzeApkOrApks(report_func, apk_path, out_dir): |
| # Create DexStatsCollector here to track unique methods across base & chrome |
| # modules. |
| dex_stats_collector = method_count.DexStatsCollector() |
| |
| if apk_path.endswith('.apk'): |
| sdk_version, _, _ = _ParseManifestAttributes(apk_path) |
| _AnalyzeInternal(apk_path, sdk_version, report_func, dex_stats_collector, |
| out_dir) |
| elif apk_path.endswith('.apks'): |
| with tempfile.NamedTemporaryFile(suffix='.apk') as f: |
| with zipfile.ZipFile(apk_path) as z: |
| # Currently bundletool is creating two apks when .apks is created |
| # without specifying an sdkVersion. Always measure the one with an |
| # uncompressed shared library. |
| try: |
| info = z.getinfo('splits/base-master_2.apk') |
| except KeyError: |
| info = z.getinfo('splits/base-master.apk') |
| _ExtractToTempFile(z, info.filename, f) |
| sdk_version, _, _ = _ParseManifestAttributes(f.name) |
| |
| orig_report_func = report_func |
| report_func = _AccumulatingReporter() |
| |
| def do_measure(split_name, on_demand): |
| logging.info('Measuring %s on_demand=%s', split_name, on_demand) |
| # Use no-op reporting functions to get normalized size for DFMs. |
| inner_report_func = report_func |
| inner_dex_stats_collector = dex_stats_collector |
| if on_demand: |
| inner_report_func = lambda *_: None |
| inner_dex_stats_collector = method_count.DexStatsCollector() |
| |
| size = _AnalyzeInternal(f.name, |
| sdk_version, |
| inner_report_func, |
| inner_dex_stats_collector, |
| out_dir, |
| apks_path=apk_path, |
| split_name=split_name) |
| report_func('DFM_' + split_name, 'Size with hindi', size, 'bytes') |
| |
| # Measure base outside of the loop since we've already extracted it. |
| do_measure('base', on_demand=False) |
| |
| for subpath, split_name in _IterSplits(z.namelist()): |
| if split_name != 'base': |
| _ExtractToTempFile(z, subpath, f) |
| _, _, on_demand = _ParseManifestAttributes(f.name) |
| do_measure(split_name, on_demand=on_demand) |
| |
| report_func.DumpReports(orig_report_func) |
| report_func = orig_report_func |
| else: |
| raise Exception('Unknown file type: ' + apk_path) |
| |
| # Report dex stats outside of _AnalyzeInternal() so that the "unique methods" |
| # metric is not just the sum of the base and chrome modules. |
| for metric, count in dex_stats_collector.GetTotalCounts().items(): |
| report_func('Dex', metric, count, 'entries') |
| report_func('Dex', 'unique methods', |
| dex_stats_collector.GetUniqueMethodCount(), 'entries') |
| report_func('DexCache', 'DexCache', |
| dex_stats_collector.GetDexCacheSize(pre_oreo=sdk_version < 26), |
| 'bytes') |
| |
| return dex_stats_collector |
| |
| |
| def _ResourceSizes(args): |
| chartjson = _BASE_CHART.copy() if args.output_format else None |
| reporter = _ChartJsonReporter(chartjson) |
| # Create DexStatsCollector here to track unique methods across trichrome APKs. |
| dex_stats_collector = method_count.DexStatsCollector() |
| |
| specs = [ |
| ('Chrome_', args.trichrome_chrome), |
| ('WebView_', args.trichrome_webview), |
| ('Library_', args.trichrome_library), |
| ] |
| for prefix, path in specs: |
| if path: |
| reporter.trace_title_prefix = prefix |
| child_dex_stats_collector = _AnalyzeApkOrApks(reporter, path, |
| args.out_dir) |
| dex_stats_collector.MergeFrom(prefix, child_dex_stats_collector) |
| |
| if any(path for _, path in specs): |
| reporter.SynthesizeTotals(dex_stats_collector.GetUniqueMethodCount()) |
| else: |
| _AnalyzeApkOrApks(reporter, args.input, args.out_dir) |
| |
| if chartjson: |
| _DumpChartJson(args, chartjson) |
| |
| |
| def _DumpChartJson(args, chartjson): |
| if args.output_file == '-': |
| json_file = sys.stdout |
| elif args.output_file: |
| json_file = open(args.output_file, 'w') |
| else: |
| results_path = os.path.join(args.output_dir, 'results-chart.json') |
| logging.critical('Dumping chartjson to %s', results_path) |
| json_file = open(results_path, 'w') |
| |
| json.dump(chartjson, json_file, indent=2) |
| |
| if json_file is not sys.stdout: |
| json_file.close() |
| |
| # We would ideally generate a histogram set directly instead of generating |
| # chartjson then converting. However, perf_tests_results_helper is in |
| # //build, which doesn't seem to have any precedent for depending on |
| # anything in Catapult. This can probably be fixed, but since this doesn't |
| # need to be super fast or anything, converting is a good enough solution |
| # for the time being. |
| if args.output_format == 'histograms': |
| histogram_result = convert_chart_json.ConvertChartJson(results_path) |
| if histogram_result.returncode != 0: |
| raise Exception('chartjson conversion failed with error: ' + |
| histogram_result.stdout) |
| |
| histogram_path = os.path.join(args.output_dir, 'perf_results.json') |
| logging.critical('Dumping histograms to %s', histogram_path) |
| with open(histogram_path, 'wb') as json_file: |
| json_file.write(histogram_result.stdout) |
| |
| |
| def main(): |
| build_utils.InitLogging('RESOURCE_SIZES_DEBUG') |
| argparser = argparse.ArgumentParser(description='Print APK size metrics.') |
| argparser.add_argument( |
| '--min-pak-resource-size', |
| type=int, |
| default=20 * 1024, |
| help='Minimum byte size of displayed pak resources.') |
| argparser.add_argument( |
| '--chromium-output-directory', |
| dest='out_dir', |
| type=os.path.realpath, |
| help='Location of the build artifacts.') |
| argparser.add_argument( |
| '--chartjson', |
| action='store_true', |
| help='DEPRECATED. Use --output-format=chartjson ' |
| 'instead.') |
| argparser.add_argument( |
| '--output-format', |
| choices=['chartjson', 'histograms'], |
| help='Output the results to a file in the given ' |
| 'format instead of printing the results.') |
| argparser.add_argument('--loadable_module', help='Obsolete (ignored).') |
| |
| # Accepted to conform to the isolated script interface, but ignored. |
| argparser.add_argument( |
| '--isolated-script-test-filter', help=argparse.SUPPRESS) |
| argparser.add_argument( |
| '--isolated-script-test-perf-output', |
| type=os.path.realpath, |
| help=argparse.SUPPRESS) |
| |
| output_group = argparser.add_mutually_exclusive_group() |
| |
| output_group.add_argument( |
| '--output-dir', default='.', help='Directory to save chartjson to.') |
| output_group.add_argument( |
| '--output-file', |
| help='Path to output .json (replaces --output-dir). Works only for ' |
| '--output-format=chartjson') |
| output_group.add_argument( |
| '--isolated-script-test-output', |
| type=os.path.realpath, |
| help='File to which results will be written in the ' |
| 'simplified JSON output format.') |
| |
| argparser.add_argument('input', help='Path to .apk or .apks file to measure.') |
| trichrome_group = argparser.add_argument_group( |
| 'Trichrome inputs', |
| description='When specified, |input| is used only as Test suite name.') |
| trichrome_group.add_argument( |
| '--trichrome-chrome', help='Path to Trichrome Chrome .apks') |
| trichrome_group.add_argument( |
| '--trichrome-webview', help='Path to Trichrome WebView .apk(s)') |
| trichrome_group.add_argument( |
| '--trichrome-library', help='Path to Trichrome Library .apk') |
| args = argparser.parse_args() |
| |
| args.out_dir = _ConfigOutDir(args.out_dir) |
| devil_chromium.Initialize(output_directory=args.out_dir) |
| |
| # TODO(bsheedy): Remove this once uses of --chartjson have been removed. |
| if args.chartjson: |
| args.output_format = 'chartjson' |
| |
| result_sink_client = result_sink.TryInitClient() |
| isolated_script_output = {'valid': False, 'failures': []} |
| |
| test_name = 'resource_sizes (%s)' % os.path.basename(args.input) |
| |
| if args.isolated_script_test_output: |
| args.output_dir = os.path.join( |
| os.path.dirname(args.isolated_script_test_output), test_name) |
| if not os.path.exists(args.output_dir): |
| os.makedirs(args.output_dir) |
| |
| try: |
| _ResourceSizes(args) |
| isolated_script_output = { |
| 'valid': True, |
| 'failures': [], |
| } |
| finally: |
| if args.isolated_script_test_output: |
| results_path = os.path.join(args.output_dir, 'test_results.json') |
| with open(results_path, 'w') as output_file: |
| json.dump(isolated_script_output, output_file) |
| with open(args.isolated_script_test_output, 'w') as output_file: |
| json.dump(isolated_script_output, output_file) |
| if result_sink_client: |
| status = result_types.PASS |
| if not isolated_script_output['valid']: |
| status = result_types.UNKNOWN |
| elif isolated_script_output['failures']: |
| status = result_types.FAIL |
| result_sink_client.Post(test_name, status, None, None, None) |
| |
| |
| if __name__ == '__main__': |
| main() |
