Android: Add APK patch size estimates to resource_sizes.py.
APK patch size estimates will be used on certain perf builders, and will
track estimated patch size based on a reference APK (built by the same
builder) for the current milestone.
BUG=695188
Review-Url: https://codereview.chromium.org/2757293002
Cr-Commit-Position: refs/heads/master@{#458181}
diff --git a/build/android/binary_size/__init__.py b/build/android/binary_size/__init__.py
new file mode 100644
index 0000000..a22a6ee
--- /dev/null
+++ b/build/android/binary_size/__init__.py
@@ -0,0 +1,3 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
diff --git a/build/android/binary_size/update_apks.py b/build/android/binary_size/apk_downloader.py
similarity index 95%
rename from build/android/binary_size/update_apks.py
rename to build/android/binary_size/apk_downloader.py
index a197e64..b90fe5f 100755
--- a/build/android/binary_size/update_apks.py
+++ b/build/android/binary_size/apk_downloader.py
@@ -23,13 +23,16 @@
def MaybeDownloadApk(builder, milestone, apk, download_path, bucket):
+ """Returns path to the downloaded APK or None if not found."""
apk_path = os.path.join(download_path, builder, milestone, apk)
sha1_path = apk_path + '.sha1'
base_url = os.path.join(bucket, builder, milestone)
if os.path.exists(apk_path):
print '%s already exists' % apk_path
+ return apk_path
elif not os.path.exists(sha1_path):
print 'Skipping %s, file not found' % sha1_path
+ return None
else:
download_from_google_storage.download_from_google_storage(
input_filename=sha1_path,
@@ -46,6 +49,7 @@
verbose=True,
auto_platform=False,
extract=False)
+ return apk_path
def main():
diff --git a/build/android/binary_size/apks/README.md b/build/android/binary_size/apks/README.md
new file mode 100644
index 0000000..474ba93
--- /dev/null
+++ b/build/android/binary_size/apks/README.md
@@ -0,0 +1,24 @@
+### Updating APKs in this folder (for new milestones, builders, or APKs)
+
+1. Find the commit as close as possible to the current branch point (i.e. if the
+latest builds are m59, we want to compare to the commit before the m58 branch
+point).
+
+2. Download and unzip build artifacts from the relevant perf builder. You can
+use this link:
+[https<nolink>://storage.cloud.google.com/chrome-perf/**Android%20Builder**/full-build-linux_**3a87aecc31cd1ffe751dd72c04e5a96a1fc8108a**.zip](https://storage.cloud.google.com/chrome-perf/Android%20Builder/full-build-linux_3a87aecc31cd1ffe751dd72c04e5a96a1fc8108a.zip)
+, replacing the bolded parts with your info OR from the
+"gsutil upload_build_product" step on the bot page (both are Googlers only).
+
+3. Upload the apk: _upload_to_google_storage.py --bucket
+'chromium-android-tools/apks/**Android_Builder**/**58**'
+**path/to/ApkTarget.apk**_ replacing the bolded parts again.
+ * Note that we use **Android_Builder** instead of **Android Builder** (replace
+spaces with underscores)
+
+4. Move the generated .sha1 file to the corresponding place in
+//build/android/binary_size/apks/. In this case, the path would be
+//build/android/binary_size/apks/Android_Builder/58
+
+5. Commit the added .sha1 files and (optionally) update the `CURRENT_MILESTONE`
+in apk_downloader.py
diff --git a/build/android/resource_sizes.py b/build/android/resource_sizes.py
index a0ff3f0..ce0d6de6 100755
--- a/build/android/resource_sizes.py
+++ b/build/android/resource_sizes.py
@@ -23,6 +23,7 @@
import zipfile
import zlib
+from binary_size import apk_downloader
import devil_chromium
from devil.android.sdk import build_tools
from devil.utils import cmd_helper
@@ -35,6 +36,8 @@
_GRIT_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'tools', 'grit')
_BUILD_UTILS_PATH = os.path.join(
host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'gyp')
+_APK_PATCH_SIZE_ESTIMATOR_PATH = os.path.join(
+ host_paths.DIR_SOURCE_ROOT, 'third_party', 'apk-patch-size-estimator')
# Prepend the grit module from the source tree so it takes precedence over other
# grit versions that might present in the search path.
@@ -47,6 +50,9 @@
with host_paths.SysPath(_BUILD_UTILS_PATH, 1):
from util import build_utils # pylint: disable=import-error
+with host_paths.SysPath(_APK_PATCH_SIZE_ESTIMATOR_PATH):
+ import apk_patch_size_estimator # pylint: disable=import-error
+
# Python had a bug in zipinfo parsing that triggers on ChromeModern.apk
# https://bugs.python.org/issue14315
@@ -674,6 +680,26 @@
'bytes')
+def _PrintPatchSizeEstimate(new_apk, builder, bucket, chartjson=None):
+ apk_name = os.path.basename(new_apk)
+ title = apk_name + '_PatchSizeEstimate'
+ # Reference APK paths have spaces replaced by underscores.
+ builder = builder.replace(' ', '_')
+ old_apk = apk_downloader.MaybeDownloadApk(
+ builder, apk_downloader.CURRENT_MILESTONE, apk_name,
+ apk_downloader.DEFAULT_DOWNLOAD_PATH, bucket)
+ if old_apk:
+ # Use a temp dir in case patch size functions fail to clean up temp files.
+ with build_utils.TempDir() as tmp:
+ tmp_name = os.path.join(tmp, 'patch.tmp')
+ bsdiff = apk_patch_size_estimator.calculate_bsdiff(
+ old_apk, new_apk, None, tmp_name)
+ ReportPerfResult(chartjson, title, 'BSDiff (gzipped)', bsdiff, 'bytes')
+ fbf = apk_patch_size_estimator.calculate_filebyfile(
+ old_apk, new_apk, None, tmp_name)
+ ReportPerfResult(chartjson, title, 'FileByFile (gzipped)', fbf, 'bytes')
+
+
@contextmanager
def Unzip(zip_file, filename=None):
"""Utility for temporary use of a single file in a zip archive."""
@@ -712,6 +738,17 @@
'output-dir')
argparser.add_argument('-d', '--device',
help='Dummy option for perf runner.')
+ argparser.add_argument('--estimate-patch-size', action='store_true',
+ help='Include patch size estimates. Useful for perf '
+ 'builders where a reference APK is available but adds '
+ '~3 mins to run time.')
+ argparser.add_argument('--reference-apk-builder',
+ default=apk_downloader.DEFAULT_BUILDER,
+ help='Builder name to use for reference APK for patch '
+ 'size estimates.')
+ argparser.add_argument('--reference-apk-bucket',
+ default=apk_downloader.DEFAULT_BUCKET,
+ help='Storage bucket holding reference APKs.')
argparser.add_argument('apk', help='APK file path.')
args = argparser.parse_args()
@@ -730,6 +767,9 @@
PrintApkAnalysis(args.apk, tools_prefix, chartjson=chartjson)
_PrintDexAnalysis(args.apk, chartjson=chartjson)
+ if args.estimate_patch_size:
+ _PrintPatchSizeEstimate(
+ args.apk, args.builder, args.bucket, chartjson=chartjson)
if not args.no_output_dir:
PrintPakAnalysis(args.apk, args.min_pak_resource_size)
_PrintStaticInitializersCountFromApk(
diff --git a/third_party/apk-patch-size-estimator/README.chromium b/third_party/apk-patch-size-estimator/README.chromium
index 67e9c73..6ed5efee5 100644
--- a/third_party/apk-patch-size-estimator/README.chromium
+++ b/third_party/apk-patch-size-estimator/README.chromium
@@ -12,4 +12,5 @@
for more details.
Local Modifications:
-Removed the following files: CONTRIBUTING.md, tests/, images/.
+- Removed the following files: CONTRIBUTING.md, tests/, images/.
+- Changed file-by-file-tools.jar path to be absolute instead of relative.
diff --git a/third_party/apk-patch-size-estimator/apk_patch_size_estimator.py b/third_party/apk-patch-size-estimator/apk_patch_size_estimator.py
index 1be08e7..64b4bb6 100755
--- a/third_party/apk-patch-size-estimator/apk_patch_size_estimator.py
+++ b/third_party/apk-patch-size-estimator/apk_patch_size_estimator.py
@@ -36,6 +36,9 @@
import os
import subprocess
+_FILEBYFILE_JAR_PATH = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), 'lib', 'file-by-file-tools.jar'))
+
bsdiff_path = None
gzip_path = None
head_path = None
@@ -259,8 +262,8 @@
# We use a jar from https://github.com/andrewhayden/archive-patcher
if os.path.exists(filebyfile_patch_path): os.remove(filebyfile_patch_path)
p = subprocess.Popen(
- [java_path, '-jar', 'lib/file-by-file-tools.jar', '--generate',
- '--old', old_file, '--new', new_file, '--patch', filebyfile_patch_path],
+ [java_path, '-jar', _FILEBYFILE_JAR_PATH, '--generate', '--old', old_file,
+ '--new', new_file, '--patch', filebyfile_patch_path],
shell=False)
ret_code = p.wait()
if ret_code != 0: raise Exception(
