blob: fd6bc7e4a190c7b709d97abc4bce6824a468b0fb [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
r"""This script downloads / packages & uploads Android SDK packages.
It could be run when we need to update sdk packages to latest version.
It has 2 usages:
1) download: downloading a new version of the SDK via sdkmanager
2) package: wrapping SDK directory into CIPD-compatible packages and
uploading the new packages via CIPD to server.
Providing '--dry-run' option to show what packages to be
created and uploaded without actually doing either.
Both downloading and uploading allows to either specify a package, or
deal with default packages (build-tools, platform-tools, platforms and
Example usage:
1) updating default packages:
$ download
(optional) $ package --dry-run
$ package
2) updating a specified package:
$ download -p "build-tools;27.0.3"
(optional) $ package --dry-run -p build-tools \
--version 27.0.3
$ package -p build-tools --version 27.0.3
Note that `package` could update the package argument to the checkout
version in .gn file //build/config/android/config.gni. If having git
changes, please prepare to upload a CL that updates the SDK version.
import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
_SRC_ROOT = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..'))
_SRC_DEPS_PATH = os.path.join(_SRC_ROOT, 'DEPS')
_SDK_PUBLIC_ROOT = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
_SDKMANAGER_PATH = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
'public', 'tools', 'bin', 'sdkmanager')
_ANDROID_CONFIG_GNI_PATH = os.path.join(_SRC_ROOT, 'build', 'config', 'android',
_TOOLS_LIB_PATH = os.path.join(_SDK_PUBLIC_ROOT, 'tools', 'lib')
'build-tools', 'platform-tools', 'platforms', 'tools'
# TODO(shenghuazhang): Search package versions from available packages through
# the sdkmanager, instead of hardcoding the package names w/ version.
# TODO(yliuyliu): we might not need the latest version if unstable,
# will double-check this later.
'build-tools': 'build-tools;27.0.3',
'platforms': 'platforms;android-28',
'sources': 'sources;android-28',
'build-tools': 'default_android_sdk_build_tools_version',
'tools': 'default_android_sdk_tools_version_suffix',
'platforms': 'default_android_sdk_version',
r'^common' # file name begins with 'common'
r'(-[\d\.]+(-dev)?)' # group of suffix e.g.'-26.0.0-dev', '-25.3.2'
r'\.jar$' # ends with .jar
def _DownloadSdk(arguments):
"""Download sdk package from sdkmanager.
If package isn't provided, update build-tools, platform-tools, platforms,
and tools.
arguments: The arguments parsed from argparser.
for pkg in arguments.package:
# If package is not a sdk-style path, try to match a default path to it.
print 'Coercing %s to %s' % (pkg, _DEFAULT_PACKAGES_DICT[pkg])
download_sdk_cmd = [
_SDKMANAGER_PATH, '--install',
'--sdk_root=%s' % arguments.sdk_root, pkg
if arguments.verbose:
def _FindPackageVersion(package, sdk_root):
"""Find sdk package version.
Two options for package version:
1) Use the version in name if package name contains ';version'
2) For simple name package, search its version from 'Installed packages'
via `sdkmanager --list`
package: The Android SDK package.
sdk_root: The Android SDK root path.
The version of package.
Exception: cannot find the version of package.
sdkmanager_list_cmd = [
'--sdk_root=%s' % sdk_root,
# Get the version after ';' from package name
package = _DEFAULT_PACKAGES_DICT[package]
return package.split(';')[1]
# Get the package version via `sdkmanager --list`. The logic is:
# Check through 'Installed packages' which is at the first section of
# `sdkmanager --list` output, example:
# Installed packages:=====================] 100% Computing updates...
# Path | Version | Description
# ------- | ------- | -------
# build-tools;27.0.3 | 27.0.3 | Android SDK Build-Tools 27.0.3
# emulator | 26.0.3 | Android Emulator
# platforms;android-27 | 1 | Android SDK Platform 27
# tools | 26.1.1 | Android SDK Tools
# Available Packages:
# ....
# When found a line containing the package path, grap its version between
# the first and second '|'. Since the 'Installed packages' list section ends
# by the first new line, the check loop should be ended when reaches a '\n'.
output = subprocess.check_output(sdkmanager_list_cmd)
for line in output.splitlines():
if ' ' + package + ' ' in line:
# if found package path, catch its version which in the first '|...|'
return line.split('|')[1].strip()
if line == '\n': # Reaches the end of 'Installed packages' list
raise Exception('Cannot find the version of package %s' % package)
def _ReplaceVersionInFile(file_path, pattern, version, dry_run=False):
"""Replace the version of sdk package argument in file.
Check whether the version in file is the same as the new version first.
Replace the version if not dry run.
file_path: Path to the file to update the version of sdk package argument.
pattern: Pattern for the sdk package argument. Must capture at least one
group that the first group is the argument line excluding version.
version: The new version of the package.
dry_run: Bool. To show what packages would be created and packages, without
actually doing either.
with tempfile.NamedTemporaryFile() as temp_file:
with open(file_path) as f:
for line in f:
new_line = re.sub(pattern, r'\g<1>\g<2>%s\g<3>\n' % version, line)
if new_line != line:
print(' Note: file "%s" argument ' % file_path +
'"%s" would be updated to "%s".' % (line.strip(), version))
if not dry_run:
shutil.move(, file_path)
temp_file.delete = False
def UploadSdkPackage(sdk_root, dry_run, service_url, package, yaml_file,
"""Build and upload a package instance file to CIPD.
This would also update gn and ensure files to the package version as
uploading to CIPD.
sdk_root: Root of the sdk packages.
dry_run: Bool. To show what packages would be created and packages, without
actually doing either.
service_url: The url of the CIPD service.
package: The package to be uploaded to CIPD.
yaml_file: Path to the yaml file that defines what to put into the package.
Default as //third_party/android_sdk/public/cipd_*.yaml
verbose: Enable more logging.
New instance ID when CIPD package created.
IOError: cannot find .yaml file or instance ID for package.
CalledProcessError: cipd command failed to create package.
pkg_yaml_file = yaml_file or os.path.join(sdk_root, 'cipd_%s.yaml' % package)
if not os.path.exists(pkg_yaml_file):
raise IOError('Cannot find .yaml file for package %s' % package)
if dry_run:
print('This `package` command (without -n/--dry-run) would create and ' +
'upload the package %s to CIPD.' % package)
upload_sdk_cmd = [
'cipd', 'create', '-pkg-def', pkg_yaml_file, '-service-url', service_url
if verbose:
upload_sdk_cmd.extend(['-log-level', 'debug'])
output = subprocess.check_output(upload_sdk_cmd)
# Need to match pattern to find new instance ID.
# e.g.: chromium/third_party/android_sdk/public/platforms:\
# Kg2t9p0YnQk8bldUv4VA3o156uPXLUfIFAmVZ-Gm5ewC
pattern = re.compile(
# Match the argument with
# "Instance: chromium/third_party/android_sdk/public/%s:"
(r'(^\s*Instance:\schromium\/third_party\/android_sdk\/public\/%s:)' %
package) +
# instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
# End of string
for line in output.splitlines():
found = re.match(pattern, line)
if found:
# Return new instance ID.
# Raises error if instance ID not found.
raise IOError('Cannot find instance ID by creating package %s' % package)
def UpdateInstanceId(package,
"""Find the sdk pkg version in DEPS and modify it as cipd uploading version.
TODO(shenghuazhang): use DEPS edition operations after issue
DEPS file hooks sdk package with version with suffix -crX, e.g. '26.0.2-cr1'.
If pkg_version is the base number of the existing version in DEPS, e.g.
'26.0.2', return '26.0.2-cr2' as the version uploading to CIPD. If not the
base number, return ${pkg_version}-cr0.
package: The name of the package.
deps_path: Path to deps file which gclient hooks sdk pkg w/ versions.
dry_run: Bool. To show what packages would be created and packages, without
actually doing either.
new_instance_id: New instance ID after CIPD package created.
release_version: Android sdk release version e.g. 'o_mr1', 'p'.
var_package = package
if release_version:
var_package = release_version + '_' + var_package
package_var_pattern = re.compile(
# Match the argument with "'android_sdk_*_version': '" with whitespaces.
r'(^\s*\'android_sdk_%s_version\'\s*:\s*\')' % var_package +
# instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
# End of string
with tempfile.NamedTemporaryFile() as temp_file:
with open(deps_path) as f:
for line in f:
new_line = line
found = re.match(package_var_pattern, line)
if found:
instance_id =
new_line = re.sub(package_var_pattern,
r'\g<1>%s\g<3>' % new_instance_id, line)
' Note: deps file "%s" argument ' % deps_path +
'"%s" would be updated to "%s".' % (instance_id, new_instance_id))
if not dry_run:
shutil.move(, deps_path)
temp_file.delete = False
def ChangeVersionInGNI(package, arg_version, gn_args_dict, gni_file_path,
"""Change the sdk package version in config.gni file."""
if package in gn_args_dict:
version_config_name = gn_args_dict.get(package)
# Regex to parse the line of sdk package version gn argument, e.g.
# ' default_android_sdk_version = "27"'. Capture a group for the line
# excluding the version.
gn_arg_pattern = re.compile(
# Match the argument with '=' and whitespaces. Capture a group for it.
r'(^\s*%s\s*=\s*)' % version_config_name +
# Optional quote.
r'("?)' +
# Version number. E.g. 27, 27.0.3, -26.0.0-dev
r'(?:[-\w\s.]+)' +
# Optional quote.
r'("?)' +
# End of string
_ReplaceVersionInFile(gni_file_path, gn_arg_pattern, arg_version, dry_run)
def GetToolsSuffix(tools_lib_path):
"""Get the gn config of package 'tools' suffix.
Check jar file name of 'common*.jar' in tools/lib, which could be
'common.jar', common-<version>-dev.jar' or 'common-<version>.jar'.
If suffix exists, return the suffix.
tools_lib_path: The path of tools/lib.
The suffix of tools package.
tools_lib_jars_list = os.listdir(tools_lib_path)
for file_name in tools_lib_jars_list:
found = re.match(_COMMON_JAR_SUFFIX_PATTERN, file_name)
if found:
def _GetArgVersion(pkg_version, package):
"""Get the argument version.
pkg_version: The package version.
package: The package name.
The argument version.
# Remove all chars except for digits and dots in version
arg_version = re.sub(r'[^\d\.]', '', pkg_version)
if package == 'tools':
suffix = GetToolsSuffix(_TOOLS_LIB_PATH)
if suffix:
arg_version = suffix
arg_version = '-%s' % arg_version
return arg_version
def _UploadSdkPackage(arguments):
"""Upload SDK packages to CIPD.
arguments: The arguments parsed by argparser.
IOError: Don't use --version/--yaml-file for default packages.
packages = arguments.package
if not packages:
if arguments.version or arguments.yaml_file:
raise IOError("Don't use --version/--yaml-file for default packages.")
for package in packages:
pkg_version = arguments.version
if not pkg_version:
pkg_version = _FindPackageVersion(package, arguments.sdk_root)
# Upload SDK package to CIPD, and update the package instance ID hooking
# in DEPS file.
new_instance_id = UploadSdkPackage(
os.path.join(arguments.sdk_root, '..'), arguments.dry_run,
arguments.service_url, package, arguments.yaml_file, arguments.verbose)
UpdateInstanceId(package, _SRC_DEPS_PATH, arguments.dry_run,
if package in _GN_ARGUMENTS_TO_UPDATE:
# Update the package version config in gn file
arg_version = _GetArgVersion(pkg_version, package)
ChangeVersionInGNI(package, arg_version, _GN_ARGUMENTS_TO_UPDATE,
_ANDROID_CONFIG_GNI_PATH, arguments.dry_run)
def main():
parser = argparse.ArgumentParser(
description='A script to download Android SDK packages '
'via sdkmanager and upload to CIPD.')
subparsers = parser.add_subparsers(title='commands')
download_parser = subparsers.add_parser(
help='Download sdk package to the latest version from sdkmanager.')
help='The package of the SDK needs to be installed/updated. '
'Note that package name should be a sdk-style path e.g. '
'"platforms;android-27" or "platform-tools". If package '
'is not specified, update "build-tools;27.0.3", "tools" '
'"platform-tools" and "platforms;android-27" by default.')
'--sdk-root', help='base path to the Android SDK root')
'-v', '--verbose', action='store_true', help='print debug information')
package_parser = subparsers.add_parser(
'package', help='Create and upload package instance file to CIPD.')
help='Dry run won\'t trigger creating instances or uploading packages. '
'It shows what packages would be created and uploaded to CIPD. '
'It also shows the possible updates of sdk version on files.')
help='The package to be uploaded to CIPD. Note that package '
'name is a simple path e.g. "platforms" or "build-tools" '
'which matches package name on CIPD service. Default by '
'build-tools, platform-tools, platforms and tools')
help='Version of the uploading package instance through CIPD.')
help='Path to *.yaml file that defines what to put into the package.'
'Default as //third_party/android_sdk/public/cipd_<package>.yaml')
help='The url of the CIPD service.',
'--sdk-root', help='base path to the Android SDK root')
'-v', '--verbose', action='store_true', help='print debug information')
args = parser.parse_args()
if not args.sdk_root:
args.sdk_root = _SDK_PUBLIC_ROOT
if __name__ == '__main__':