| #!/usr/bin/env python |
| # Copyright 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| ''' |
| Script to help uploading and downloading the Google Play services library to |
| and from a Google Cloud storage. |
| ''' |
| |
| import argparse |
| import logging |
| import os |
| import re |
| import shutil |
| import sys |
| import zipfile |
| |
| sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) |
| import devil_chromium |
| from devil.utils import cmd_helper |
| from play_services import utils |
| from py_utils import tempfile_ext |
| from pylib import constants |
| from pylib.constants import host_paths |
| from pylib.utils import logging_utils |
| from pylib.utils import maven_downloader |
| |
| sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')) |
| import find_depot_tools # pylint: disable=import-error,unused-import |
| import breakpad |
| import download_from_google_storage |
| import upload_to_google_storage |
| |
| |
| # Directory where the SHA1 files for the zip and the license are stored |
| # It should be managed by git to provided information about new versions. |
| SHA1_DIRECTORY = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', 'android', |
| 'play_services') |
| |
| # Default bucket used for storing the files. |
| GMS_CLOUD_STORAGE = 'chromium-android-tools/play-services' |
| |
| # Path to the default configuration file. It exposes the currently installed |
| # version of the library in a human readable way. |
| CONFIG_DEFAULT_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', |
| 'android', 'play_services', 'config.json') |
| |
| LICENSE_FILE_NAME = 'LICENSE' |
| ZIP_FILE_NAME = 'google_play_services_library.zip' |
| |
| LICENSE_PATTERN = re.compile(r'^Pkg\.License=(?P<text>.*)$', re.MULTILINE) |
| |
| COM_GOOGLE_ANDROID_GMS = os.path.join('com', 'google', 'android', 'gms') |
| EXTRAS_GOOGLE_M2REPOSITORY = os.path.join('extras', 'google', 'm2repository') |
| |
| def main(raw_args): |
| parser = argparse.ArgumentParser( |
| description=__doc__ + 'Please see the subcommand help for more details.', |
| formatter_class=utils.DefaultsRawHelpFormatter) |
| subparsers = parser.add_subparsers(title='commands') |
| |
| # Download arguments |
| parser_download = subparsers.add_parser( |
| 'download', |
| help='download the library from the cloud storage', |
| description=Download.__doc__, |
| formatter_class=utils.DefaultsRawHelpFormatter) |
| parser_download.set_defaults(func=Download) |
| AddBasicArguments(parser_download) |
| AddBucketArguments(parser_download) |
| |
| # SDK Update arguments |
| parser_sdk = subparsers.add_parser( |
| 'sdk', |
| help='get the latest Google Play services SDK using Maven', |
| description=UpdateSdk.__doc__, |
| formatter_class=utils.DefaultsRawHelpFormatter) |
| parser_sdk.set_defaults(func=UpdateSdk) |
| AddBasicArguments(parser_sdk) |
| |
| # Upload arguments |
| parser_upload = subparsers.add_parser( |
| 'upload', |
| help='upload the library to the cloud storage', |
| description=Upload.__doc__, |
| formatter_class=utils.DefaultsRawHelpFormatter) |
| |
| parser_upload.set_defaults(func=Upload) |
| AddBasicArguments(parser_upload) |
| AddBucketArguments(parser_upload) |
| |
| args = parser.parse_args(raw_args) |
| if args.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| logging_utils.ColorStreamHandler.MakeDefault(not _IsBotEnvironment()) |
| devil_chromium.Initialize() |
| return args.func(args) |
| |
| |
| def AddBasicArguments(parser): |
| ''' |
| Defines the common arguments on subparser rather than the main one. This |
| allows to put arguments after the command: `foo.py upload --debug --force` |
| instead of `foo.py --debug upload --force` |
| ''' |
| |
| parser.add_argument('--config', |
| help='JSON Configuration file', |
| default=CONFIG_DEFAULT_PATH) |
| |
| parser.add_argument('--sdk-root', |
| help='base path to the Android SDK tools root', |
| default=constants.ANDROID_SDK_ROOT) |
| |
| parser.add_argument('-v', '--verbose', |
| action='store_true', |
| help='print debug information') |
| |
| |
| def AddBucketArguments(parser): |
| parser.add_argument('--bucket', |
| help='name of the bucket where the files are stored', |
| default=GMS_CLOUD_STORAGE) |
| |
| parser.add_argument('--dry-run', |
| action='store_true', |
| help=('run the script in dry run mode. Files will be ' |
| 'copied to a local directory instead of the ' |
| 'cloud storage. The bucket name will be as path ' |
| 'to that directory relative to the repository ' |
| 'root.')) |
| |
| parser.add_argument('-f', '--force', |
| action='store_true', |
| help='run even if the library is already up to date') |
| |
| |
| def Download(args): |
| ''' |
| Downloads the Google Play services library from a Google Cloud Storage bucket |
| and installs it to |
| //third_party/android_tools/sdk/extras/google/m2repository. |
| |
| A license check will be made, and the user might have to accept the license |
| if that has not been done before. |
| ''' |
| |
| if not os.path.isdir(args.sdk_root): |
| logging.debug('Did not find the Android SDK root directory at "%s".', |
| args.sdk_root) |
| if not args.force: |
| logging.info('Skipping, not on an android checkout.') |
| return 0 |
| |
| config = utils.ConfigParser(args.config) |
| paths = PlayServicesPaths(args.sdk_root, config.version_number, |
| config.clients) |
| |
| if os.path.isdir(paths.package) and not os.access(paths.package, os.W_OK): |
| logging.error('Failed updating the Google Play Services library. ' |
| 'The location is not writable. Please remove the ' |
| 'directory (%s) and try again.', paths.package) |
| return -2 |
| |
| new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, ZIP_FILE_NAME + '.sha1') |
| |
| logging.debug('Comparing zip hashes: %s and %s', new_lib_zip_sha1, |
| paths.lib_zip_sha1) |
| if utils.FileEquals(new_lib_zip_sha1, paths.lib_zip_sha1) and not args.force: |
| logging.info('Skipping, the Google Play services library is up to date.') |
| return 0 |
| |
| bucket_path = _VerifyBucketPathFormat(args.bucket, |
| config.version_number, |
| args.dry_run) |
| |
| with tempfile_ext.NamedTemporaryDirectory() as tmp_root: |
| # setup the destination directory |
| _MakeDirIfAbsent(paths.package) |
| |
| # download license file from bucket/{version_number}/license.sha1 |
| new_license = os.path.join(tmp_root, LICENSE_FILE_NAME) |
| |
| license_sha1 = os.path.join(SHA1_DIRECTORY, LICENSE_FILE_NAME + '.sha1') |
| _DownloadFromBucket(bucket_path, license_sha1, new_license, |
| args.verbose, args.dry_run) |
| |
| if (not _IsBotEnvironment() and |
| not _CheckLicenseAgreement(new_license, paths.license, |
| config.version_number)): |
| logging.warning('Your version of the Google Play services library is ' |
| 'not up to date. You might run into issues building ' |
| 'or running the app. Please run `%s download` to ' |
| 'retry downloading it.', __file__) |
| return 0 |
| |
| new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME) |
| _DownloadFromBucket(bucket_path, new_lib_zip_sha1, new_lib_zip, |
| args.verbose, args.dry_run) |
| |
| try: |
| # Remove the deprecated sdk directory. |
| deprecated_package_path = os.path.join(args.sdk_root, 'extras', 'google', |
| 'google_play_services') |
| if os.path.exists(deprecated_package_path): |
| shutil.rmtree(deprecated_package_path) |
| |
| # We remove the current version of the Google Play services SDK. |
| if os.path.exists(paths.package): |
| shutil.rmtree(paths.package) |
| os.makedirs(paths.package) |
| |
| logging.debug('Extracting the library to %s', paths.package) |
| with zipfile.ZipFile(new_lib_zip, "r") as new_lib_zip_file: |
| new_lib_zip_file.extractall(paths.package) |
| |
| logging.debug('Copying %s to %s', new_license, paths.license) |
| shutil.copy(new_license, paths.license) |
| |
| logging.debug('Copying %s to %s', new_lib_zip_sha1, paths.lib_zip_sha1) |
| shutil.copy(new_lib_zip_sha1, paths.lib_zip_sha1) |
| |
| logging.info('Update complete.') |
| |
| except Exception as e: # pylint: disable=broad-except |
| logging.error('Failed updating the Google Play Services library. ' |
| 'An error occurred while installing the new version in ' |
| 'the SDK directory: %s ', e) |
| return -3 |
| |
| return 0 |
| |
| |
| def _MakeDirIfAbsent(path): |
| try: |
| os.makedirs(path) |
| except OSError as e: |
| if e.errno != os.errno.EEXIST: |
| raise |
| |
| |
| def UpdateSdk(args): |
| ''' |
| Uses Maven to download the latest Google Play Services SDK. Its installation |
| path is //third_party/android_tools/sdk/extras/google/m2repository. |
| ''' |
| |
| # This should function should not run on bots and could fail for many user |
| # and setup related reasons. Also, exceptions here are not caught, so we |
| # disable breakpad to avoid spamming the logs. |
| breakpad.IS_ENABLED = False |
| |
| config = utils.ConfigParser(args.config) |
| target_repo = os.path.join(args.sdk_root, EXTRAS_GOOGLE_M2REPOSITORY) |
| |
| # Remove the old SDK. |
| # TODO(https://crbug.com/778650) not everything should be deleted. |
| shutil.rmtree(target_repo, ignore_errors=True) |
| |
| downloader = maven_downloader.MavenDownloader() |
| artifact_list = [ |
| 'com.google.android.gms:{}:{}:aar'.format(client, config.version_number) |
| for client in config.clients] |
| downloader.Install(target_repo, artifact_list) |
| return 0 |
| |
| |
| def Upload(args): |
| ''' |
| Uploads the library from the local Google Play services SDK to a Google Cloud |
| storage bucket. The version of the library and the list of clients to be |
| uploaded will be taken from the configuration file. (see --config parameter) |
| |
| By default, a local commit will be made at the end of the operation. |
| ''' |
| |
| # This should function should not run on bots and could fail for many user |
| # and setup related reasons. Also, exceptions here are not caught, so we |
| # disable breakpad to avoid spamming the logs. |
| breakpad.IS_ENABLED = False |
| |
| config = utils.ConfigParser(args.config) |
| paths = PlayServicesPaths(args.sdk_root, config.version_number, |
| config.clients) |
| logging.debug('-- Loaded paths --\n%s\n------------------', paths) |
| |
| with tempfile_ext.NamedTemporaryDirectory() as tmp_root: |
| new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME) |
| new_license = os.path.join(tmp_root, LICENSE_FILE_NAME) |
| |
| _ZipLibrary(new_lib_zip, paths.client_paths, paths.package) |
| _ExtractLicenseFile(new_license, paths.source_prop) |
| |
| bucket_path = _VerifyBucketPathFormat(args.bucket, config.version_number, |
| args.dry_run) |
| files_to_upload = [new_lib_zip, new_license] |
| logging.debug('Uploading %s to %s', files_to_upload, bucket_path) |
| _UploadToBucket(bucket_path, files_to_upload, args.dry_run) |
| |
| new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, |
| ZIP_FILE_NAME + '.sha1') |
| new_license_sha1 = os.path.join(SHA1_DIRECTORY, |
| LICENSE_FILE_NAME + '.sha1') |
| shutil.copy(new_lib_zip + '.sha1', new_lib_zip_sha1) |
| shutil.copy(new_license + '.sha1', new_license_sha1) |
| |
| logging.info('Update to version %s complete', config.version_number) |
| return 0 |
| |
| |
| def _DownloadFromBucket(bucket_path, sha1_file, destination, verbose, |
| is_dry_run): |
| '''Downloads the file designated by the provided sha1 from a cloud bucket.''' |
| |
| download_from_google_storage.download_from_google_storage( |
| input_filename=sha1_file, |
| base_url=bucket_path, |
| gsutil=_InitGsutil(is_dry_run), |
| num_threads=1, |
| directory=None, |
| recursive=False, |
| force=False, |
| output=destination, |
| ignore_errors=False, |
| sha1_file=sha1_file, |
| verbose=verbose, |
| auto_platform=True, |
| extract=False) |
| |
| |
| def _UploadToBucket(bucket_path, files_to_upload, is_dry_run): |
| '''Uploads the files designated by the provided paths to a cloud bucket. ''' |
| |
| upload_to_google_storage.upload_to_google_storage( |
| input_filenames=files_to_upload, |
| base_url=bucket_path, |
| gsutil=_InitGsutil(is_dry_run), |
| force=False, |
| use_md5=False, |
| num_threads=1, |
| skip_hashing=False, |
| gzip=None) |
| |
| |
| def _InitGsutil(is_dry_run): |
| '''Initialize the Gsutil object as regular or dummy version for dry runs. ''' |
| |
| if is_dry_run: |
| return DummyGsutil() |
| else: |
| return download_from_google_storage.Gsutil( |
| download_from_google_storage.GSUTIL_DEFAULT_PATH) |
| |
| |
| def _ExtractLicenseFile(license_path, prop_file_path): |
| with open(prop_file_path, 'r') as prop_file: |
| prop_file_content = prop_file.read() |
| |
| match = LICENSE_PATTERN.search(prop_file_content) |
| if not match: |
| raise AttributeError('The license was not found in ' + |
| os.path.abspath(prop_file_path)) |
| |
| with open(license_path, 'w') as license_file: |
| license_file.write(match.group('text')) |
| |
| |
| def _CheckLicenseAgreement(expected_license_path, actual_license_path, |
| version_number): |
| ''' |
| Checks that the new license is the one already accepted by the user. If it |
| isn't, it prompts the user to accept it. Returns whether the expected license |
| has been accepted. |
| ''' |
| |
| if utils.FileEquals(expected_license_path, actual_license_path): |
| return True |
| |
| with open(expected_license_path) as license_file: |
| # Uses plain print rather than logging to make sure this is not formatted |
| # by the logger. |
| print ('Updating the Google Play services SDK to ' |
| 'version %s.' % version_number) |
| |
| # The output is buffered when running as part of gclient hooks. We split |
| # the text here and flush is explicitly to avoid having part of it dropped |
| # out. |
| # Note: text contains *escaped* new lines, so we split by '\\n', not '\n'. |
| for license_part in license_file.read().split('\\n'): |
| print license_part |
| sys.stdout.flush() |
| |
| # Need to put the prompt on a separate line otherwise the gclient hook buffer |
| # only prints it after we received an input. |
| print ('Do you accept the license for version %s of the Google Play services ' |
| 'client library? [y/n]: ' % version_number) |
| sys.stdout.flush() |
| return raw_input('> ') in ('Y', 'y') |
| |
| |
| def _IsBotEnvironment(): |
| return bool(os.environ.get('CHROME_HEADLESS')) |
| |
| |
| def _VerifyBucketPathFormat(bucket_name, version_number, is_dry_run): |
| ''' |
| Formats and checks the download/upload path depending on whether we are |
| running in dry run mode or not. Returns a supposedly safe path to use with |
| Gsutil. |
| ''' |
| |
| if is_dry_run: |
| bucket_path = os.path.abspath(os.path.join(bucket_name, |
| str(version_number))) |
| if not os.path.isdir(bucket_path): |
| os.makedirs(bucket_path) |
| else: |
| if bucket_name.startswith('gs://'): |
| # We enforce the syntax without gs:// for consistency with the standalone |
| # download/upload scripts and to make dry run transition easier. |
| raise AttributeError('Please provide the bucket name without the gs:// ' |
| 'prefix (e.g. %s)' % GMS_CLOUD_STORAGE) |
| bucket_path = 'gs://%s/%s' % (bucket_name, version_number) |
| |
| return bucket_path |
| |
| def _ZipLibrary(zip_name, files, zip_root): |
| with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf: |
| for file_name in files: |
| zipf.write(file_name, os.path.relpath(file_name, zip_root)) |
| |
| |
| class PlayServicesPaths(object): |
| ''' |
| Describes the different paths to be used in the update process. |
| |
| Filesystem hierarchy | Exposed property / notes |
| ---------------------------------------------------|------------------------- |
| [sdk_root] | sdk_root / (1) |
| +- extras | |
| +- google | |
| +- m2repository | package / (2) |
| +- source.properties | source_prop / (3) |
| +- LICENSE | license / (4) |
| +- google_play_services_library.zip.sha1 | lib_zip_sha1 / (5) |
| +- com/google/android/gms/ | |
| +- [play-services-foo] | |
| +- [X.Y.Z] | |
| +- play-services-foo-X.Y.Z.aar | client_paths / (6) |
| |
| Notes: |
| |
| 1. sdk_root: Path provided as a parameter to the script (--sdk_root) |
| 2. package: This directory contains the Google Play services SDK itself. |
| When downloaded via the Android SDK manager, it will be a complete maven, |
| repository with the different versions of the library. When the update |
| script downloads the library from our cloud storage, it is cleared. |
| 3. source_prop: File created by the Android SDK manager that contains |
| the package information, such as the version info and the license. |
| 4. license: File created by the update script. Contains the license accepted |
| by the user. |
| 5. lib_zip_sha1: sha1 of the library that has been installed by the |
| update script. It is compared with the one required by the config file to |
| check if an update is necessary. |
| 6. client_paths: The client library jars we care about. They are zipped |
| zipped together and uploaded to the cloud storage. |
| |
| ''' |
| |
| def __init__(self, sdk_root, version_number, client_names): |
| ''' |
| sdk_root: path to the root of the sdk directory |
| version_number: version of the library supposed to be installed locally. |
| client_names: names of client libraries to be uploaded. See |
| utils.ConfigParser for more info. |
| ''' |
| self.sdk_root = sdk_root |
| self.version_number = version_number |
| |
| self.package = os.path.join(sdk_root, EXTRAS_GOOGLE_M2REPOSITORY) |
| self.lib_zip_sha1 = os.path.join(self.package, ZIP_FILE_NAME + '.sha1') |
| self.license = os.path.join(self.package, LICENSE_FILE_NAME) |
| self.source_prop = os.path.join(self.package, 'source.properties') |
| |
| self.client_paths = [] |
| for client in client_names: |
| self.client_paths.append(os.path.join( |
| self.package, COM_GOOGLE_ANDROID_GMS, client, version_number, |
| '{}-{}.aar'.format(client, version_number))) |
| |
| def __repr__(self): |
| return ("\nsdk_root: " + self.sdk_root + |
| "\nversion_number: " + self.version_number + |
| "\npackage: " + self.package + |
| "\nlib_zip_sha1: " + self.lib_zip_sha1 + |
| "\nlicense: " + self.license + |
| "\nsource_prop: " + self.source_prop + |
| "\nclient_paths: \n - " + '\n - '.join(self.client_paths)) |
| |
| |
| class DummyGsutil(download_from_google_storage.Gsutil): |
| ''' |
| Class that replaces Gsutil to use a local directory instead of an online |
| bucket. It relies on the fact that Gsutil commands are very similar to shell |
| ones, so for the ones used here (ls, cp), it works to just use them with a |
| local directory. |
| ''' |
| |
| def __init__(self): |
| super(DummyGsutil, self).__init__( |
| download_from_google_storage.GSUTIL_DEFAULT_PATH) |
| |
| def call(self, *args): |
| logging.debug('Calling command "%s"', str(args)) |
| return cmd_helper.GetCmdStatusOutputAndError(args) |
| |
| def check_call(self, *args): |
| logging.debug('Calling command "%s"', str(args)) |
| return cmd_helper.GetCmdStatusOutputAndError(args) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
