blob: c7018685a05c18f2168aa1559e581c2824e84b2d [file] [log] [blame]
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import datetime
import multiprocessing
import optparse
import os
import re
import sys
import tempfile
import time
if __name__ == '__main__':
import constants
from chromite.lib import cros_build_lib
from chromite.lib.binpkg import (GrabLocalPackageIndex, GrabRemotePackageIndex)
This script is used to upload host prebuilts as well as board BINHOSTS.
If the URL starts with 'gs://', we upload using gsutil to Google Storage.
Otherwise, rsync is used.
After a build is successfully uploaded a file is updated with the proper
BINHOST version as well as the target board. This file is defined in GIT_FILE
To read more about prebuilts/binhost binary packages please refer to:
Example of uploading prebuilt amd64 host files to Google Storage:
./ -p /b/cbuild/build -s -u gs://chromeos-prebuilt
Example of uploading x86-dogfood binhosts to Google Storage:
./ -b x86-dogfood -p /b/cbuild/build/ -u gs://chromeos-prebuilt -g
Example of uploading prebuilt amd64 host files using rsync:
./ -p /b/cbuild/build -s -u codf30.jail:/tmp
# as per always filter the below packages
_GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil'
_HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
_CATEGORIES_PATH = 'chroot/etc/portage/categories'
_HOST_TARGET = 'amd64'
_BOARD_PATH = 'chroot/build/%(board)s'
# board/board-target/version/'
_REL_BOARD_PATH = 'board/%(board)s/%(version)s'
# host/host-target/version/'
_REL_HOST_PATH = 'host/%(target)s/%(version)s'
# Private overlays to look at for builds to filter
# relative to build path
_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
_GOOGLESTORAGE_ACL_FILE = 'googlestorage_acl.xml'
_PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
# Created in the event of new host targets becoming available
_PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'
class FiltersEmpty(Exception):
"""Raised when filters are used but none are found."""
class UploadFailed(Exception):
"""Raised when one of the files uploaded failed."""
class UnknownBoardFormat(Exception):
"""Raised when a function finds an unknown board format."""
def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
"""Update the key in file with the value passed.
File format:
Note quotes are added automatically
filename: Name of file to modify.
value: Value to write with the key.
key: The variable key to update. (Default: PORTAGE_BINHOST)
if os.path.exists(filename):
file_fh = open(filename)
file_fh = open(filename, 'w+')
file_lines = []
found = False
keyval_str = '%(key)s=%(value)s'
for line in file_fh:
# Strip newlines from end of line. We already add newlines below.
line = line.rstrip("\n")
if len(line.split('=')) != 2:
# Skip any line that doesn't fit key=val.
file_var, file_val = line.split('=')
if file_var == key:
found = True
print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value)
value = '"%s"' % value
file_lines.append(keyval_str % {'key': key, 'value': value})
file_lines.append(keyval_str % {'key': file_var, 'value': file_val})
if not found:
file_lines.append(keyval_str % {'key': key, 'value': value})
# write out new file
new_file_fh = open(filename, 'w')
new_file_fh.write('\n'.join(file_lines) + '\n')
def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'):
"""Update and push the git file.
filename: file to modify that is in a git repo already
value: string representing the version of the prebuilt that has been
retries: The number of times to retry before giving up, default: 5
key: The variable key to update in the git file.
prebuilt_branch = 'prebuilt_branch'
cwd = os.path.abspath(os.path.dirname(filename))
commit = cros_build_lib.RunCommand(['git', 'rev-parse', 'HEAD'], cwd=cwd,
git_ssh_config_cmd = [
cros_build_lib.RunCommand(git_ssh_config_cmd, cwd=cwd)
cros_build_lib.RunCommand(['git', 'remote', 'update'], cwd=cwd)
cros_build_lib.RunCommand(['repo', 'start', prebuilt_branch, '.'], cwd=cwd)
description = 'Update %s="%s" in %s' % (key, value, filename)
print description
UpdateLocalFile(filename, value, key)
cros_build_lib.RunCommand(['git', 'config', 'push.default', 'tracking'],
cros_build_lib.RunCommand(['git', 'add', filename], cwd=cwd)
cros_build_lib.RunCommand(['git', 'commit', '-m', description], cwd=cwd)
cros_build_lib.GitPushWithRetry(prebuilt_branch, cwd=cwd)
cros_build_lib.RunCommand(['git', 'checkout', commit], cwd=cwd)
cros_build_lib.RunCommand(['repo', 'abandon', 'prebuilt_branch', '.'],
def GetVersion():
"""Get the version to put in LATEST and update the git version with."""
def LoadPrivateFilters(build_path):
"""Load private filters based on ebuilds found under _PRIVATE_OVERLAY_DIR.
This function adds filters to the global set _FILTER_PACKAGES.
build_path: Path that _PRIVATE_OVERLAY_DIR is in.
# TODO(scottz): eventually use manifest.xml to find the proper
# private overlay path.
filter_path = os.path.join(build_path, _PRIVATE_OVERLAY_DIR)
files = cros_build_lib.ListFiles(filter_path)
filters = []
for file in files:
if file.endswith('.ebuild'):
basename = os.path.basename(file)
match = re.match('(.*?)-\d.*.ebuild', basename)
if match:
if not filters:
raise FiltersEmpty('No filters were returned')
def ShouldFilterPackage(file_path):
"""Skip a particular file if it matches a pattern.
Skip any files that machine the list of packages to filter in
file_path: string of a file path to inspect against _FILTER_PACKAGES
True if we should filter the package,
False otherwise.
for name in _FILTER_PACKAGES:
if name in file_path:
print 'FILTERING %s' % file_path
return True
return False
def _RetryRun(cmd, print_cmd=True, cwd=None):
"""Run the specified command, retrying if necessary.
cmd: The command to run.
print_cmd: Whether to print out the cmd.
shell: Whether to treat the command as a shell.
cwd: Working directory to run command in.
True if the command succeeded. Otherwise, returns False.
# TODO(scottz): port to use _Run or similar when it is available in
# cros_build_lib.
for attempt in range(_RETRIES):
output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd,
return True
except cros_build_lib.RunCommandError:
print 'Failed to run %r' % cmd
print 'Retry failed run %r, giving up' % cmd
return False
def _GsUpload(args):
"""Upload to GS bucket.
args: a tuple of three arguments that contains local_file, remote_file, and
the acl used for uploading the file.
Return the arg tuple of two if the upload failed
(local_file, remote_file, acl) = args
CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
'authenticated-read', 'bucket-owner-full-control',
acl_cmd = None
if acl in CANNED_ACLS:
cmd = [_GSUTIL_BIN, 'cp', '-a', acl, local_file, remote_file]
# For private uploads we assume that the overlay board is set up properly
# and a googlestore_acl.xml is present, if not this script errors
cmd = [_GSUTIL_BIN, 'cp', '-a', 'private', local_file, remote_file]
if not os.path.exists(acl):
print >> sys.stderr, ('You are specifying either a file that does not '
'exist or an unknown canned acl: %s. Aborting '
'upload') % acl
# emulate the failing of an upload since we are not uploading the file
return (local_file, remote_file)
acl_cmd = [_GSUTIL_BIN, 'setacl', acl, remote_file]
if not _RetryRun(cmd, print_cmd=False):
return (local_file, remote_file)
if acl_cmd:
# Apply the passed in ACL xml file to the uploaded object.
_RetryRun(acl_cmd, print_cmd=False)
def RemoteUpload(acl, files, pool=10):
"""Upload to google storage.
Create a pool of process and call _GsUpload with the proper arguments.
acl: The canned acl used for uploading. acl can be one of: "public-read",
"public-read-write", "authenticated-read", "bucket-owner-read",
"bucket-owner-full-control", or "private".
files: dictionary with keys to local files and values to remote path.
pool: integer of maximum proesses to have at the same time.
Return a set of tuple arguments of the failed uploads
# TODO(scottz) port this to use _RunManyParallel when it is available in
# cros_build_lib
pool = multiprocessing.Pool(processes=pool)
workers = []
for local_file, remote_path in files.iteritems():
workers.append((local_file, remote_path, acl))
result = pool.map_async(_GsUpload, workers, chunksize=1)
while True:
return set(result.get(60 * 60))
except multiprocessing.TimeoutError:
def GenerateUploadDict(base_local_path, base_remote_path, pkgs):
"""Build a dictionary of local remote file key pairs to upload.
base_local_path: The base path to the files on the local hard drive.
remote_path: The base path to the remote paths.
pkgs: The packages to upload.
Returns a dictionary of local_path/remote_path pairs
upload_files = {}
for pkg in pkgs:
suffix = pkg['CPV'] + '.tbz2'
local_path = os.path.join(base_local_path, suffix)
assert os.path.exists(local_path)
remote_path = '%s/%s' % (base_remote_path.rstrip('/'), suffix)
upload_files[local_path] = remote_path
return upload_files
def GetBoardPathFromCrosOverlayList(build_path, target):
"""Use the cros_overlay_list to determine the path to the board overlay
build_path: The path to the root of the build directory
target: The target that we are looking for, could consist of board and
board_variant, we handle that properly
The last line from cros_overlay_list as a string
script_dir = os.path.join(build_path, 'src/platform/dev/host')
cmd = ['./cros_overlay_list']
if re.match('.*?_.*', target):
(board, variant) = target.split('_')
cmd += ['--board', board, '--variant', variant]
elif re.match('.*?-\w+', target):
cmd += ['--board', target]
raise UnknownBoardFormat('Unknown format: %s' % target)
cmd_output = cros_build_lib.RunCommand(cmd, redirect_stdout=True,
# We only care about the last entry
return cmd_output.output.splitlines().pop()
def DeterminePrebuiltConfFile(build_path, target):
"""Determine the prebuilt.conf file that needs to be updated for prebuilts.
build_path: The path to the root of the build directory
target: String representation of the board. This includes host and board
A string path to a prebuilt.conf file to be updated.
if _HOST_TARGET == target:
# We are host.
# Without more examples of hosts this is a kludge for now.
# TODO(Scottz): as new host targets come online expand this to
# work more like boards.
make_path = _PREBUILT_MAKE_CONF[target]
# We are a board
board = GetBoardPathFromCrosOverlayList(build_path, target)
make_path = os.path.join(board, 'prebuilt.conf')
return make_path
def UpdateBinhostConfFile(path, key, value):
"""Update binhost config file file with key=value.
path: Filename to update.
key: Key to update.
value: New value for key.
cwd = os.path.dirname(os.path.abspath(path))
filename = os.path.basename(path)
if not os.path.isdir(cwd):
if not os.path.isfile(path):
config_file = file(path, 'w')
UpdateLocalFile(path, value, key)
cros_build_lib.RunCommand(['git', 'add', filename], cwd=cwd)
description = 'Update %s=%s in %s' % (key, value, filename)
cros_build_lib.RunCommand(['git', 'commit', '-m', description], cwd=cwd)
def _GrabAllRemotePackageIndexes(binhost_urls):
"""Grab all of the packages files associated with a list of binhost_urls.
binhost_urls: The URLs for the directories containing the Packages files we
want to grab.
A list of PackageIndex objects.
pkg_indexes = []
for url in binhost_urls:
pkg_index = GrabRemotePackageIndex(url)
if pkg_index:
return pkg_indexes
class PrebuiltUploader(object):
"""Synchronize host and board prebuilts."""
def __init__(self, upload_location, acl, binhost_base_url, pkg_indexes):
"""Constructor for prebuilt uploader object.
This object can upload host or prebuilt files to Google Storage.
upload_location: The upload location.
acl: The canned acl used for uploading to Google Storage. acl can be one
of: "public-read", "public-read-write", "authenticated-read",
"bucket-owner-read", "bucket-owner-full-control", or "private". If
we are not uploading to Google Storage, this parameter is unused.
binhost_base_url: The URL used for downloading the prebuilts.
pkg_indexes: Old uploaded prebuilts to compare against. Instead of
uploading duplicate files, we just link to the old files.
self._upload_location = upload_location
self._acl = acl
self._binhost_base_url = binhost_base_url
self._pkg_indexes = pkg_indexes
def _UploadPrebuilt(self, package_path, url_suffix):
"""Upload host or board prebuilt files to Google Storage space.
package_path: The path to the packages dir.
url_suffix: The remote subdirectory where we should upload the packages.
# Process Packages file, removing duplicates and filtered packages.
pkg_index = GrabLocalPackageIndex(package_path)
pkg_index.SetUploadLocation(self._binhost_base_url, url_suffix)
uploads = pkg_index.ResolveDuplicateUploads(self._pkg_indexes)
# Write Packages file.
tmp_packages_file = pkg_index.WriteToNamedTemporaryFile()
remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
if remote_location.startswith('gs://'):
# Build list of files to upload.
upload_files = GenerateUploadDict(package_path, remote_location, uploads)
remote_file = '%s/Packages' % remote_location.rstrip('/')
upload_files[] = remote_file
failed_uploads = RemoteUpload(self._acl, upload_files)
if len(failed_uploads) > 1 or (None not in failed_uploads):
error_msg = ['%s -> %s\n' % args for args in failed_uploads if args]
raise UploadFailed('Error uploading:\n%s' % error_msg)
pkgs = [p['CPV'] + '.tbz2' for p in uploads]
ssh_server, remote_path = remote_location.split(':', 1)
remote_path = remote_path.rstrip('/')
pkg_index =
remote_location = remote_location.rstrip('/')
remote_packages = '%s/Packages' % remote_location
cmds = [['ssh', ssh_server, 'mkdir', '-p', remote_path],
['rsync', '-av', '--chmod=a+r', pkg_index, remote_packages]]
if pkgs:
cmds.append(['rsync', '-Rav'] + pkgs + [remote_location + '/'])
for cmd in cmds:
if not _RetryRun(cmd, cwd=package_path):
raise UploadFailed('Could not run %r' % cmd)
def _UploadBoardTarball(self, board_path, url_suffix):
"""Upload a tarball of the board at the specified path to Google Storage.
board_path: The path to the board dir.
url_suffix: The remote subdirectory where we should upload the packages.
remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
assert remote_location.startswith('gs://')
cwd, boardname = os.path.split(board_path.rstrip(os.path.sep))
tmpdir = tempfile.mkdtemp()
tarfile = os.path.join(tmpdir, '%s.tbz2' % boardname)
cmd = ['sudo', 'tar', '-I', 'pbzip2', '-cf', tarfile]
excluded_paths = ('usr/lib/debug', 'usr/local/autotest', 'packages',
for path in excluded_paths:
cmd.append('--exclude=%s/%s/*' % (boardname, path))
cros_build_lib.RunCommand(cmd, cwd=cwd)
remote_tarfile = '%s/%s.tbz2' % (remote_location.rstrip('/'), boardname)
if _GsUpload((tarfile, remote_tarfile, self._acl)):
cros_build_lib.RunCommand(['sudo', 'rm', '-rf', tmpdir], cwd=cwd)
def _SyncHostPrebuilts(self, build_path, version, key, git_sync,
"""Synchronize host prebuilt files.
This function will sync both the standard host packages, plus the host
packages associated with all targets that have been "setup" with the
current host's chroot. For instance, if this host has been used to build
x86-generic, it will sync the host packages associated with
'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
it will also sync the host packages associated with
build_path: The path to the directory containing the chroot.
version: A unique string, intended to be included in the upload path,
which identifies the version number of the uploaded prebuilts.
key: The variable key to update in the git file.
git_sync: If set, update make.conf of target to reference the latest
prebuilt packages generated here.
sync_binhost_conf: If set, update binhost config file in
chromiumos-overlay for the host.
# Upload prebuilts.
package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
self._UploadPrebuilt(package_path, packages_url_suffix)
# Record URL where prebuilts were uploaded.
url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
if git_sync:
git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
RevGitFile(git_file, url_value, key=key)
if sync_binhost_conf:
binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host',
'%s-%s.conf' % (_HOST_TARGET, key))
UpdateBinhostConfFile(binhost_conf, key, url_value)
def _SyncBoardPrebuilts(self, board, build_path, version, key, git_sync,
sync_binhost_conf, upload_board_tarball):
"""Synchronize board prebuilt files.
board: The board to upload to Google Storage.
build_path: The path to the directory containing the chroot.
version: A unique string, intended to be included in the upload path,
which identifies the version number of the uploaded prebuilts.
key: The variable key to update in the git file.
git_sync: If set, update make.conf of target to reference the latest
prebuilt packages generated here.
sync_binhost_conf: If set, update binhost config file in
chromiumos-overlay for the current board.
upload_board_tarball: Include a tarball of the board in our upload.
board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
package_path = os.path.join(board_path, 'packages')
url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
# Upload board tarballs in the background.
if upload_board_tarball:
tar_process = multiprocessing.Process(target=self._UploadBoardTarball,
args=(board_path, url_suffix))
# Upload prebuilts.
self._UploadPrebuilt(package_path, packages_url_suffix)
# Make sure we finished uploading the board tarballs.
if upload_board_tarball:
assert tar_process.exitcode == 0
# Record URL where prebuilts were uploaded.
url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
if git_sync:
git_file = DeterminePrebuiltConfFile(build_path, board)
RevGitFile(git_file, url_value, key=key)
if sync_binhost_conf:
binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target',
'%s-%s.conf' % (board, key))
UpdateBinhostConfFile(binhost_conf, key, url_value)
def usage(parser, msg):
"""Display usage message and parser help then exit with 1."""
print >> sys.stderr, msg
def ParseOptions():
parser = optparse.OptionParser()
parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url',
help='Base URL to use for binhost in make.conf updates')
parser.add_option('', '--previous-binhost-url', action='append',
default=[], dest='previous_binhost_url',
help='Previous binhost URL')
parser.add_option('-b', '--board', dest='board', default=None,
help='Board type that was built on this machine')
parser.add_option('-p', '--build-path', dest='build_path',
help='Path to the directory containing the chroot')
parser.add_option('-s', '--sync-host', dest='sync_host',
default=False, action='store_true',
help='Sync host prebuilts')
parser.add_option('-g', '--git-sync', dest='git_sync',
default=False, action='store_true',
help='Enable git version sync (This commits to a repo)')
parser.add_option('-u', '--upload', dest='upload',
help='Upload location')
parser.add_option('-V', '--prepend-version', dest='prepend_version',
help='Add an identifier to the front of the version')
parser.add_option('-f', '--filters', dest='filters', action='store_true',
help='Turn on filtering of private ebuild packages')
parser.add_option('-k', '--key', dest='key',
help='Key to update in make.conf / binhost.conf')
parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
default=False, action='store_true',
help='Update binhost.conf')
parser.add_option('-P', '--private', dest='private', action='store_true',
default=False, help='Mark gs:// uploads as private.')
parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
action='store_true', default=False,
help='Upload board tarball to Google Storage.')
options, args = parser.parse_args()
if not options.build_path:
usage(parser, 'Error: you need provide a chroot path')
if not options.upload:
usage(parser, 'Error: you need to provide an upload location using -u')
if options.upload_board_tarball and not options.upload.startswith('gs://'):
usage(parser, 'Error: --upload-board-tarball only works with gs:// URLs.\n'
'--upload must be a gs:// URL.')
if options.private:
if options.sync_host:
usage(parser, 'Error: --private and --sync-host/-s cannot be specified '
'together, we do not support private host prebuilts')
if not options.upload.startswith('gs://'):
usage(parser, 'Error: --private is only valid for gs:// URLs.\n'
'--upload must be a gs:// URL.')
if options.binhost_base_url != _BINHOST_BASE_URL:
usage(parser, 'Error: when using --private the --binhost-base-url '
'is automatically derived.')
return options
def main():
options = ParseOptions()
if options.filters:
# Calculate a list of Packages index files to compare against. Whenever we
# upload a package, we check to make sure it's not already stored in one of
# the packages files we uploaded. This list of packages files might contain
# both board and host packages.
pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url)
version = GetVersion()
if options.prepend_version:
version = '%s-%s' % (options.prepend_version, version)
acl = 'public-read'
binhost_base_url = options.binhost_base_url
if options.private:
binhost_base_url = options.upload
board_path = GetBoardPathFromCrosOverlayList(options.build_path,
acl = os.path.join(board_path, _GOOGLESTORAGE_ACL_FILE)
uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
if options.sync_host:
uploader._SyncHostPrebuilts(options.build_path, version, options.key,
options.git_sync, options.sync_binhost_conf)
if options.board:
uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
options.key, options.git_sync,
if __name__ == '__main__':