blob: af03d4cdef243ec664d7721082a3283754188fe1 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A tool to extract a build, executed by a buildbot slave.
"""
import optparse
import os
import shutil
import sys
import traceback
import urllib
from common import chromium_utils
import build_directory
import slave_utils
class ExtractHandler(object):
def __init__(self, url, archive_name):
self.url = url
self.archive_name = archive_name
class GSHandler(ExtractHandler):
def __init__(self, url, archive_name, gsutil_py_path=None):
super(GSHandler, self).__init__(url, archive_name)
self.gsutil_py_path = gsutil_py_path
def download(self):
override_gsutil = None
if self.gsutil_py_path:
override_gsutil = [sys.executable, self.gsutil_py_path]
status = slave_utils.GSUtilCopy(
self.url, '.', override_gsutil=override_gsutil
)
if 0 != status:
return False
try:
shutil.move(os.path.basename(self.url), self.archive_name)
except OSError:
os.remove(self.archive_name)
shutil.move(os.path.basename(self.url), self.archive_name)
return True
class WebHandler(ExtractHandler):
@chromium_utils.RunAndPrintDots
def download(self):
try:
rc = urllib.urlretrieve(self.url, self.archive_name)
print '\nDownload complete'
except IOError:
print '\nFailed to download build'
return False
return rc
def GetBuildUrl(options, build_revision):
"""Compute the url to download the build from. This will use as a base
string, in order of preference:
0) options.build_archive_url
1) options.build_url
2) options.build_properties.build_url
3) build url constructed from build_properties. This last type of
construction is not compatible with the 'force build' button.
Args:
options: options object as specified by parser below.
build_revision: Revision for the build.
"""
if options.build_archive_url:
return options.build_archive_url, None
base_filename, version_suffix = slave_utils.GetZipFileNames(
options.builder_group,
options.build_number,
options.parent_build_number,
build_revision,
extract=True
)
replace_dict = {
'base_filename': base_filename,
'parentname': options.parent_builder_name,
'parentslavename': options.parent_slave_name,
'parent_builddir': options.parent_build_dir,
}
# If builddir isn't specified, assume buildbot used the builder name
# as the root folder for the build.
if not replace_dict.get('parent_builddir') and replace_dict.get('parentname'):
replace_dict['parent_builddir'] = replace_dict.get('parentname', '')
url = options.build_url
if not url:
url = (
'http://%(parentslavename)s/b/build/slave/%(parent_builddir)s/'
'chrome_staging'
)
if url[-4:] != '.zip': # assume filename not specified
# Append the filename to the base URL. First strip any trailing slashes.
url = url.rstrip('/')
url = '%s/%s' % (url, '%(base_filename)s.zip')
url = url % replace_dict
archive_name = url.split('/')[-1]
versioned_url = url.replace('.zip', version_suffix + '.zip')
return versioned_url, archive_name
def real_main(options):
""" Download a build, extract it to build\\BuildDir\\full-build-win32
and rename it to build\\BuildDir\\Target
"""
abs_build_dir = os.path.abspath(
build_directory.GetBuildOutputDirectory(options.src_dir)
)
target_build_output_dir = os.path.join(abs_build_dir, options.target)
# Generic name for the archive.
archive_name = 'full-build-%s.zip' % chromium_utils.PlatformName()
# Just take the zip off the name for the output directory name.
output_dir = os.path.join(abs_build_dir, archive_name.replace('.zip', ''))
src_dir = os.path.dirname(abs_build_dir)
if not options.build_revision and not options.build_archive_url:
build_revision = slave_utils.GetBuildRevisions(
src_dir, revision_dir=options.revision_dir
)
else:
build_revision = options.build_revision
url, archive_name = GetBuildUrl(options, build_revision)
if archive_name is None:
archive_name = 'build.zip'
base_url = None
else:
base_url = '/'.join(url.split('/')[:-1] + [archive_name])
if url.startswith('gs://'):
handler = GSHandler(
url=url,
archive_name=archive_name,
gsutil_py_path=options.gsutil_py_path
)
else:
handler = WebHandler(url=url, archive_name=archive_name)
# We try to download and extract 3 times.
for tries in range(1, 4):
print 'Try %d: Fetching build from %s...' % (tries, url)
failure = False
# If the url is valid, we download the file.
if not failure:
if not handler.download():
return slave_utils.ERROR_EXIT_CODE
# If the versioned url failed, we try to get the latest build.
if failure:
if url.startswith('gs://') or not base_url:
continue
else:
print 'Fetching latest build at %s' % base_url
base_handler = handler.__class__(base_url, handler.archive_name)
if not base_handler.download():
continue
print 'Extracting build %s to %s...' % (archive_name, abs_build_dir)
try:
chromium_utils.RemoveDirectory(target_build_output_dir)
chromium_utils.ExtractZip(archive_name, abs_build_dir)
# For Chrome builds, the build will be stored in chrome-win32.
if 'full-build-win32' in output_dir:
chrome_dir = output_dir.replace('full-build-win32', 'chrome-win32')
if os.path.exists(chrome_dir):
output_dir = chrome_dir
print 'Moving build from %s to %s' % (output_dir, target_build_output_dir)
shutil.move(output_dir, target_build_output_dir)
except (OSError, IOError, chromium_utils.ExternalError):
print 'Failed to extract the build.'
# Print out the traceback in a nice format
traceback.print_exc()
# Try again...
continue
# If we got the latest build, then figure out its revision number.
if failure:
print "Trying to determine the latest build's revision number..."
try:
build_revision_file_name = os.path.join(
target_build_output_dir, chromium_utils.FULL_BUILD_REVISION_FILENAME
)
build_revision_file = open(build_revision_file_name, 'r')
print 'Latest build is revision: %s' % build_revision_file.read()
build_revision_file.close()
except IOError:
print "Could not determine the latest build's revision number"
if failure:
# We successfully extracted the archive, but it was the generic one.
return slave_utils.WARNING_EXIT_CODE
return 0
# If we get here, that means that it failed 3 times. We return a failure.
return slave_utils.ERROR_EXIT_CODE
def main():
option_parser = optparse.OptionParser()
option_parser.add_option(
'--target', help='build target to archive (Debug or Release)'
)
option_parser.add_option(
'--src-dir',
default='src',
help='path to the top-level sources directory'
)
option_parser.add_option('--build-dir', help='ignored')
option_parser.add_option('--builder-group', help='Name of the builder group.')
option_parser.add_option(
'--build-number', type=int, help='Buildbot build number.'
)
option_parser.add_option(
'--parent-build-dir',
help='Path to build directory on parent buildbot '
'builder.'
)
option_parser.add_option(
'--parent-builder-name', help='Name of parent buildbot builder.'
)
option_parser.add_option(
'--parent-slave-name', help='Name of parent buildbot slave.'
)
option_parser.add_option(
'--parent-build-number', type=int, help='Buildbot parent build number.'
)
option_parser.add_option(
'--build-url', help='Base url where to find the build to extract'
)
option_parser.add_option(
'--build-archive-url',
help='Exact url where to find the build to extract'
)
option_parser.add_option(
'--build_revision',
help='Revision of the build that is being '
'archived. Overrides the revision found on '
'the local disk'
)
option_parser.add_option(
'--revision-dir',
help=(
'Directory path that shall be used to decide '
'the revision number for the archive, '
'relative to the src/ dir.'
)
)
option_parser.add_option('--build-output-dir', help='ignored')
option_parser.add_option(
'--gsutil-py-path', help='Specify path to gsutil.py script.'
)
chromium_utils.AddPropertiesOptions(option_parser)
slave_utils_callback = slave_utils.AddOpts(option_parser)
options, args = option_parser.parse_args()
if args:
print 'Unknown options: %s' % args
return 1
slave_utils_callback(options)
if not options.builder_group:
options.builder_group = options.build_properties.get('builder_group', '')
if not options.build_number:
options.build_number = options.build_properties.get('buildnumber')
if not options.parent_build_dir:
options.parent_build_dir = options.build_properties.get('parent_builddir')
if not options.parent_builder_name:
options.parent_builder_name = options.build_properties.get('parentname')
if not options.parent_slave_name:
options.parent_slave_name = options.build_properties.get('parentslavename')
if not options.parent_build_number:
options.parent_build_number = int_if_given(
options.build_properties.get('parent_buildnumber')
)
if not options.build_url:
options.build_url = options.build_properties.get('build_url')
if not options.target:
options.target = options.build_properties.get('target', 'Release')
if not options.revision_dir:
options.revision_dir = options.build_properties.get('revision_dir')
options.src_dir = (
options.build_properties.get('extract_build_src_dir') or options.src_dir
)
return real_main(options)
def int_if_given(value):
if value is None:
return None
return int(value)
if '__main__' == __name__:
sys.exit(main())