blob: 9f73a7de33ed9e8d6952636c1e198e2018612cf4 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import glob
import hashlib
import os
import shutil
import subprocess
import sys
import bootstrap
from util import ROOT, WHEELHOUSE, WHEELS_URL, SOURCE_URL, LOCAL_STORAGE_PATH
from util import tempdir, tempname, platform_tag, merge_deps, print_deps
REPO_HOST = 'git+https://chromium.googlesource.com/'
GSUTIL = os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'depot_tools', 'gsutil.py')
def has_custom_build(name):
return os.path.exists(os.path.join(ROOT, 'custom_builds', name + '.py'))
def pip(*args, **kwargs):
bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
subprocess.check_call(
(os.path.join(sys.prefix, bin_dir, 'pip'),) + args, **kwargs)
def wheel(arg, source_sha, build, build_options):
with tempdir() as tdir:
args = ['wheel', '--no-index', '--no-deps', '--wheel-dir', tdir]
for op in build_options:
args += ['--global-option', op]
args += [arg]
pip(*args)
grab_wheel(tdir, WHEELHOUSE, source_sha, build)
def grab_wheel(src, dst, source_sha, build):
# late import lets us grab the virtualenv pip
from pip.wheel import Wheel # pylint: disable=E0611
items = os.listdir(src)
assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items
wheelfile = items[0]
wheel_info = Wheel.wheel_file_re.match(wheelfile)
assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile
plat_tag = ''
if not wheelfile.endswith('none-any.whl'):
plat_tag = platform_tag()
shutil.copyfile(
os.path.join(src, wheelfile),
os.path.join(dst, '{}-{}_{}{}{}'.format(
wheel_info.group('namever'),
build,
source_sha,
plat_tag,
wheel_info.group(4),
)))
def run_custom_build(name, link, sha, build):
from pip.index import Link
from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url
assert has_custom_build(name)
link = Link(link, trusted=True)
unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url
with tempdir() as tmpd, tempdir() as wheeld: # pylint: disable=C0321
unpack(link, tmpd)
m = getattr(__import__('custom_builds.%s' % (name,)), name)
m.Build(tmpd, wheeld)
grab_wheel(wheeld, WHEELHOUSE, sha, build)
def process_git(name, rev, build, build_options, repo):
print
print 'Processing (git)', name, rev
url = repo + '@' + rev
if not url.startswith('git+https://'):
url = REPO_HOST + url
if not has_custom_build(name):
wheel(url, rev, build, build_options)
else:
run_custom_build(name, url, rev, build)
def process_gs(name, sha_ext, build, build_options):
print
print 'Processing (gs)', name, sha_ext
sha, ext = sha_ext.split('.', 1)
with tempname(ext) as tmp:
link = 'file://' + tmp
subprocess.check_call([
sys.executable, GSUTIL, '--force-version', '4.7', 'cp',
SOURCE_URL.format(sha_ext), tmp])
with open(tmp, 'rb') as f:
assert hashlib.sha1(f.read()).hexdigest() == sha
if not has_custom_build(name):
wheel(link, sha, build, build_options)
else:
run_custom_build(name, link, sha, build)
def clear_wheelhouse():
shutil.rmtree(WHEELHOUSE, ignore_errors=True)
try:
os.makedirs(WHEELHOUSE)
except OSError:
pass
def copy_wheel_to_local_cache():
if not os.path.isdir(LOCAL_STORAGE_PATH):
os.makedirs(LOCAL_STORAGE_PATH)
for filename in glob.glob(os.path.join(WHEELHOUSE, '*')):
shutil.copy(filename, LOCAL_STORAGE_PATH)
def push_wheelhouse():
status = subprocess.call([
sys.executable, GSUTIL, '--force-version', '4.7', '-m', 'cp',
os.path.join(WHEELHOUSE, '*'), WHEELS_URL])
if status:
print('Failed to upload wheels, falling back to local cache')
copy_wheel_to_local_cache()
return status
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument(
'--upload', action='store_true',
help='Upload wheelhouse instead of copying to local cache')
parser.add_argument(
'--deps_file', action='append',
help='Path to deps.pyl file (default: bootstrap/deps.pyl)')
parser.add_argument(
'to_build', nargs='*',
help='Names of packages to build. Defaults to all packages.')
opts = parser.parse_args(args)
if 'Ubuntu' in platform_tag() and ROOT.startswith('/usr/local/'):
print >> sys.stderr, "\n".join([
"Due to a bug in Ubuntu's python distribution, build_deps.py does not",
"work when run from under a path beginning with /usr/local/. Please ",
"clone to a different path, and try again.",
"",
"Bug: https://github.com/pypa/virtualenv/issues/118"
])
return 1
deps_files = opts.deps_file or [os.path.join(ROOT, 'deps.pyl')]
to_build = set(opts.to_build)
build_env = os.path.join(ROOT, 'BUILD_ENV')
print 'Parsing deps.pyl'
deps = merge_deps(deps_files)
bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})
print 'Finding missing deps'
missing_deps = {}
for name, entry in deps.iteritems():
if to_build and name not in to_build:
continue
try:
bootstrap.get_links({name: entry})
except bootstrap.NoWheelException:
missing_deps[name] = entry
if not missing_deps:
print 'Nothing to process'
return
print 'Processing deps:'
print_deps(missing_deps)
for name, options in missing_deps.iteritems():
clear_wheelhouse()
# TODO(iannucci): skip entries which already exist in gs
if 'repo' in options and 'rev' in options:
process_git(name, options['rev'], options['build'],
options.get('build_options', ()),
options['repo'])
elif 'gs' in options:
process_gs(name, options['gs'], options['build'],
options.get('build_options', ()))
else:
raise Exception('Invalid options %r for %r' % (options, name))
if opts.upload:
push_wheelhouse()
else:
copy_wheel_to_local_cache()
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))