blob: 3d02d0882a28ea8897f8bacad6e68058f36c35d1 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Starts all masters and verify they can server /json/project fine.
"""
import collections
import contextlib
import glob
import logging
import optparse
import os
import subprocess
import sys
import tempfile
import threading
import time
BUILD_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MAX_CONCURRENT_THREADS = 8
sys.path.insert(0, os.path.join(BUILD_DIR, 'scripts'))
import common.env
common.env.Install()
import masters_util
from common import chromium_utils
from common import master_cfg_utils
def do_master_imports():
# Import scripts/slave/bootstrap.py to get access to the ImportMasterConfigs
# function that will pull in every site_config for us. The master config
# classes are saved as attributes of config_bootstrap.Master. The import
# takes care of knowing which set of site_configs to use.
import slave.bootstrap
slave.bootstrap.ImportMasterConfigs()
return getattr(sys.modules['config_bootstrap'], 'Master')
@contextlib.contextmanager
def BackupPaths(base_path, path_globs):
tmpdir = tempfile.mkdtemp(prefix='.tmpMastersTest', dir=base_path)
paths_to_restore = []
try:
for path_glob in path_globs:
for path in glob.glob(os.path.join(base_path, path_glob)):
bkup_path = os.path.join(tmpdir, os.path.relpath(path, base_path))
os.rename(path, bkup_path)
paths_to_restore.append((path, bkup_path))
yield
finally:
for path, bkup_path in paths_to_restore:
if subprocess.call(['rm', '-rf', path]) != 0:
print >> sys.stderr, 'ERROR: failed to remove tmp %s' % path
continue
if subprocess.call(['mv', bkup_path, path]) != 0:
print >> sys.stderr, 'ERROR: mv %s %s' % (bkup_path, path)
os.rmdir(tmpdir)
def test_master(master, path, name, ports):
if not masters_util.stop_master(master, path):
return False
logging.info('%s Starting', master)
start = time.time()
with BackupPaths(path, ['twistd.log', 'twistd.log.?', 'git_poller_*.git',
'state.sqlite']):
try:
if not masters_util.start_master(master, path, dry_run=True):
return False
res = masters_util.wait_for_start(master, name, path, ports)
if not res:
logging.info('%s Success in %1.1fs', master, (time.time() - start))
return res
finally:
masters_util.stop_master(master, path, force=True)
class MasterTestThread(threading.Thread):
# Class static. Only access this from the main thread.
port_lock_map = collections.defaultdict(threading.Lock)
def __init__(self, master, master_class, master_path):
super(MasterTestThread, self).__init__()
self.master = master
self.master_path = master_path
self.name = master_class.project_name
all_ports = [
master_class.master_port, master_class.master_port_alt,
master_class.slave_port, getattr(master_class, 'try_job_port', 0)]
# Sort port locks numerically to prevent deadlocks.
self.port_locks = [self.port_lock_map[p] for p in sorted(all_ports) if p]
# We pass both the read/write and read-only ports, even though querying
# either one alone would be sufficient sign of success.
self.ports = [p for p in all_ports[:2] if p]
self.result = None
def run(self):
with contextlib.nested(*self.port_locks):
self.result = test_master(
self.master, self.master_path, self.name, self.ports)
def join_threads(started_threads, failed):
success = 0
for cur_thread in started_threads:
cur_thread.join(30)
if cur_thread.result:
print '\n=== Error running %s === ' % cur_thread.master
print cur_thread.result
failed.add(cur_thread.master)
else:
success += 1
return success
def real_main(all_expected):
start = time.time()
master_classes = do_master_imports()
all_masters = {}
for base in all_expected:
base_dir = os.path.join(base, 'masters')
all_masters[base] = sorted(p for p in
os.listdir(base_dir) if
os.path.exists(os.path.join(base_dir, p, 'master.cfg')))
failed = set()
skipped = 0
success = 0
# First make sure no master is started. Otherwise it could interfere with
# conflicting port binding.
if not masters_util.check_for_no_masters():
return 1
for base, masters in all_masters.iteritems():
for master in masters:
pid_path = os.path.join(base, 'masters', master, 'twistd.pid')
if os.path.isfile(pid_path):
pid_value = int(open(pid_path).read().strip())
if masters_util.pid_exists(pid_value):
print >> sys.stderr, ('%s is still running as pid %d.' %
(master, pid_value))
print >> sys.stderr, 'Please stop it before running the test.'
return 1
with master_cfg_utils.TemporaryMasterPasswords():
for base, masters in all_masters.iteritems():
started_threads = []
for master in masters[:]:
if not master in all_expected[base]:
continue
masters.remove(master)
classname = all_expected[base].pop(master)
if not classname:
skipped += 1
continue
cur_thread = MasterTestThread(
master=master,
master_class=getattr(master_classes, classname),
master_path=os.path.join(base, 'masters', master))
cur_thread.start()
started_threads.append(cur_thread)
# Avoid having too many concurrent threads; join if we have reached the
# limit.
if len(started_threads) == MAX_CONCURRENT_THREADS:
success += join_threads(started_threads, failed)
started_threads = []
# Join to the remaining started threads.
if len(started_threads):
success += join_threads(started_threads, failed)
if failed:
print >> sys.stderr, (
'%d masters failed:\n%s' % (len(failed), '\n'.join(sorted(failed))))
remaining_masters = []
for masters in all_masters.itervalues():
remaining_masters.extend(masters)
if any(remaining_masters):
print >> sys.stderr, (
'%d masters were not expected:\n%s' %
(len(remaining_masters), '\n'.join(sorted(remaining_masters))))
outstanding_expected = []
for expected in all_expected.itervalues():
outstanding_expected.extend(expected)
if outstanding_expected:
print >> sys.stderr, (
'%d masters were expected but not found:\n%s' %
(len(outstanding_expected), '\n'.join(sorted(outstanding_expected))))
print >> sys.stderr, (
'%s masters succeeded, %d failed, %d skipped in %1.1fs.' % (
success, len(failed), skipped, time.time() - start))
return int(bool(remaining_masters or outstanding_expected or failed))
def main(argv):
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='count', default=0)
options, args = parser.parse_args(argv[1:])
if args:
parser.error('Unknown args: %s' % args)
levels = (logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG)
logging.basicConfig(level=levels[min(options.verbose, len(levels)-1)])
# Remove site_config's we don't add ourselves. Can cause issues when running
# this test under a buildbot-spawned process.
sys.path = [x for x in sys.path if not x.endswith('site_config')]
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
build_internal = os.path.join(os.path.dirname(base_dir), 'build_internal')
sys.path.extend(os.path.normpath(os.path.join(base_dir, d)) for d in (
'site_config',
os.path.join(build_internal, 'site_config'),
))
public_masters = {
'master.chromium': 'Chromium',
'master.chromium.clang': 'ChromiumClang',
'master.chromium.perf': 'ChromiumPerf',
'master.tryserver.chromium.mac': 'TryServerChromiumMac',
}
all_masters = {base_dir: public_masters}
if os.path.exists(build_internal):
internal_test_data = chromium_utils.ParsePythonCfg(
os.path.join(build_internal, 'tests', 'internal_masters_cfg.py'),
fail_hard=True)
all_masters[build_internal] = internal_test_data['masters_test']
return real_main(all_masters)
if __name__ == '__main__':
sys.exit(main(sys.argv))