| #!/usr/bin/env python |
| # Copyright 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Starts all masters and verify they can server /json/project fine. |
| """ |
| |
| import collections |
| import contextlib |
| import glob |
| import logging |
| import optparse |
| import os |
| import subprocess |
| import sys |
| import tempfile |
| import threading |
| import time |
| |
| BUILD_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| MAX_CONCURRENT_THREADS = 8 |
| sys.path.insert(0, os.path.join(BUILD_DIR, 'scripts')) |
| import common.env |
| common.env.Install() |
| |
| import masters_util |
| from common import chromium_utils |
| from common import master_cfg_utils |
| |
| |
| def do_master_imports(): |
| # Import scripts/slave/bootstrap.py to get access to the ImportMasterConfigs |
| # function that will pull in every site_config for us. The master config |
| # classes are saved as attributes of config_bootstrap.Master. The import |
| # takes care of knowing which set of site_configs to use. |
| import slave.bootstrap |
| slave.bootstrap.ImportMasterConfigs() |
| return getattr(sys.modules['config_bootstrap'], 'Master') |
| |
| |
| @contextlib.contextmanager |
| def BackupPaths(base_path, path_globs): |
| tmpdir = tempfile.mkdtemp(prefix='.tmpMastersTest', dir=base_path) |
| paths_to_restore = [] |
| try: |
| for path_glob in path_globs: |
| for path in glob.glob(os.path.join(base_path, path_glob)): |
| bkup_path = os.path.join(tmpdir, os.path.relpath(path, base_path)) |
| os.rename(path, bkup_path) |
| paths_to_restore.append((path, bkup_path)) |
| yield |
| finally: |
| for path, bkup_path in paths_to_restore: |
| if subprocess.call(['rm', '-rf', path]) != 0: |
| print >> sys.stderr, 'ERROR: failed to remove tmp %s' % path |
| continue |
| if subprocess.call(['mv', bkup_path, path]) != 0: |
| print >> sys.stderr, 'ERROR: mv %s %s' % (bkup_path, path) |
| os.rmdir(tmpdir) |
| |
| def test_master(master, path, name, ports): |
| if not masters_util.stop_master(master, path): |
| return False |
| logging.info('%s Starting', master) |
| start = time.time() |
| with BackupPaths(path, ['twistd.log', 'twistd.log.?', 'git_poller_*.git', |
| 'state.sqlite']): |
| try: |
| if not masters_util.start_master(master, path, dry_run=True): |
| return False |
| res = masters_util.wait_for_start(master, name, path, ports) |
| if not res: |
| logging.info('%s Success in %1.1fs', master, (time.time() - start)) |
| return res |
| finally: |
| masters_util.stop_master(master, path, force=True) |
| |
| |
| class MasterTestThread(threading.Thread): |
| # Class static. Only access this from the main thread. |
| port_lock_map = collections.defaultdict(threading.Lock) |
| |
| def __init__(self, master, master_class, master_path): |
| super(MasterTestThread, self).__init__() |
| self.master = master |
| self.master_path = master_path |
| self.name = master_class.project_name |
| all_ports = [ |
| master_class.master_port, master_class.master_port_alt, |
| master_class.slave_port, getattr(master_class, 'try_job_port', 0)] |
| # Sort port locks numerically to prevent deadlocks. |
| self.port_locks = [self.port_lock_map[p] for p in sorted(all_ports) if p] |
| |
| # We pass both the read/write and read-only ports, even though querying |
| # either one alone would be sufficient sign of success. |
| self.ports = [p for p in all_ports[:2] if p] |
| self.result = None |
| |
| def run(self): |
| with contextlib.nested(*self.port_locks): |
| self.result = test_master( |
| self.master, self.master_path, self.name, self.ports) |
| |
| |
| def join_threads(started_threads, failed): |
| success = 0 |
| for cur_thread in started_threads: |
| cur_thread.join(30) |
| if cur_thread.result: |
| print '\n=== Error running %s === ' % cur_thread.master |
| print cur_thread.result |
| failed.add(cur_thread.master) |
| else: |
| success += 1 |
| return success |
| |
| |
| def real_main(all_expected): |
| start = time.time() |
| master_classes = do_master_imports() |
| all_masters = {} |
| for base in all_expected: |
| base_dir = os.path.join(base, 'masters') |
| all_masters[base] = sorted(p for p in |
| os.listdir(base_dir) if |
| os.path.exists(os.path.join(base_dir, p, 'master.cfg'))) |
| failed = set() |
| skipped = 0 |
| success = 0 |
| |
| # First make sure no master is started. Otherwise it could interfere with |
| # conflicting port binding. |
| if not masters_util.check_for_no_masters(): |
| return 1 |
| for base, masters in all_masters.iteritems(): |
| for master in masters: |
| pid_path = os.path.join(base, 'masters', master, 'twistd.pid') |
| if os.path.isfile(pid_path): |
| pid_value = int(open(pid_path).read().strip()) |
| if masters_util.pid_exists(pid_value): |
| print >> sys.stderr, ('%s is still running as pid %d.' % |
| (master, pid_value)) |
| print >> sys.stderr, 'Please stop it before running the test.' |
| return 1 |
| |
| |
| with master_cfg_utils.TemporaryMasterPasswords(): |
| for base, masters in all_masters.iteritems(): |
| started_threads = [] |
| for master in masters[:]: |
| if not master in all_expected[base]: |
| continue |
| masters.remove(master) |
| classname = all_expected[base].pop(master) |
| if not classname: |
| skipped += 1 |
| continue |
| cur_thread = MasterTestThread( |
| master=master, |
| master_class=getattr(master_classes, classname), |
| master_path=os.path.join(base, 'masters', master)) |
| cur_thread.start() |
| started_threads.append(cur_thread) |
| # Avoid having too many concurrent threads; join if we have reached the |
| # limit. |
| if len(started_threads) == MAX_CONCURRENT_THREADS: |
| success += join_threads(started_threads, failed) |
| started_threads = [] |
| # Join to the remaining started threads. |
| if len(started_threads): |
| success += join_threads(started_threads, failed) |
| |
| if failed: |
| print >> sys.stderr, ( |
| '%d masters failed:\n%s' % (len(failed), '\n'.join(sorted(failed)))) |
| remaining_masters = [] |
| for masters in all_masters.itervalues(): |
| remaining_masters.extend(masters) |
| if any(remaining_masters): |
| print >> sys.stderr, ( |
| '%d masters were not expected:\n%s' % |
| (len(remaining_masters), '\n'.join(sorted(remaining_masters)))) |
| outstanding_expected = [] |
| for expected in all_expected.itervalues(): |
| outstanding_expected.extend(expected) |
| if outstanding_expected: |
| print >> sys.stderr, ( |
| '%d masters were expected but not found:\n%s' % |
| (len(outstanding_expected), '\n'.join(sorted(outstanding_expected)))) |
| print >> sys.stderr, ( |
| '%s masters succeeded, %d failed, %d skipped in %1.1fs.' % ( |
| success, len(failed), skipped, time.time() - start)) |
| return int(bool(remaining_masters or outstanding_expected or failed)) |
| |
| |
| def main(argv): |
| parser = optparse.OptionParser() |
| parser.add_option('-v', '--verbose', action='count', default=0) |
| options, args = parser.parse_args(argv[1:]) |
| if args: |
| parser.error('Unknown args: %s' % args) |
| levels = (logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG) |
| logging.basicConfig(level=levels[min(options.verbose, len(levels)-1)]) |
| |
| # Remove site_config's we don't add ourselves. Can cause issues when running |
| # this test under a buildbot-spawned process. |
| sys.path = [x for x in sys.path if not x.endswith('site_config')] |
| base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| build_internal = os.path.join(os.path.dirname(base_dir), 'build_internal') |
| sys.path.extend(os.path.normpath(os.path.join(base_dir, d)) for d in ( |
| 'site_config', |
| os.path.join(build_internal, 'site_config'), |
| )) |
| public_masters = { |
| 'master.chromium': 'Chromium', |
| 'master.tryserver.chromium.mac': 'TryServerChromiumMac', |
| } |
| all_masters = {base_dir: public_masters} |
| if os.path.exists(build_internal): |
| internal_test_data = chromium_utils.ParsePythonCfg( |
| os.path.join(build_internal, 'tests', 'internal_masters_cfg.py'), |
| fail_hard=True) |
| all_masters[build_internal] = internal_test_data['masters_test'] |
| return real_main(all_masters) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv)) |