blob: 71d6c5eb68d52c153c724f143f4ed7098f5c74c2 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Routines to list, select, and load masters and builders in master.cfg.
These routines help to load up master.cfgs in all directories, then locate
masters and builders among those loaded. This is intended to simplify master
selection and processing in frontend and build analysis tools, especially the
When run standalone, the script acts as example usage which lists masters
and builders of a selected master.
# pylint: disable=C0323
import contextlib
import os
import optparse
import sys
import traceback
BASE_DIR = os.path.abspath(os.path.join(
os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
# Install the common Infra environment (main only).
if __name__ == '__main__':
sys.path.insert(0, os.path.join(BASE_DIR, 'scripts'))
print sys.path[0]
import common.env
from common import chromium_utils
def TemporaryMasterPasswords():
all_paths = [os.path.join(BASE_DIR, 'site_config', '.bot_password')]
all_paths.extend(os.path.join(path, '.apply_issue_password')
for path in chromium_utils.ListMasters())
created_paths = []
for path in all_paths:
if not os.path.exists(path):
with open(path, 'w') as f:
f.write('reindeer flotilla\n')
except OSError:
for path in created_paths:
except OSError:
print 'WARNING: Could not remove %s!' % path
def ExecuteConfig(canonical_config):
"""Execute a master.cfg file and return its dictionary.
WARNING: executing a master.cfg loads modules into the python process.
Attempting to load another master.cfg with similar module names will
cause subtle (and not-so-subtle) errors. It is recommended to only call
this once per process.
localDict = {'basedir': os.path.dirname(canonical_config),
'__file__': canonical_config}
f = open(canonical_config, 'r')
mycwd = os.getcwd()
beforepath = list(sys.path) # make a 'backup' of it
exec f in localDict
return localDict
sys.path = beforepath
def LoadConfig(basedir, config_file='master.cfg', suppress=False):
"""Load and execute a master.cfg file from a directory.
This is a nicer wrapper around ExecuteConfig which will trap IO or execution
errors and provide an informative message if one occurs.
WARNING: executing a master.cfg loads modules into the python process.
Attempting to load another master.cfg with similar module names will
cause subtle (and not-so-subtle) errors. It is recommended to only call
this once per process.
canonical_basedir = os.path.abspath(os.path.expanduser(basedir))
canonical_config = os.path.join(canonical_basedir, config_file)
with TemporaryMasterPasswords():
localdict = ExecuteConfig(canonical_config)
except IOError as err:
errno, strerror = err
filename = err.filename
print >>sys.stderr, 'error %d executing %s: %s: %s' % (errno,
canonical_config, strerror, filename)
print >>sys.stderr, traceback.format_exc()
return None
except Exception:
if not suppress:
print >>sys.stderr, ('error while parsing %s: ' % canonical_config)
print >>sys.stderr, traceback.format_exc()
return None
return localdict
def PrettyPrintInternal(items, columns, title, notfound, spacing=4):
"""Display column-based information from an array of hashes."""
if not items:
print notfound
itemdata = {}
for col in columns:
itemdata[col] = [s[col] if col in s else 'n/a' for s in items]
lengths = {}
for col in columns:
datalen = max([len(x) for x in itemdata[col]])
lengths[col] = max(len(col), datalen)
maxwidth = sum([lengths[col] for col in columns]) + (
spacing * (len(columns) - 1))
spac = ' ' * spacing
print title
print spac.join([col.rjust(lengths[col]) for col in columns])
print '-' * maxwidth
for i in range(len(items)):
print spac.join([itemdata[col][i].rjust(lengths[col]) for col in columns])
def PrettyPrintBuilders(builders, master):
"""Pretty-print a list of builders from a master."""
columns = ['name', 'slavename', 'category']
title = 'outputting builders for: %s' % master
notfound = 'no builders found.'
builders = Denormalize(builders, 'slavenames', 'slavename', columns)
PrettyPrintInternal(builders, columns, title, notfound)
def PrettyPrintMasters(masterpairs):
masters = []
for mastername, path in masterpairs:
abspath = os.path.abspath(path)
relpath = os.path.relpath(path)
shortpath = abspath if len(abspath) < len(relpath) else relpath
master = {}
master['mastername'] = mastername
master['path'] = shortpath
columns = ['mastername', 'path']
title = 'listing available masters:'
notfound = 'no masters found.'
PrettyPrintInternal(masters, columns, title, notfound)
def Denormalize(items, over, newcol, wanted):
"""Splits a one-to-many hash into many one-to-ones.
PrettyPrintInternal needs a list of many builders with one slave, this will
properly format the data as such.
items: a list of dictionaries to be denormalized
over: the column (key) over which to separate items
newcol: the new name of 'over' in the new item
wanted: the desired keys in the new item
Example: take some diners with different meals:
[{'name': 'diner1', 'toasts': ['rye', 'wheat'], eggs:['scrambled']},
{'name': 'diner2', 'toasts': ['rye', 'white'], eggs:['fried']}]
Let's say you only cared about your diner/toast options. If you denormalized
with over=toasts, newcol=toast, wanted=['name', toast'], you'd get:
[{'name': 'diner1', 'toast': 'rye'},
{'name': 'diner1', 'toast': 'wheat'},
{'name': 'diner2', 'toast': 'rye'},
{'name': 'diner2', 'toast': 'white'}]
def arrayify(possible_array):
"""Convert 'string' into ['string']. Leave actual arrays alone."""
if isinstance(possible_array, basestring):
return [possible_array]
return possible_array
wanted_cols = set(wanted)
result = []
for row in items:
for element in arrayify(row[over]):
newitem = {}
# Only bring over the requested columns, instead of all.
for col in wanted_cols:
if col in row:
newitem[col] = row[col]
newitem[newcol] = element
return result
def OnlyGetOne(seq, key, source):
"""Confirm a sequence only contains one unique value and return it.
This is used when searching for a specific builder. If a match turns up
multiple results that all share the same builder, then select that builder.
def uniquify(seq):
return list(frozenset(seq))
res = uniquify([s[key] for s in seq])
if len(res) > 1:
print >>sys.stderr, 'error: %s too many %ss:' % (source, key)
for r in res:
print ' ', r
return None
elif not res:
print 'error: %s zero %ss' % (source, key)
return None
return res[0]
def GetMasters(include_public=True, include_internal=True):
"""Return a pair of (mastername, path) for all masters found."""
# note: ListMasters uses master.cfg hardcoded as part of its search path
def parse_master_name(masterpath):
"""Returns a mastername from a pathname to a master."""
_, tail = os.path.split(masterpath)
sep = '.'
hdr = 'master'
chunks = tail.split(sep)
if not chunks or chunks[0] != hdr or len(chunks) < 2:
raise ValueError('unable to parse mastername from path! (%s)' % tail)
return sep.join(chunks[1:])
return [(parse_master_name(m), m) for m in
def ChooseMaster(searchname):
"""Given a string, find all masters and pick the master that matches."""
masters = GetMasters()
masternames = []
master_lookup = {}
for mn, path in masters:
master = {}
master['mastername'] = mn
master_lookup[mn] = path
candidates = [mn for mn in masternames if mn['mastername'] == searchname]
errstring = 'string \'%s\' matches' % searchname
master = OnlyGetOne(candidates, 'mastername', errstring)
if not master:
return None
return master_lookup[master]
def SearchBuilders(builders, spec):
"""Return a list of builders which match what is specified in 'spec'.
'spec' can be a hash with a key of either 'name', 'slavename', or 'either'.
This allows for flexibility in how a frontend gets information from the user.
if 'builder' in spec:
return [b for b in builders if b['name'] ==
elif 'hostname' in spec:
return [b for b in builders if b['slavename']
== spec['hostname']]
return [b for b in builders if (b['name'] ==
spec['either']) or (b['slavename'] == spec['either'])]
def GetBuilderName(builders, keyval):
"""Return unique builder name from a list of builders."""
errstring = 'string \'%s\' matches' % keyval
return OnlyGetOne(builders, 'name', errstring)
def ChooseBuilder(builders, spec):
"""Search through builders matching 'spec' and return it."""
denormedbuilders = Denormalize(builders, 'slavenames', 'slavename', ['name'])
candidates = SearchBuilders(denormedbuilders, spec)
buildername = GetBuilderName(candidates, spec.values()[0])
if not buildername:
return None
builder = [b for b in builders if b['name'] == buildername][0]
if 'hostname' in spec:
builder['slavename'] = spec['hostname']
elif 'either' in spec and spec['either'] in builder['slavenames']:
builder['slavename'] = spec['either']
# User selected builder instead of slavename, so just pick the first
# slave the builder has.
builder['slavename'] = builder['slavenames'][0]
return builder
def main():
prog_desc = 'List all masters or builders within a master.'
usage = '%prog [master] [builder or slave]'
parser = optparse.OptionParser(usage=(usage + '\n\n' + prog_desc))
(_, args) = parser.parse_args()
if len(args) > 2:
parser.error("Too many arguments specified!")
masterpairs = GetMasters()
if len(args) < 1:
return 0
master_path = ChooseMaster(args[0])
if not master_path:
return 2
config = LoadConfig(master_path)
if not config:
return 2
mastername = config['BuildmasterConfig']['properties']['mastername']
builders = config['BuildmasterConfig']['builders']
if len(args) < 2:
PrettyPrintBuilders(builders, mastername)
return 0
my_builder = ChooseBuilder(builders, {'either': args[1]})
if not my_builder:
return 2
print "Matched %s/%s." % (mastername, my_builder['name'])
return 0
if __name__ == '__main__':