masters/master.client.dart.packages/master.cfg - chromium/tools/build - Git at Google

 # -*- python -*-
 # ex: set syntax=python:

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.


 from buildbot.scheduler import Dependent
 from buildbot.scheduler import Scheduler
 from buildbot.scheduler import Periodic
 from buildbot.schedulers import triggerable
 from buildbot.schedulers.filter import ChangeFilter

 from common import chromium_utils
 from master import build_utils
 from master import master_utils
 from master import slaves_list
 from master import status_logger
 from master.factory import gclient_factory, annotator_factory
 from master.factory.dart import dart_factory
 from master.factory.dart import pub_poller
 from master.factory.dart.dart_factory import linux_env, windows_env
 from master.factory.dart.dart_factory import linux_clang_env
 from master.factory.dart.channels import CHANNELS
 from twisted.python import log

 from packages import PACKAGES, GITHUB_TESTING_PACKAGES, PUBLISHED_PACKAGE_NAMES

 import config
 import master_site_config

 ActiveMaster = master_site_config.DartPackages
 utils = dart_factory.DartUtils(ActiveMaster)

 MASTER_HOST = ActiveMaster.master_host
 WEB_STATUS = True
 MAIL_NOTIFIER = ActiveMaster.is_production_host
 GOOD_REVISIONS = False

 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}

 config.DatabaseSetup(c)

 # 'slavePortnum' defines the TCP port to listen on. This must match the value
 # configured into the buildslaves (with their --master option)
 c['slavePortnum'] = ActiveMaster.slave_port

 slaves = slaves_list.SlavesList('slaves.cfg', 'DartPackages')

 variants = [ ]

 # The default set of systems that we create for all packages
 systems = ['windows', 'linux', 'mac']

 def AddVariant(package, system, branch=None):
   repo = utils.get_github_gclient_repo(package.github_project,
                                        package.github_repo,
                                        branch)
   variants.append({
       'name': package.builderName(system, branch),
       'category': package.builderCategory(),
       'platform': 'packages',
       'os': system,
       'deps': [('dart/third_party/pkg/%s' % package.name, repo),
                ('dart/third_party/package-bots',
                utils.get_github_repo('dart-lang', 'package-bots'))],
   })

 for package in GITHUB_TESTING_PACKAGES:
   for system in systems:
     AddVariant(package, system)
     for branch in package.extra_branches:
       AddVariant(package, system, branch)

 ####### Factory setup

 utils.setup_factories(variants)

 ####### Schedulers
 c['schedulers'] = []

 builder_names = utils.get_builder_names(variants)

 def AddChangeFilter(package, branch=None):
   # We always trigger on the trigger bot package - used when updating the sdk
   projects = ['package-bots', 'pub_packages']
   projects.extend(package.dependencies);
   project_name = package.github_repo
   if branch:
     project_name = '%s-%s' % (package.github_repo, branch)
   projects.append(project_name)
   change_filter = ChangeFilter(project=projects)

   builders = [package.builderName(system, branch) for system in systems]
   for builder in builders:
     assert builder in builder_names
   c['schedulers'].append(Scheduler(
       name='%s-%s' % (package.name, branch or 'master'),
       change_filter=change_filter,
       treeStableTimer=0,
       builderNames=builders
   ))

 for package in GITHUB_TESTING_PACKAGES:
   AddChangeFilter(package)
   for branch in package.extra_branches:
     AddChangeFilter(package, branch)

 ####### Builders

 c['builders'] = utils.get_builders_from_variants(variants, slaves, [])

 ####### CHANGESOURCES

 # General trigger used to trigger all bots when we have a new sdk.
 c['change_source'] = [utils.get_github_poller('dart-lang', 'package-bots')]

 poller = pub_poller.PubPoller(PUBLISHED_PACKAGE_NAMES, project='pub_packages')
 c['change_source'].append(poller)

 interval = 300
 for package in PACKAGES:
   interval += 7;
   if interval > 400: interval -= 100
   if package.isGithubPackage() and package.run_tests:
     c['change_source'].append(
         utils.get_github_poller(package.github_project, package.github_repo,
                                 interval=interval))
     for branch in package.extra_branches:
       c['change_source'].append(
         utils.get_github_poller(package.github_project, package.github_repo,
                                 branch, interval=interval))

 ####### BUILDSLAVES

 c['slaves'] = utils.get_slaves(c['builders'])

 # Make sure everything works together.
 master_utils.VerifySetup(c, slaves)

 # Prioritize the builders depending on channel.
 c['prioritizeBuilders'] = utils.prioritize_builders

 ####### STATUS TARGETS

 # 'status' is a list of Status Targets. The results of each build will be
 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 # including web pages, email senders, and IRC bots.

 c['status'] = [status_logger.StatusEventLogger()]

 if WEB_STATUS:
   for status in utils.get_web_statuses(order_console_by_time=True,
                                        extra_templates=['templates']):
     c['status'].append(status)

 if MAIL_NOTIFIER:
   # We have people that are interested in a specific subset of the builders
   # and want to be notified whenever they break.
   mail_notifiers = [
     {
       'extraRecipients': ['kustermann@google.com', 'sgjesse@google.com'],
       'builders': ['packages-linux-googleapis_auth', 'packages-linux-gcloud'],
       'subject' : 'BB | Package Waterfall > GCloud builder failed.',
     },
     {
       'extraRecipients': ['whesse@google.com'],
       'builders': None,
       'sendToInterestedUsers': True,
       'subject' : 'BB | Package Waterfall failed.',
     },
   ]

   for notifier in utils.get_mail_notifier_statuses(mail_notifiers):
     c['status'].append(notifier)

 # Keep last build logs, the default is too low.
 c['buildHorizon'] = 1000
 c['logHorizon'] = 500
 # Must be at least 2x the number of slaves.
 c['eventHorizon'] = 600

 # Template generation is using 40 entries, so give it a little extra.
 c['buildCacheSize'] = 42

 c['properties'] = {'mastername': master_utils.GetMastername()}

 ####### PROJECT IDENTITY

 # the 'projectName' string will be used to describe the project that this
 # buildbot is working on. For example, it is used as the title of the
 # waterfall HTML page. The 'projectURL' string will be used to provide a link
 # from buildbot HTML pages to your project's home page.

 c['projectName'] = ActiveMaster.project_name
 c['projectURL'] = config.Master.project_url

 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server (usually the html.Waterfall page) is visible. This
 # typically uses the port number set in the Waterfall 'status' entry, but
 # with an externally-visible host name which the buildbot cannot figure out
 # without some help.

 c['buildbotURL'] = ActiveMaster.buildbot_url
	# -- python --
	# ex: set syntax=python:

	# Copyright 2014 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.


	from buildbot.scheduler import Dependent
	from buildbot.scheduler import Scheduler
	from buildbot.scheduler import Periodic
	from buildbot.schedulers import triggerable
	from buildbot.schedulers.filter import ChangeFilter

	from common import chromium_utils
	from master import build_utils
	from master import master_utils
	from master import slaves_list
	from master import status_logger
	from master.factory import gclient_factory, annotator_factory
	from master.factory.dart import dart_factory
	from master.factory.dart import pub_poller
	from master.factory.dart.dart_factory import linux_env, windows_env
	from master.factory.dart.dart_factory import linux_clang_env
	from master.factory.dart.channels import CHANNELS
	from twisted.python import log

	from packages import PACKAGES, GITHUB_TESTING_PACKAGES, PUBLISHED_PACKAGE_NAMES

	import config
	import master_site_config

	ActiveMaster = master_site_config.DartPackages
	utils = dart_factory.DartUtils(ActiveMaster)

	MASTER_HOST = ActiveMaster.master_host
	WEB_STATUS = True
	MAIL_NOTIFIER = ActiveMaster.is_production_host
	GOOD_REVISIONS = False

	# This is the dictionary that the buildmaster pays attention to. We also use
	# a shorter alias to save typing.
	c = BuildmasterConfig = {}

	config.DatabaseSetup(c)

	# 'slavePortnum' defines the TCP port to listen on. This must match the value
	# configured into the buildslaves (with their --master option)
	c['slavePortnum'] = ActiveMaster.slave_port

	slaves = slaves_list.SlavesList('slaves.cfg', 'DartPackages')

	variants = [ ]

	# The default set of systems that we create for all packages
	systems = ['windows', 'linux', 'mac']

	def AddVariant(package, system, branch=None):
	repo = utils.get_github_gclient_repo(package.github_project,
	package.github_repo,
	branch)
	variants.append({
	'name': package.builderName(system, branch),
	'category': package.builderCategory(),
	'platform': 'packages',
	'os': system,
	'deps': [('dart/third_party/pkg/%s' % package.name, repo),
	('dart/third_party/package-bots',
	utils.get_github_repo('dart-lang', 'package-bots'))],
	})

	for package in GITHUB_TESTING_PACKAGES:
	for system in systems:
	AddVariant(package, system)
	for branch in package.extra_branches:
	AddVariant(package, system, branch)

	####### Factory setup

	utils.setup_factories(variants)

	####### Schedulers
	c['schedulers'] = []

	builder_names = utils.get_builder_names(variants)

	def AddChangeFilter(package, branch=None):
	# We always trigger on the trigger bot package - used when updating the sdk
	projects = ['package-bots', 'pub_packages']
	projects.extend(package.dependencies);
	project_name = package.github_repo
	if branch:
	project_name = '%s-%s' % (package.github_repo, branch)
	projects.append(project_name)
	change_filter = ChangeFilter(project=projects)

	builders = [package.builderName(system, branch) for system in systems]
	for builder in builders:
	assert builder in builder_names
	c['schedulers'].append(Scheduler(
	name='%s-%s' % (package.name, branch or 'master'),
	change_filter=change_filter,
	treeStableTimer=0,
	builderNames=builders
	))

	for package in GITHUB_TESTING_PACKAGES:
	AddChangeFilter(package)
	for branch in package.extra_branches:
	AddChangeFilter(package, branch)

	####### Builders

	c['builders'] = utils.get_builders_from_variants(variants, slaves, [])

	####### CHANGESOURCES

	# General trigger used to trigger all bots when we have a new sdk.
	c['change_source'] = [utils.get_github_poller('dart-lang', 'package-bots')]

	poller = pub_poller.PubPoller(PUBLISHED_PACKAGE_NAMES, project='pub_packages')
	c['change_source'].append(poller)

	interval = 300
	for package in PACKAGES:
	interval += 7;
	if interval > 400: interval -= 100
	if package.isGithubPackage() and package.run_tests:
	c['change_source'].append(
	utils.get_github_poller(package.github_project, package.github_repo,
	interval=interval))
	for branch in package.extra_branches:
	c['change_source'].append(
	utils.get_github_poller(package.github_project, package.github_repo,
	branch, interval=interval))

	####### BUILDSLAVES

	c['slaves'] = utils.get_slaves(c['builders'])

	# Make sure everything works together.
	master_utils.VerifySetup(c, slaves)

	# Prioritize the builders depending on channel.
	c['prioritizeBuilders'] = utils.prioritize_builders

	####### STATUS TARGETS

	# 'status' is a list of Status Targets. The results of each build will be
	# pushed to these targets. buildbot/status/*.py has a variety to choose from,
	# including web pages, email senders, and IRC bots.

	c['status'] = [status_logger.StatusEventLogger()]

	if WEB_STATUS:
	for status in utils.get_web_statuses(order_console_by_time=True,
	extra_templates=['templates']):
	c['status'].append(status)

	if MAIL_NOTIFIER:
	# We have people that are interested in a specific subset of the builders
	# and want to be notified whenever they break.
	mail_notifiers = [
	{
	'extraRecipients': ['kustermann@google.com', 'sgjesse@google.com'],
	'builders': ['packages-linux-googleapis_auth', 'packages-linux-gcloud'],
	'subject' : 'BB \| Package Waterfall > GCloud builder failed.',
	},
	{
	'extraRecipients': ['whesse@google.com'],
	'builders': None,
	'sendToInterestedUsers': True,
	'subject' : 'BB \| Package Waterfall failed.',
	},
	]

	for notifier in utils.get_mail_notifier_statuses(mail_notifiers):
	c['status'].append(notifier)

	# Keep last build logs, the default is too low.
	c['buildHorizon'] = 1000
	c['logHorizon'] = 500
	# Must be at least 2x the number of slaves.
	c['eventHorizon'] = 600

	# Template generation is using 40 entries, so give it a little extra.
	c['buildCacheSize'] = 42

	c['properties'] = {'mastername': master_utils.GetMastername()}

	####### PROJECT IDENTITY

	# the 'projectName' string will be used to describe the project that this
	# buildbot is working on. For example, it is used as the title of the
	# waterfall HTML page. The 'projectURL' string will be used to provide a link
	# from buildbot HTML pages to your project's home page.

	c['projectName'] = ActiveMaster.project_name
	c['projectURL'] = config.Master.project_url

	# the 'buildbotURL' string should point to the location where the buildbot's
	# internal web server (usually the html.Waterfall page) is visible. This
	# typically uses the port number set in the Waterfall 'status' entry, but
	# with an externally-visible host name which the buildbot cannot figure out
	# without some help.

	c['buildbotURL'] = ActiveMaster.buildbot_url