blob: 937ab83b11cbb00b34d765d0002e8af53d92c49c [file] [log] [blame]
# -*- python -*-
# ex: set syntax=python:
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# READ THIS:
# See http://dev.chromium.org/developers/testing/chromium-build-infrastructure
import os
import socket
from buildbot.scheduler import Triggerable
# These modules come from scripts, which must be in the PYTHONPATH.
from master import master_utils
from master import slaves_list
from master.builders_pools import BuildersPools
from master.factory import annotator_factory
from master.factory import remote_run_factory
from master.try_job_http import TryJobHTTP
import config
import master_site_config
ActiveMaster = master_site_config.TryServerChromiumLinux
MAIL_NOTIFIER = ActiveMaster.is_production_host
m_annotator = annotator_factory.AnnotatorFactory(ActiveMaster)
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
config.DatabaseSetup(c)
####### CHANGESOURCES
c['change_source'] = []
# Avoid merging requests.
c['mergeRequests'] = False
####### BUILDERS
# Try queues.
pools = BuildersPools('chrome')
chromium_builders = []
chromeos_boards = ('x86-generic', 'amd64-generic', 'daisy')
def base_factory(recipe, factory_properties=None, **kwargs):
fp = factory_properties or {}
fp.update({
'$recipe_engine': {'mode_flags': {'use_subprocess42': True}}
})
# Disable "max time without output" timeout if total build timeout
# is in effect.
if 'max_time' in kwargs and 'timeout' not in kwargs:
kwargs['timeout'] = None
return m_annotator.BaseFactory(recipe, factory_properties=fp, **kwargs)
def m_remote_run(recipe, **kwargs):
# Disable "max time without output" timeout if total build timeout
# is in effect.
if 'max_time' in kwargs and 'timeout' not in kwargs:
kwargs['timeout'] = None
return remote_run_factory.RemoteRunFactory(
active_master=ActiveMaster,
repository='https://chromium.googlesource.com/chromium/tools/build.git',
recipe=recipe,
factory_properties={'path_config': 'kitchen'},
**kwargs)
def m_remote_run_chromium_src(recipe, **kwargs):
# Disable "max time without output" timeout if total build timeout
# is in effect.
if 'max_time' in kwargs and 'timeout' not in kwargs:
kwargs['timeout'] = None
return remote_run_factory.RemoteRunFactory(
active_master=ActiveMaster,
repository='https://chromium.googlesource.com/chromium/src.git',
recipe=recipe,
factory_properties={'path_config': 'kitchen'},
use_gitiles=True,
**kwargs)
for targ in ('_rel', '_dbg'):
chromium_builders.extend([{
'name': 'linux_chromium_compile%s_ng' % targ,
'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600),
'slavebuilddir': 'remote_run'
}, {
'name': 'linux_chromium%s_ng' % targ,
'factory': m_remote_run_chromium_src(
'chromium_trybot', timeout=3600, max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
}, {
'name': 'linux_chromium_chromeos%s_ng' % targ,
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
}, {
'name': 'linux_chromium_chromeos_compile%s_ng' % targ,
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
}, {
'name': 'linux_chromium_gn_chromeos%s' % targ,
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
}])
chromium_builders.append({
'name': 'linux_chromium_chromeos_asan_rel_ng',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
})
chromium_builders.append({
'name': 'linux_chromium_chromeos_ozone_rel_ng',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
})
for board in chromeos_boards:
chromium_builders.append({
'name': 'chromeos_%s_chromium_compile_only_ng' % (board,),
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run',
})
chromium_builders.append({
'name': 'linux_chromium_msan_rel_ng',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
})
chromium_builders.append({
'name': 'linux_chromium_chromeos_msan_rel_ng',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
})
chromium_builders.append({
'name': 'linux_chromium_tsan_rel_ng',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
})
chromium_builders.append({
'name': 'linux_chromium_cfi_rel_ng',
'factory': m_remote_run_chromium_src('chromium_trybot', timeout=12000),
'slavebuilddir': 'remote_run',
})
chromium_builders.extend([{
'name': 'linux_arm',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
},
])
chromium_builders.append({
'name': 'linux_chromium_compile_dbg_32_ng',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
})
chromium_builders.append({
'name': 'linux_chromium_dbg_32_ng',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run'
})
chromium_builders.append({
'name': 'linux_chromium_archive_rel_ng',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
})
chromium_builders.append({
'name': 'linux_chromium_clobber_rel_ng',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run'
})
# This bot builds and upload GN binaries to cloud storage.
# We do not reuse the 'linux_chromium_gn' slavebuilddir because the upload
# bots need the linux sysroots and hence require src-internal.
chromium_builders.append({
'name': 'linux_chromium_gn_upload',
'factory': base_factory('chromium_gn_upload'),
'slavebuilddir': 'build_and_upload_gn',
})
chromium_builders.append({
'name': 'linux_upload_clang',
'factory': base_factory('chromium_upload_clang'),
'slavebuilddir': 'linux_upload_clang',
})
# Chromecast builders using chromium_trybot
chromium_builders.extend([{
'name': 'cast_shell_linux',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
},
])
chromium_builders.extend([{
'name': 'blimp_linux_dbg',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run',
},
])
b_linux_site_isolation = {
'name': 'linux_site_isolation',
'factory': m_remote_run_chromium_src('chromium_trybot'),
'slavebuilddir': 'remote_run',
}
b_linux_chromium_asan_rel_ng = {
'name': 'linux_chromium_asan_rel_ng',
'factory': m_remote_run_chromium_src(
'chromium_trybot', max_time=master_utils.CQ_MAX_TIME),
'slavebuilddir': 'remote_run',
}
# Optional GPU tryserver
chromium_builders.append({
'name': 'linux_optional_gpu_tests_rel',
'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600),
'slavebuilddir': 'remote_run'
})
# Optional browser-side navigation trybot
b_linux_chromium_browser_side_navigation_rel = {
'name': 'linux_chromium_browser_side_navigation_rel',
'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600),
'slavebuilddir': 'remote_run'
}
# NaCl SDK try bots
b_linux_nacl_sdk = {
'name': 'linux_nacl_sdk',
'factory': base_factory('nacl/sdk'),
}
# NaCl SDK build-only try bots
b_linux_nacl_sdk_build = {
'name': 'linux_nacl_sdk_build',
'factory': base_factory('nacl/sdk_build'),
}
# Codesearch builders
b_codesearch_linux_builder = {
'name': 'Chromium Linux Codesearch Builder',
'factory': base_factory('chromium_codesearch'),
}
b_codesearch_chromeos_builder = {
'name': 'ChromiumOS Codesearch Builder',
'factory': base_factory('chromium_codesearch'),
}
# Optional Closure Compilation builder
chromium_builders.append({
'name': 'closure_compilation',
'factory': base_factory('closure_compilation'),
# Share build directory with compatible existing builders to save space.
'slavebuilddir': 'linux',
})
# Builder specifically to run layout tests with a particular flag (see crbug.com/622865).
chromium_builders.append({
'name': 'linux_layout_tests_slimming_paint_v2',
'factory': m_remote_run_chromium_src('chromium_trybot'),
})
# Presubmit bots
def presubmit(name, slavebuilddir=None, extra_props=None):
properties = {'repo_name': name}
properties.update(extra_props or {})
return {
'name': '%s_presubmit' % name,
'slavebuilddir': (slavebuilddir or ('presubmit_%s' % name)),
'factory': base_factory(
'run_presubmit', properties,
max_time=master_utils.CQ_MAX_TIME),
}
b_chromium_presubmit = presubmit('chromium', slavebuilddir='linux',
extra_props={'runhooks': True})
# These variable builders are used by Findit to identify culprit commits for
# compile or test failures on the main waterfall. They run on optional slaves
# and NOT CQ slaves.
variable_builders = [
{
'name': 'linux_chromium_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux',
},
{
'name': 'linux_chromium_variable_32',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_32',
},
{
'name': 'linux_chromium_variable_chrome',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chrome',
},
{
'name': 'linux_chromium_variable_clobber',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_clobber',
},
{
'name': 'linux_chromium_variable_32_chrome',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chrome_32',
},
{
'name': 'linux_chromium_variable_32_clobber',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_clobber_32',
},
{
'name': 'linux_chromium_asan_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_asan',
},
{
'name': 'linux_chromium_chromeos_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chromeos',
},
{
'name': 'linux_chromium_chromeos_variable_chrome',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chromeos_chrome',
},
{
'name': 'linux_chromium_chromeos_asan_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chromeos_asan',
},
{
'name': 'linux_chromium_gn_chromeos_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chromium_gn_chromeos',
},
{
'name': 'linux_chromium_blimp_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_blimp',
},
{
'name': 'linux_chromium_cast_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_cast',
},
{
'name': 'linux_chromium_webkit_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_layout',
},
{
'name': 'linux_chromium_webkit_asan_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_layout_asan',
},
{
'name': 'linux_chromium_webkit_leak_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_layout_leak',
},
{
'name': 'linux_chromium_webkit_msan_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_layout_msan',
},
{
'name': 'linux_chromiumos_full_variable',
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'linux_chromeos',
},
]
for board in chromeos_boards:
variable_builders.append({
'name': 'chromeos_%s_variable' % (board,),
'factory': m_remote_run('findit/chromium/compile'),
'slavebuilddir': 'chromeos_%s' % (board,),
})
c['builders'] = [
b_chromium_presubmit,
b_codesearch_chromeos_builder,
b_codesearch_linux_builder,
b_linux_chromium_asan_rel_ng,
b_linux_chromium_browser_side_navigation_rel,
b_linux_nacl_sdk,
b_linux_nacl_sdk_build,
b_linux_site_isolation,
] + chromium_builders + variable_builders
# Slaves are loaded from slaves.cfg.
slaves = slaves_list.SlavesList('slaves.cfg', 'TryServerChromiumLinux')
for builder in c['builders']:
# Associate the slaves to the builders. The configuration is in slaves.cfg.
builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
# Sets nextSlave function for slave affinity.
builder['nextSlave'] = master_utils.PreferredBuilderNextSlaveFunc()
# Don't enable auto_reboot for people testing locally.
builder.setdefault('auto_reboot', ActiveMaster.is_production_host)
####### BUILDSLAVES
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(
c['builders'],
config.Master.GetBotPassword(),
preferred_builder_dict=slaves.GetPreferredBuildersDict())
# Make sure everything works together.
master_utils.VerifySetup(c, slaves)
####### SCHEDULERS
# Configure the Schedulers;
# Main Tryscheduler for the try queue. groups is defined in the loop above.
c['schedulers'] = []
last_good_urls = {'chrome': ActiveMaster.last_good_url,
'blink': ActiveMaster.last_good_blink_url}
code_review_sites = {'chrome': ActiveMaster.code_review_site,
'blink': ActiveMaster.code_review_site}
c['schedulers'].append(TryJobHTTP(
name='try_job_http',
port=ActiveMaster.try_job_port,
last_good_urls=last_good_urls,
code_review_sites=code_review_sites,
pools=pools))
####### STATUS TARGETS
# Buildbot master url:
# Must come before AutoSetupMaster().
if ActiveMaster.is_production_host:
c['buildbotURL'] = ActiveMaster.buildbot_url
else:
c['buildbotURL'] = 'http://%s:%d/' % (
socket.getfqdn(), ActiveMaster.master_port)
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(c, ActiveMaster)
if MAIL_NOTIFIER:
# Add a dumb MailNotifier first so it will be used for BuildSlave with
# notify_on_missing set when they go missing.
from buildbot.status import mail
c['status'].append(mail.MailNotifier(
fromaddr=ActiveMaster.from_address,
builders=[],
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Try job result emails.
from master.try_mail_notifier import TryMailNotifier
c['status'].append(TryMailNotifier(
fromaddr=ActiveMaster.from_address,
reply_to=ActiveMaster.reply_to,
subject="try %(result)s for %(reason)s @ r%(revision)s",
mode='all',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# The followings are what is kept on disk.
# Keep last try jobs, the default is too low. Must keep at least a few days
# worth of try jobs. 3000 is not even a full day but the server is full. Keep
# more build objects than log since they are much smaller.
c['buildHorizon'] = 6000
c['logHorizon'] = 3000
# Must be at least 2x the number of slaves.
c['eventHorizon'] = 200
# Adjust the buildCaches to be 3x the number of slaves per builder.
c['autoBuildCacheRatio'] = 3
# Hack buildbot so the Stop build button doesn't work anymore. Otherwise it's
# just a pain, user misuse this button all the time.
def hack_stop(function):
def hook(*args, **kwargs):
result = function(*args, **kwargs)
result = result.replace('<input type="submit" value="Stop',
'<input type="button" onclick="alert(\''
'For more information, visit '
'http://dev.chromium.org/developers/try-server-usage'
'\');"'
' value="Stop')
return result
return hook
# Only do the hack_stop if we are the production master. This allows
# us to keep the STOP button live for local/test instances.
#if ActiveMaster.is_production_host:
# from buildbot.status.web.builder import StatusResourceBuilder
# StatusResourceBuilder.build_line = hack_stop(StatusResourceBuilder.build_line)
# from buildbot.status.web import base
# base.make_stop_form = hack_stop(base.make_stop_form)
####### PROJECT IDENTITY
# The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.
c['projectURL'] = 'http://dev.chromium.org/developers/testing/try-server-usage'
# vi: set ts=4 sts=2 sw=2 et: