blob: 65aa5efdace494ba99480a851a6457b11bdfd2ad [file] [log] [blame]
# vim: ft=python:
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from buildbot.changes.filter import ChangeFilter
from buildbot.process.properties import WithProperties
from buildbot.scheduler import Dependent
from buildbot.scheduler import Nightly
from buildbot.scheduler import Periodic
from buildbot.scheduler import Scheduler
from buildbot.scheduler import Triggerable
from buildbot.status.mail import MailNotifier
from common import chromium_utils
# These modules come from scripts/master, which must be in the PYTHONPATH.
from master import build_utils
from master import chromium_notifier
from master import failures_notifier
from master import master_config
from master import master_utils
from master import slaves_list
from master import gitiles_poller
from master.factory import annotator_factory
from master.factory import remote_run_factory
# These modules come from scripts/common, which must be in the PYTHONPATH.
import config
import master_site_config
ActiveMaster = master_site_config.ChromiumFYI
STATUS_PUSH = False
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
c['status'] = []
# Force merging of requests.
c['mergeRequests'] = lambda *args, **kwargs: True
config.DatabaseSetup(c)
####### CHANGESOURCES
# Polls config.Master.trunk_url for changes
chromium_repo_url = config.Master.git_server_url + '/chromium/src'
trunk_poller = gitiles_poller.GitilesPoller(chromium_repo_url,
project='chromium')
c['change_source'] = [trunk_poller]
# Number of machines to shard ash browser_tests over.
ASH_SHARDS = 2
####### SCHEDULERS
## configure the Schedulers
# Main scheduler for all changes in trunk.
s_chromium = Scheduler(
name='chromium',
treeStableTimer=60,
change_filter=ChangeFilter(project='chromium', branch='master'),
builderNames=['Chromium Windows Analyze',
'Win7 Builder Goma Canary',
'Win7 Builder (dbg) Goma Canary',
'Win 10 Fast Ring',
'Afl Upload Linux ASan',
'Libfuzzer Upload Mac ASan',
'Linux remote_run Builder',
'ios-device-goma-canary-clobber',
'ios11-beta-simulator',
'Mac deterministic',
'Windows deterministic',
'Windows Clang deterministic',
'Mac deterministic (dbg)',
'Out of Process Profiling Linux',
'Out of Process Profiling Mac',
'Out of Process Profiling Windows',
'Linux Clang Analyzer',
'linux-annotator-rel',
])
# Triggerable scheduler for Android Builder
s_chromium_android_dbg = Triggerable(
name='android_dbg',
builderNames=['Android ChromeDriver Tests (dbg)',
'Android Remoting Tests',
'Android VR Tests',])
s_android_find_annotated_test = Nightly(
name='android_find_annotated_test',
branch='master',
minute=0,
hour=range(0,24,3),
builderNames=['Android Find Annotated Test'])
c['schedulers'] = [s_chromium,
s_chromium_android_dbg,
s_android_find_annotated_test,
]
####### BUILDERS
# buildbot/process/factory.py provides several BuildFactory classes you can
# start with, which implement build processes for common targets (GNU
# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
# base class, and is configured with a series of BuildSteps. When the build
# is run, the appropriate buildslave is told to execute each Step in turn.
# the first BuildStep is typically responsible for obtaining a copy of the
# sources. There are source-obtaining Steps in buildbot/process/step.py for
# CVS, SVN, and others.
builders = []
# ----------------------------------------------------------------------------
# FACTORIES
chromium_rel_archive = master_config.GetGSUtilUrl(
'chromium-build-transfer', 'Chromium FYI Builder')
chromium_dbg_archive = master_config.GetGSUtilUrl(
'chromium-build-transfer', 'Chromium FYI Builder (dbg)')
chromium_android_fyi_dbg_archive = master_config.GetGSUtilUrl(
'chromium-fyi-archive', 'chromium.fyi/Android Builder (dbg)')
f_win_analyze = \
annotator_factory.AnnotatorFactory().BaseFactory(
recipe='win_analyze')
def without(these, notthese):
return [x for x in these if x not in notthese]
tests_win_1 = ['base_unittests',
'cacheinvalidation_unittests',
'chrome_elf_unittests',
'components_unittests',
'courgette_unittests',
'crypto_unittests',
'device_unittests',
'extensions_unittests',
'gcm_unit_tests',
'google_apis_unittests',
'gpu',
'installer',
'jingle',
'media',
'net',
'ppapi_unittests',
'printing',
'remoting',
'sandbox',
'unit',
'url_unittests',
]
tests_win_2 = ['browser_tests',
'content_browsertests']
win_x64_tests = [
'base_unittests',
'cacheinvalidation_unittests',
'cc_unittests',
'chromedriver_unittests',
'components_unittests',
'content_browsertests',
'courgette_unittests',
'crypto_unittests',
'gcm_unit_tests',
'gpu',
'installer',
'interactive_ui_tests',
'jingle',
'media',
'nacl_integration',
'net',
'ppapi_unittests',
'printing',
'remoting',
'sandbox',
'unit',
'url_unittests',
# These are long, so do them last
'browser_tests',
'sync_integration',
]
f_chromium_dbg_android_mips_builder = \
annotator_factory.AnnotatorFactory().BaseFactory(
recipe='android/builder')
f_chromium_dbg_android_x86_builder = \
annotator_factory.AnnotatorFactory().BaseFactory(
recipe='android/builder')
def m_remote_run(recipe, **kwargs):
return remote_run_factory.RemoteRunFactory(
active_master=ActiveMaster,
repository='https://chromium.googlesource.com/chromium/tools/build.git',
recipe=recipe,
factory_properties={'path_config': 'kitchen'},
**kwargs)
f_chromium_mac_views_rel = m_remote_run('chromium')
f_chromium_mac_1013 = m_remote_run('chromium')
# ----------------------------------------------------------------------------
# BUILDER DEFINITIONS
# The 'builders' list defines the Builders. Each one is configured with a
# dictionary, using the following keys:
# name (required): the name used to describe this bilder
# slavename (required): which slave to use, must appear in c['slaves']
# builddir (optional): which subdirectory to run the builder in
# factory (required): a BuildFactory to define how the build is run
# periodicBuildTime (optional): if set, force a build every N seconds
# category (optional): it is not used in the normal 'buildbot' meaning. It is
# used by gatekeeper to determine which steps it should
# look for to close the tree.
#
m_annotator = annotator_factory.AnnotatorFactory()
b_chromium_win_analyze = {'name': 'Chromium Windows Analyze',
'builddir': 'Chromium_Windows_Analyze',
'factory': f_win_analyze,
'auto_reboot': False,
}
b_chromium_win7_goma = {
'name': 'Win7 Builder Goma Canary',
'factory': m_remote_run('chromium'),
'category': 'goma',
'auto_reboot': True,
}
b_chromium_win7_goma_dbg = {
'name': 'Win7 Builder (dbg) Goma Canary',
'factory': m_remote_run('chromium', timeout=3600),
'category': 'goma',
'auto_reboot': True,
}
b_clang_tot_linux_static_analyzer = {
'name': 'Linux Clang Analyzer',
'factory': m_remote_run('chromium'),
'category': 'clang tot',
'auto_reboot': False,
}
b_chromium_dbg_android_builder = {
'name': 'Android Builder (dbg)',
'factory': m_remote_run(
recipe='chromium', triggers=['android_dbg']),
'category': 'android_builder',
}
b_chromium_rel_android_remoting_tests = {
'name': 'Android Remoting Tests',
'factory': m_remote_run('chromium'),
'category': 'android_tests',
}
b_chromium_rel_android_find_annotated_test = {
'name': 'Android Find Annotated Test',
'factory': m_remote_run('chromium'),
'category': 'android_tests',
}
b_chromium_dbg_android_chromedriver_tests = {
'name': 'Android ChromeDriver Tests (dbg)',
'factory': annotator_factory.AnnotatorFactory().BaseFactory(
recipe='chromedriver'),
'category': 'android_tests',
}
b_chromium_dbg_android_vr_tests = {
'name': 'Android VR Tests',
'factory': m_remote_run('chromium'),
'category': 'android_tests',
}
b_win_10_fast_ring = {
'name': 'Win 10 Fast Ring',
'factory': m_remote_run('chromium'),
'category': 'win10',
'auto_reboot': True,
}
b_chromium_afl_upload_linux_asan = {
'name': 'Afl Upload Linux ASan',
'factory': m_annotator.BaseFactory('chromium_afl',
timeout=24000),
'category': 'chromium_afl',
'auto_reboot': True,
}
b_chromium_libfuzzer_upload_mac_asan = {
'name': 'Libfuzzer Upload Mac ASan',
'factory': m_annotator.BaseFactory('chromium_libfuzzer',
timeout=24000),
'category': 'chromium_libfuzzer',
'auto_reboot': True,
}
b_beta11_ios = {
'name': 'ios11-beta-simulator',
'factory': m_remote_run(
'ios/unified_builder_tester'
),
'category': 'web_view',
'auto_reboot': True,
}
b_gomacanary_ios = {
'name': 'ios-device-goma-canary-clobber',
'factory': m_remote_run(
'ios/unified_builder_tester'
),
'category': 'goma',
'auto_reboot': True,
}
b_chromium_oop_profiling_linux = {
'name': 'Out of Process Profiling Linux',
'factory': m_remote_run('chromium'),
'category': 'oop_profiling',
'auto_reboot': True,
}
b_chromium_oop_profiling_mac = {
'name': 'Out of Process Profiling Mac',
'factory': m_remote_run('chromium'),
'category': 'oop_profiling',
'auto_reboot': True,
}
b_chromium_oop_profiling_windows = {
'name': 'Out of Process Profiling Windows',
'factory': m_remote_run('chromium'),
'category': 'oop_profiling',
'auto_reboot': True,
}
def DeterministicBuilder(platform, debug=False):
suffix = ''
configuration = 'Release'
if debug:
suffix = ' (dbg)'
configuration = 'Debug'
return {
'name': '%s deterministic%s' % (platform, suffix),
# Work around an issue with NaCl build.
'slavebuilddir': 'chr',
'factory': annotator_factory.AnnotatorFactory().BaseFactory(
'swarming/deterministic_build',
factory_properties = {
'configuration': configuration,
}),
# Auto-reboot:
# - On windows to try to clear up TEMP fill up issues.
# - On OSX due to fork: Resource temporarily unavailable
'auto_reboot': True,
'category': 'deterministic',
}
b_mac_deterministic_build = DeterministicBuilder('Mac')
b_windows_deterministic_build = DeterministicBuilder('Windows')
b_windows_clang_deterministic_build = DeterministicBuilder('Windows Clang')
b_mac_deterministic_dbg_build = DeterministicBuilder('Mac', debug=True)
b_linux_annotator_rel = {
'name': 'linux-annotator-rel',
'factory': m_remote_run('chromium', max_time=12*3600, timeout=12*3600),
'category': 'linux',
'auto_reboot': True,
}
c['builders'] = [
# Windows release
b_chromium_win7_goma,
b_chromium_win7_goma_dbg,
# Windows /Analyze builder
b_chromium_win_analyze,
# Android
b_chromium_dbg_android_builder,
b_chromium_rel_android_remoting_tests,
b_chromium_rel_android_find_annotated_test,
b_chromium_dbg_android_vr_tests,
b_chromium_dbg_android_chromedriver_tests,
# Sanitizers
b_chromium_afl_upload_linux_asan,
b_chromium_libfuzzer_upload_mac_asan,
b_clang_tot_linux_static_analyzer,
# Windows 10 Fast Ring (early releases of Win 10).
b_win_10_fast_ring,
# Goma canary
b_gomacanary_ios,
# ios11-beta bot
b_beta11_ios,
# Deterministic build bot
b_mac_deterministic_build,
b_windows_deterministic_build,
b_windows_clang_deterministic_build,
b_mac_deterministic_dbg_build,
# Out of Process Profiling
b_chromium_oop_profiling_linux,
b_chromium_oop_profiling_mac,
b_chromium_oop_profiling_windows,
# linux-annotator-rel runnnins slower annotation-checking tests.
b_linux_annotator_rel,
]
c['builders'].extend([
{
'name': 'Linux remote_run Builder',
'factory': m_remote_run('chromium', timeout=2400, max_time=10800),
'category': 'remote_run',
'auto_reboot': True,
},
{
'name': 'Linux remote_run Tester',
'factory': m_remote_run('chromium'),
'category': 'remote_run',
'auto_reboot': True,
},
])
# Associate the slaves to the manual builders. The configuration is in
# slaves.cfg.
slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumFYI')
for builder in c['builders']:
builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
####### BUILDSLAVES
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
config.Master.GetBotPassword())
# Make sure everything works together.
master_utils.VerifySetup(c, slaves)
####### STATUS TARGETS
# Buildbot master url:
# Must come before AutoSetupMaster().
c['buildbotURL'] = ActiveMaster.buildbot_url
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(
c, ActiveMaster,
public_html="../master.chromium/public_html",
templates=['../master.chromium/templates'],
enable_http_status_push=ActiveMaster.is_production_host,
order_console_by_time=True)
# Add more.
if STATUS_PUSH:
from buildbot.status.status_push import HttpStatusPush
c['status'].append(HttpStatusPush(
'http://craebuild.appspot.com/status-listener'))
c['status'].append(failures_notifier.FailuresNotifier(
fromaddr=ActiveMaster.from_address, # Reply-To address
mode='failing',
categories=['goma'],
relayhost=config.Master.smtp,
subject='goma canary buildbot %(result)s in %(projectName)s '
'on %(builder)s',
extraRecipients=[
'ukai@google.com',
'yyanagisawa@google.com',
'shinyak@google.com',
'tikuta@google.com',
],
sendToInterestedUsers=False,
forgiving_steps=['diagnose_goma']))