masters/master.tryserver.webrtc/master.cfg

#!/usr/bin/env python
#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
#  Use of this source code is governed by a BSD-style license
#  that can be found in the LICENSE file in the root of the source
#  tree. An additional intellectual property rights grant can be found
#  in the file PATENTS.  All contributing project authors may
#  be found in the AUTHORS file in the root of the source tree.

import os

# These modules come from scripts, which must be in the PYTHONPATH.
from master import master_utils
from master import slaves_list
from master.builders_pools import BuildersPools
from master.factory import annotator_factory
from master.factory import remote_run_factory
from master.try_job_http import TryJobHTTP
from master.try_mail_notifier import TryMailNotifier

import config
import master_site_config

ActiveMaster = master_site_config.WebRTCTryServer

MAIL_NOTIFIER = True
LISTEN_TO_SVN = ActiveMaster.svn_url and ActiveMaster.is_production_host

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

config.DatabaseSetup(c)

####### CHANGESOURCES
c['change_source'] = []

# Avoid merging requests.
c['mergeRequests'] = lambda *_: False

####### BUILDERS

# Recipe based builders.
specs = [
  {'name': 'win_compile_dbg', 'slavebuilddir': 'win'},
  {'name': 'win_compile_rel', 'slavebuilddir': 'win'},
  {'name': 'win_compile_x64_dbg', 'slavebuilddir': 'win'},
  {'name': 'win_compile_x64_rel', 'slavebuilddir': 'win'},
  {'name': 'win_dbg', 'slavebuilddir': 'win'},
  {'name': 'win_rel', 'slavebuilddir': 'win'},
  {'name': 'win_x64_dbg', 'slavebuilddir': 'win'},
  {'name': 'win_x64_rel', 'slavebuilddir': 'win'},
  {'name': 'win_x64_gyp_dbg', 'slavebuilddir': 'win_gyp'},
  {'name': 'win_x64_gyp_rel', 'slavebuilddir': 'win_gyp'},
  {'name': 'win_clang_dbg', 'slavebuilddir': 'win_clang'},
  {'name': 'win_clang_rel', 'slavebuilddir': 'win_clang'},
  {'name': 'win_x64_clang_dbg', 'slavebuilddir': 'win_clang'},
  {'name': 'win_x64_clang_rel', 'slavebuilddir': 'win_clang'},
  {'name': 'win_baremetal', 'slavebuilddir': 'win'},
  {'name': 'win_asan', 'slavebuilddir': 'win_asan'},
  {'name': 'win_drmemory_light', 'slavebuilddir': 'win-drmemory'},
  {'name': 'win_drmemory_full', 'slavebuilddir': 'win-drmemory'},
  {'name': 'win_swarming', 'slavebuilddir': 'win_swarming'},
  {'name': 'mac_compile_dbg', 'slavebuilddir': 'mac32'},
  {'name': 'mac_compile_rel', 'slavebuilddir': 'mac32'},
  {'name': 'mac_dbg', 'slavebuilddir': 'mac32'},
  {'name': 'mac_rel', 'slavebuilddir': 'mac32'},
  {'name': 'mac_gyp_dbg', 'slavebuilddir': 'mac_gyp'},
  {'name': 'mac_gyp_rel', 'slavebuilddir': 'mac_gyp'},
  {'name': 'mac_baremetal', 'slavebuilddir': 'mac32'},
  {'name': 'mac_swarming', 'slavebuilddir': 'mac64_swarming'},
  {'name': 'ios_dbg', 'slavebuilddir': 'mac32', 'recipe': 'webrtc/ios'},
  {'name': 'ios_rel', 'slavebuilddir': 'mac32', 'recipe': 'webrtc/ios'},
  {'name': 'ios_arm64_dbg', 'slavebuilddir': 'mac64', 'recipe': 'webrtc/ios'},
  {'name': 'ios_arm64_rel', 'slavebuilddir': 'mac64', 'recipe': 'webrtc/ios'},
  {'name': 'ios32_sim_dbg', 'slavebuilddir': 'mac32', 'recipe': 'webrtc/ios'},
  {'name': 'ios64_sim_dbg', 'slavebuilddir': 'mac64', 'recipe': 'webrtc/ios'},
  {'name': 'ios64_gyp_dbg', 'slavebuilddir': 'mac64_gyp', 'recipe': 'webrtc/ios'},
  {'name': 'ios64_gyp_rel', 'slavebuilddir': 'mac64_gyp', 'recipe': 'webrtc/ios'},
  {
    'name': 'ios_api_framework',
    'slavebuilddir': 'mac64',
    'recipe': 'webrtc/ios_api_framework',
  },
  {'name': 'mac_asan', 'slavebuilddir': 'mac_asan'},
  {'name': 'linux_compile_dbg', 'slavebuilddir': 'linux64'},
  {'name': 'linux_compile_rel', 'slavebuilddir': 'linux64'},
  {'name': 'linux_dbg', 'slavebuilddir': 'linux64'},
  {'name': 'linux_rel', 'slavebuilddir': 'linux64'},
  {'name': 'linux32_dbg', 'slavebuilddir': 'linux32'},
  {'name': 'linux32_rel', 'slavebuilddir': 'linux32'},
  {'name': 'linux_baremetal', 'slavebuilddir': 'linux64'},
  {'name': 'linux_memcheck', 'slavebuilddir': 'linux_memcheck_or_tsan'},
  {'name': 'linux_msan', 'slavebuilddir': 'linux_msan'},
  {'name': 'linux_tsan2', 'slavebuilddir': 'linux_tsan2'},
  {'name': 'linux_arm', 'slavebuilddir': 'linux_arm'},
  {'name': 'linux_asan', 'slavebuilddir': 'linux_asan'},
  {'name': 'linux_ubsan', 'slavebuilddir': 'linux_ubsan'},
  {'name': 'linux_ubsan_vptr', 'slavebuilddir': 'linux_ubsan_vptr'},
  {'name': 'linux_gyp_dbg', 'slavebuilddir': 'linux64_gyp'},
  {'name': 'linux_gyp_rel', 'slavebuilddir': 'linux64_gyp'},
  {
    'name': 'linux_libfuzzer_rel',
    'recipe': 'webrtc/libfuzzer',
    'slavebuilddir': 'linux64_libfuzzer',
  },
  {'name': 'linux_swarming', 'slavebuilddir': 'linux64_swarming'},
  {'name': 'android_compile_dbg', 'slavebuilddir': 'android'},
  {'name': 'android_compile_rel', 'slavebuilddir': 'android'},
  {'name': 'android_compile_arm64_dbg', 'slavebuilddir': 'android_arm64'},
  {'name': 'android_compile_arm64_rel', 'slavebuilddir': 'android_arm64'},
  {'name': 'android_compile_mips_dbg', 'slavebuilddir': 'android_mips'},
  {'name': 'android_compile_x86_rel', 'slavebuilddir': 'android_x86'},
  {'name': 'android_compile_x86_dbg', 'slavebuilddir': 'android_x86'},
  {'name': 'android_compile_x64_dbg', 'slavebuilddir': 'android_x64'},
  {'name': 'android_dbg', 'slavebuilddir': 'android'},
  {'name': 'android_rel', 'slavebuilddir': 'android'},
  {'name': 'android_clang_dbg', 'slavebuilddir': 'android_clang'},
  {'name': 'android_arm64_rel', 'slavebuilddir': 'android_arm64'},
  {'name': 'android_n6', 'slavebuilddir': 'android'},
  {'name': 'android_gyp_dbg', 'slavebuilddir': 'android_gyp'},
  {'name': 'android_gyp_rel', 'slavebuilddir': 'android_gyp'},
]


m_annotator = annotator_factory.AnnotatorFactory()


def m_remote_run(recipe, **kwargs):
  return remote_run_factory.RemoteRunFactory(
      active_master=ActiveMaster,
      repository='https://chromium.googlesource.com/chromium/tools/build.git',
      recipe=recipe,
      factory_properties={'path_config': 'kitchen'},
      **kwargs)


c['builders'] = [
    {
      'name': spec['name'],
      # TODO(sergiyb): Remove the timeout below after all bots have synched past
      # Blink merge commit.
      # remote_run is not working for the webrtc/ios recipe: crbug.com/637666.
      'factory': m_annotator.BaseFactory(spec['recipe'], timeout=3600)
           if 'recipe' in spec and spec['recipe'] in ('webrtc/ios',
                                                      'webrtc/libfuzzer')
           else m_remote_run(spec.get('recipe', 'webrtc/standalone'),
                             timeout=3600),
      'slavebuilddir': spec['slavebuilddir'],
    } for spec in specs
]


# Presubmit builder.
c['builders'].append(
  {
    'name': 'presubmit',
    'factory': m_annotator.BaseFactory('run_presubmit',
                                       {'repo_name': 'webrtc',
                                        'runhooks': True}),
    'slavebuilddir': 'linux64',
  }
)

# Slaves are loaded from slaves.cfg.
slaves = slaves_list.SlavesList('slaves.cfg', 'WebRTCTryServer')

for builder in c['builders']:
  # Associate the slaves to the builders. The configuration is in slaves.cfg.
  builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
  # Disable auto_reboot when testing locally, but don't change for production.
  if not ActiveMaster.is_production_host:
    builder['auto_reboot'] = False


####### BUILDSLAVES

# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                           config.Master.GetBotPassword())
# Make sure everything works together.
master_utils.VerifySetup(c, slaves)

####### SCHEDULERS

pools = BuildersPools('webrtc')
for spec in specs:
  pools['webrtc'].append(spec['name'])

# Configure the Schedulers;
c['schedulers'] = []

last_good_urls = {'webrtc': ActiveMaster.last_good_url}
code_review_sites = {'webrtc': ActiveMaster.code_review_site}

c['schedulers'].append(TryJobHTTP(
    name='webrtc_try_job_http',
    port=ActiveMaster.try_job_port,
    last_good_urls=last_good_urls,
    code_review_sites=code_review_sites,
    pools=pools))

if LISTEN_TO_SVN:
  from master.try_job_svn import TryJobSubversion
  c['schedulers'].append(TryJobSubversion(
      name='webrtc_try_job_svn',
      svn_url=ActiveMaster.svn_url,
      last_good_urls=last_good_urls,
      code_review_sites=code_review_sites,
      pools=pools))

####### STATUS TARGETS

# Buildbot master url:
# Must come before AutoSetupMaster().
c['buildbotURL'] = ActiveMaster.buildbot_url

# Adds common status and tools to this master.
# Use our own mail notifier.
master_utils.AutoSetupMaster(c, ActiveMaster,
                             order_console_by_time=True,
                             public_html='../master.chromium/public_html',
                             templates=['./templates',
                                        '../master.chromium/templates'])

if MAIL_NOTIFIER:
  # Add a dumb MailNotifier first so it will be used for BuildSlave with
  # notify_on_missing set when they go missing.
  from buildbot.status import mail
  c['status'].append(mail.MailNotifier(
      fromaddr=ActiveMaster.from_address,
      builders=[],
      relayhost=config.Master.smtp,
      lookup=master_utils.UsersAreEmails()))

  # Try job result emails.
  from master.try_mail_notifier import TryMailNotifier
  c['status'].append(TryMailNotifier(
      fromaddr=ActiveMaster.from_address,
      reply_to=ActiveMaster.reply_to,
      subject='try %(result)s for %(reason)s @ r%(revision)s',
      mode='all',
      relayhost=config.Master.smtp,
      lookup=master_utils.UsersAreEmails()))

# Keep last try jobs, the default is too low. Must keep at least a few days
# worth of try jobs.
c['buildHorizon'] = 2000
c['logHorizon'] = 2000
# Must be at least 2x the number of slaves.
c['eventHorizon'] = 100
# Must be at least 2x the number of on-going builds.
c['buildCacheSize'] = 100

####### PROJECT IDENTITY

# The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.
c['projectURL'] = 'http://dev.chromium.org/developers/testing/try-server-usage'