masters/master.chromium.perf/master.cfg

# Copyright 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is the buildmaster config file for the 'chromium.perf' bot. It must
# be installed as 'master.cfg' in your buildmaster's base directory
# (although the filename can be changed with the --basedir option to
# 'mktap buildbot master').

# It has one job: define a dictionary named BuildmasterConfig. This
# dictionary has a variety of keys to control different aspects of the
# buildmaster. They are documented in docs/config.xhtml .

# This file follows this naming convention:
# Factories: f_cr_rel_[type]
# Builders:  b_chromium_rel_[os]_[type]
# BuildDir:  chromium-rel-[os]-[type]
#
# os = xp/vista/linux/mac
# type = perf

from buildbot.process.properties import WithProperties
from buildbot.scheduler import Scheduler
from buildbot.scheduler import Triggerable

from common import chromium_utils

from master import chromium_notifier
from master import gitiles_poller
from master import master_config
from master import master_utils
from master import slaves_list
from master.factory import annotator_factory
from master.factory import remote_run_factory

import config
import master_site_config

ActiveMaster = master_site_config.ChromiumPerf


# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)
c['slavePortnum'] = ActiveMaster.slave_port

# Enable compression for any stdio log file larger than 10 MB.
c['logCompressionLimit'] = 1024 * 1024 * 10 # 10 MB

# Load the list of slaves.
slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumPerf')

config.DatabaseSetup(c)

def recipe_factory(recipe, **kwargs):
  f_annotations = annotator_factory.AnnotatorFactory(ActiveMaster)
  return f_annotations.BaseFactory(recipe=recipe, **kwargs)


revision_getter = master_utils.ConditionalProperty(
    lambda build: build.getProperty('revision'),
    WithProperties('%(revision)s'),
    'master')


def m_remote_run_chromium_src(recipe, **kwargs):
  kwargs.setdefault('revision', revision_getter)
  return remote_run_factory.RemoteRunFactory(
      active_master=ActiveMaster,
      repository='https://chromium.googlesource.com/chromium/src.git',
      recipe=recipe,
      factory_properties={'path_config': 'kitchen'},
      use_gitiles=True,
      **kwargs)


# ------------------------------------------------------------------------------
# Change Sources.

# Polls config.Master.trunk_url for changes
master_poller = gitiles_poller.GitilesPoller(
    'https://chromium.googlesource.com/chromium/src')

c['change_source'] = [master_poller]

# End Change Sources.
# ------------------------------------------------------------------------------


# ----------------------------------------------------------------------------
# Builders.

# The 'builders' list defines the Builders. Each one is configured with a
# dictionary, using the following keys:
#  name (required): the name used to describe this bilder
#  builddir (required): which subdirectory to run the builder in
#  factory (required): a BuildFactory to define how the build is run
#  periodicBuildTime (optional): if set, force a build every N seconds
#  category (optional): it is not used in the normal 'buildbot' meaning. It is
#                       used by gatekeeper to determine which steps it should
#                       look for to close the tree.
#

def _TriggerName(platform, target_bits):
  return '%s_%d' % (platform, target_bits)


_category_index = enumerate(
    ('builders', 'android', 'win', 'mac', 'linux'))
_category_index = {category: index + 1 for index, category in _category_index}
def _CategoryIndex(category):
  return _category_index[category]


_builder_names = []
def _AddBuilder(name, platform, timeout=None, target_bits=64):
  if platform == 'android':
    factory = recipe_factory
    recipe = 'android/builder'
  else:
    factory = m_remote_run_chromium_src
    recipe = 'chromium'

  factory_kwargs = dict(
    recipe=recipe,
    triggers=[_TriggerName(platform, target_bits)],
  )
  if timeout:
    factory_kwargs['timeout'] = timeout

  c['builders'].append({
      'name': name,
      'factory': factory(**factory_kwargs),
      'category':
          '%d%s|%s' % (_CategoryIndex('builders'), 'builders', platform),
      'mergeRequests': False,
  })

  _builder_names.append(name)


_trigger_testers = {}
def _AddTesterShard(name, factory, platform, target_bits):
  c['builders'].append({
      'name': name,
      'factory': factory,
      'category': '%d%s|builder_testers' % (_CategoryIndex(platform), platform),
  })

  trigger_name = _TriggerName(platform, target_bits)
  if trigger_name not in _trigger_testers:
    _trigger_testers[trigger_name] = []
  _trigger_testers[trigger_name].append(name)


def _AddTester(name, platform, num_shards=1, target_bits=64):
  if platform == 'android':
    factory = m_remote_run_chromium_src('android/perf')
  else:
    factory = m_remote_run_chromium_src('chromium')

  if num_shards == 1:
    _AddTesterShard(name, factory, platform, target_bits)
  else:
    for index in xrange(num_shards):
      indexed_name = '%s (%d)' % (name, index + 1)
      _AddTesterShard(indexed_name, factory, platform, target_bits)


c['builders'] = []


_AddBuilder('Android Builder', 'android', target_bits=32)
_AddBuilder('Android arm64 Builder', 'android')
_AddBuilder('Win Builder', 'win', timeout=3600, target_bits=32)
# TODO(dtu): decrease the timeout when https://crbug.com/617982 is fixed.
_AddBuilder('Win x64 Builder', 'win', timeout=7200)
_AddBuilder('Mac Builder', 'mac')
# TODO(krasin): decrease the timeout when https://crbug.com/569732 is fixed.
_AddBuilder('Linux Builder', 'linux', timeout=4800)


_AddTester('Android Galaxy S5 Perf', 'android', num_shards=3, target_bits=32)
_AddTester('Android Nexus5 Perf', 'android', num_shards=3, target_bits=32)
_AddTester('Android Nexus5X Perf', 'android', num_shards=3)
_AddTester('Android Nexus6 Perf', 'android', num_shards=3, target_bits=32)
_AddTester('Android Nexus7v2 Perf', 'android', num_shards=3, target_bits=32)
_AddTester('Android Nexus9 Perf', 'android', num_shards=3)
_AddTester('Android One Perf', 'android', num_shards=3, target_bits=32)
_AddTester('Android Nexus5X WebView Perf', 'android', num_shards=3)
_AddTester('Android Nexus6 WebView Perf', 'android', num_shards=3,
           target_bits=32)


_AddTester('Win Zenbook Perf', 'win', num_shards=5)
_AddTester('Win 10 High-DPI Perf', 'win', num_shards=5)
_AddTester('Win 10 Perf', 'win', num_shards=5)
_AddTester('Win 8 Perf', 'win', num_shards=5)
_AddTester('Win 7 Perf', 'win', num_shards=5, target_bits=32)
_AddTester('Win 7 x64 Perf', 'win', num_shards=5)
_AddTester('Win 7 ATI GPU Perf', 'win', num_shards=5)
_AddTester('Win 7 Intel GPU Perf', 'win', num_shards=5)
_AddTester('Win 7 Nvidia GPU Perf', 'win', num_shards=5)


_AddTester('Mac 10.11 Perf', 'mac', num_shards=5)
_AddTester('Mac 10.10 Perf', 'mac', num_shards=5)
_AddTester('Mac Retina Perf', 'mac', num_shards=5)
_AddTester('Mac HDD Perf', 'mac', num_shards=5)


_AddTester('Linux Perf', 'linux', num_shards=5)

# End Builders.
# ------------------------------------------------------------------------------


# ------------------------------------------------------------------------------
# Schedulers.

## configure the Schedulers

c['schedulers'] = [
    Scheduler(name='chromium',
              branch='master',
              builderNames=_builder_names)
]


for trigger_name, tester_names in _trigger_testers.iteritems():
  c['schedulers'].append(Triggerable(trigger_name, tester_names))

# End Schedulers.
# ------------------------------------------------------------------------------


####### BUILDSLAVES

# Associate the slaves to the manual builders. The configuration is in
# slaves.cfg.
for builder in c['builders']:
  builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])

# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                           config.Master.GetBotPassword())
master_utils.VerifySetup(c, slaves)


####### STATUS TARGETS

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.Waterfall page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

#Must come before AutoSetupMaster().
c['buildbotURL'] = ActiveMaster.buildbot_url

# Adds common status and tools to this master.
master_utils.AutoSetupMaster(c, ActiveMaster,
    public_html='../master.chromium/public_html',
    templates=['./templates', '../master.chromium/templates'],
    tagComparator=master_poller.comparator,
    enable_http_status_push=ActiveMaster.is_production_host)

# The default is to keep 1000 builds and 500 log files.  The log files from this
# master are very big, so keep fewer.
c['buildHorizon'] = 500
c['logHorizon'] = 200


####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = ActiveMaster.project_name
c['projectURL'] = config.Master.project_url