blob: 6178eee44a96c4074127f57e6535e08d3cac47a0 [file] [log] [blame]
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# pylint: disable=line-too-long
import collections
import copy
import json
import re
from components.config import validation
from go.chromium.org.luci.buildbucket.proto import project_config_pb2
import errors
import experiments
import flatten_swarmingcfg
_DIMENSION_KEY_RGX = re.compile(r'^[a-zA-Z\_\-]+$')
# Copied from
# https://github.com/luci/luci-py/blob/75de6021b50a73e140eacfb80760f8c25aa183ff/appengine/swarming/server/task_request.py#L101
# Keep it synchronized.
_CACHE_NAME_RE = re.compile(ur'^[a-z0-9_]{1,4096}$')
# See https://chromium.googlesource.com/infra/luci/luci-py/+/master/appengine/swarming/server/service_accounts.py
_SERVICE_ACCOUNT_RE = re.compile(r'^[0-9a-zA-Z_\-\.\+\%]+@[0-9a-zA-Z_\-\.]+$')
def _validate_hostname(hostname, ctx):
if not hostname:
ctx.error('unspecified')
if '://' in hostname:
ctx.error('must not contain "://"')
def _validate_service_account(service_account, ctx):
if (service_account != 'bot' and
not _SERVICE_ACCOUNT_RE.match(service_account)):
ctx.error(
'value "%s" does not match %s', service_account,
_SERVICE_ACCOUNT_RE.pattern
)
# The below is covered by swarming_test.py and swarmbucket_api_test.py
def read_dimensions(builder_cfg): # pragma: no cover
"""Read the dimensions for a builder config.
This different from flatten_swarmingcfg.parse_dimensions in that this:
* Factors in the auto_builder_dimensions field.
Returns:
dimensions is returned as dict {key: {(value, expiration_secs)}}.
"""
dimensions = flatten_swarmingcfg.parse_dimensions(builder_cfg.dimensions)
if (builder_cfg.auto_builder_dimension == project_config_pb2.YES and
u'builder' not in dimensions):
dimensions[u'builder'] = {(builder_cfg.name, 0)}
return dimensions
def _validate_tag(tag, ctx):
# a valid swarming tag is a string that contains ":"
if ':' not in tag:
ctx.error('does not have ":": %s', tag)
name = tag.split(':', 1)[0]
if name.lower() == 'builder':
ctx.error(
'do not specify builder tag; '
'it is added by swarmbucket automatically'
)
def _validate_resultdb(resultdb, ctx):
with ctx.prefix('history_options: '):
if resultdb.history_options.HasField('commit'):
ctx.error('commit must be unset')
def _validate_dimensions(field_name, dimensions, ctx):
parsed = collections.defaultdict(set) # {key: {(value, expiration_secs)}}
expirations = set()
for dim in dimensions:
with ctx.prefix('%s "%s": ', field_name, dim):
parts = dim.split(':', 1)
if len(parts) != 2:
ctx.error('does not have ":"')
continue
key, value = parts
expiration_secs = 0
try:
expiration_secs = int(key)
except ValueError:
pass
else:
parts = value.split(':', 1)
if len(parts) != 2 or not parts[1]:
ctx.error('has expiration_secs but missing value')
continue
key, value = parts
valid_key = False
if not key:
ctx.error('no key')
elif not _DIMENSION_KEY_RGX.match(key):
ctx.error(
'key "%s" does not match pattern "%s"', key,
_DIMENSION_KEY_RGX.pattern
)
elif key == 'caches':
ctx.error(
'dimension key must not be "caches"; '
'caches must be declared via caches field'
)
else:
valid_key = True
valid_expiration_secs = False
if expiration_secs < 0 or expiration_secs > 21 * 24 * 60 * 60:
ctx.error('expiration_secs is outside valid range; up to 21 days')
elif expiration_secs % 60:
ctx.error('expiration_secs must be a multiple of 60 seconds')
else:
expirations.add(expiration_secs)
valid_expiration_secs = True
if valid_key and valid_expiration_secs:
parsed[key].add((value, expiration_secs))
if len(expirations) >= 6:
ctx.error('at most 6 different expiration_secs values can be used')
# Ensure that tombstones are not mixed with non-tomstones for the same key.
TOMBSTONE = ('', 0)
for key, entries in parsed.iteritems():
if TOMBSTONE not in entries or len(entries) == 1:
continue
for value, expiration_secs in entries:
if (value, expiration_secs) == TOMBSTONE:
continue
dim = flatten_swarmingcfg.format_dimension(key, value, expiration_secs)
with ctx.prefix('%s "%s": ', field_name, dim):
ctx.error('mutually exclusive with "%s:"', key)
def _validate_relative_path(path, ctx):
if not path:
ctx.error('required')
if '\\' in path:
ctx.error(
'cannot contain \\. On Windows forward-slashes will be '
'replaced with back-slashes.'
)
if '..' in path.split('/'):
ctx.error('cannot contain ".."')
if path.startswith('/'):
ctx.error('cannot start with "/"')
def _validate_exe_cfg(exe, ctx, final=True):
"""Validates an Executable message.
If final is False, does not validate for completeness.
"""
if final:
if not exe.cipd_package:
ctx.error('cipd_package: unspecified')
def _validate_recipe_cfg(recipe, ctx, final=True):
"""Validates a Recipe message.
If final is False, does not validate for completeness.
"""
if final:
if not recipe.name:
ctx.error('name: unspecified')
if not recipe.cipd_package:
ctx.error('cipd_package: unspecified')
validate_recipe_properties(recipe.properties, recipe.properties_j, ctx)
def validate_recipe_property(key, value, ctx):
if not key:
ctx.error('key not specified')
elif key == 'buildbucket':
ctx.error('reserved property')
elif key == '$recipe_engine/runtime':
if not isinstance(value, dict):
ctx.error('not a JSON object')
else:
for k in ('is_luci', 'is_experimental'):
if k in value:
ctx.error('key %r: reserved key', k)
def validate_recipe_properties(properties, properties_j, ctx):
keys = set()
def validate(props, is_json):
for p in props:
with ctx.prefix('%r: ', p):
if ':' not in p:
ctx.error('does not have a colon')
continue
key, value = p.split(':', 1)
if is_json:
try:
value = json.loads(value)
except ValueError as ex:
ctx.error('%s', ex)
continue
validate_recipe_property(key, value, ctx)
if key in keys:
ctx.error('duplicate property')
else:
keys.add(key)
with ctx.prefix('properties '):
validate(properties, False)
with ctx.prefix('properties_j '):
validate(properties_j, True)
def _validate_properties(properties, ctx):
try:
properties_d = json.loads(properties)
except ValueError as ex:
ctx.error('%s', ex)
return
if not isinstance(properties_d, dict):
ctx.error('properties is not a dict')
def validate_builder_cfg(builder, mixin_names, final, ctx):
"""Validates a Builder message.
Does not apply mixins, only checks that a referenced mixin exists.
If final is False, does not validate for completeness.
"""
if final or builder.name:
try:
errors.validate_builder_name(builder.name)
except errors.InvalidInputError as ex:
ctx.error('name: %s', ex.message)
if final or builder.swarming_host:
with ctx.prefix('swarming_host: '):
_validate_hostname(builder.swarming_host, ctx)
for i, t in enumerate(builder.swarming_tags):
with ctx.prefix('tag #%d: ', i + 1):
_validate_tag(t, ctx)
_validate_dimensions('dimension', builder.dimensions, ctx)
with ctx.prefix('resultdb:'):
_validate_resultdb(builder.resultdb, ctx)
cache_paths = set()
cache_names = set()
fallback_secs = set()
for i, c in enumerate(builder.caches):
with ctx.prefix('cache #%d: ', i + 1):
_validate_cache_entry(c, ctx)
if c.name:
if c.name in cache_names:
ctx.error('duplicate name')
else:
cache_names.add(c.name)
if c.path:
if c.path in cache_paths:
ctx.error('duplicate path')
else:
cache_paths.add(c.path)
if c.wait_for_warm_cache_secs:
with ctx.prefix('wait_for_warm_cache_secs: '):
if c.wait_for_warm_cache_secs < 60:
ctx.error('must be at least 60 seconds')
elif c.wait_for_warm_cache_secs % 60:
ctx.error('must be rounded on 60 seconds')
fallback_secs.add(c.wait_for_warm_cache_secs)
if len(fallback_secs) > 7:
# There can only be 8 task_slices.
ctx.error(
'too many different (%d) wait_for_warm_cache_secs values; max 7' %
len(fallback_secs)
)
has_exe = builder.HasField('exe')
has_recipe = builder.HasField('recipe')
if final and ((not has_exe and not has_recipe) or (has_exe and has_recipe)):
ctx.error('exactly one of exe or recipe must be specified')
if has_exe:
with ctx.prefix('exe: '):
_validate_exe_cfg(builder.exe, ctx, final=final)
elif has_recipe:
if builder.properties:
ctx.error('recipe and properties cannot be set together')
with ctx.prefix('recipe: '):
_validate_recipe_cfg(builder.recipe, ctx, final=final)
if builder.priority and (builder.priority < 20 or builder.priority > 255):
ctx.error('priority: must be in [20, 255] range; got %d', builder.priority)
if builder.properties:
_validate_properties(builder.properties, ctx)
if builder.service_account:
with ctx.prefix('service_account: '):
_validate_service_account(builder.service_account, ctx)
with ctx.prefix('experiments: '):
for exp_name, percent in sorted(builder.experiments.items()):
with ctx.prefix('"%s": ' % (exp_name)):
err = experiments.check_invalid_name(exp_name)
if err:
ctx.error(err)
if percent < 0 or percent > 100:
ctx.error('value must be in [0, 100]')
for m in builder.mixins:
if not m:
ctx.error('referenced mixin name is empty')
elif m not in mixin_names:
ctx.error('mixin "%s" is not defined', m)
# Limit (expiration+execution) to 47h. See max_grant_validity_duration in
# https://chrome-internal.googlesource.com/infradata/config/+/master/configs/luci-token-server/service_accounts.cfg
if builder.expiration_secs + builder.execution_timeout_secs > 47 * 60 * 60:
ctx.error('expiration_secs + execution_timeout_secs must be at most 47h')
def _validate_cache_entry(entry, ctx):
if not entry.name:
ctx.error('name: required')
elif not _CACHE_NAME_RE.match(entry.name):
ctx.error(
'name: "%s" does not match %s', entry.name, _CACHE_NAME_RE.pattern
)
with ctx.prefix('path: '):
_validate_relative_path(entry.path, ctx)
def validate_builder_mixins(mixins, ctx):
"""Validates mixins.
Checks that:
- mixins' attributes have valid values
- mixins have unique names
- mixins do not have circular references.
"""
by_name = {m.name: m for m in mixins}
seen = set()
for i, m in enumerate(mixins):
with ctx.prefix('builder_mixin %s: ' % (m.name or '#%s' % (i + 1))):
if not m.name:
# with final=False below, validate_builder_cfg will ignore name.
ctx.error('name: unspecified')
elif m.name in seen:
ctx.error('name: duplicate')
else:
seen.add(m.name)
validate_builder_cfg(m, by_name, False, ctx)
# Check circular references.
circles = set()
def check_circular(chain):
mixin = by_name[chain[-1]]
for sub_name in mixin.mixins:
if not sub_name or sub_name not in by_name:
# This may happen if validation above fails.
# We've already reported this, so ignore here.
continue
try:
recurrence = chain.index(sub_name)
except ValueError:
recurrence = -1
if recurrence >= 0:
circle = chain[recurrence:]
# make circle deterministic
smallest = circle.index(min(circle))
circle = circle[smallest:] + circle[:smallest]
circles.add(tuple(circle))
continue
chain.append(sub_name)
try:
check_circular(chain)
finally:
chain.pop()
for name in by_name:
check_circular([name])
for circle in sorted(circles):
circle = list(circle) + [circle[0]]
ctx.error('circular mixin chain: %s', ' -> '.join(circle))
def validate_project_cfg(swarming, mixins, mixins_are_valid, ctx):
"""Validates a project_config_pb2.Swarming message.
Args:
swarming (project_config_pb2.Swarming): the config to validate.
mixins (dict): {mixin_name: mixin}, builder mixins that may be used by
builders.
mixins_are_valid (bool): if True, mixins are valid.
"""
def make_subctx():
return validation.Context(
on_message=lambda msg: ctx.msg(msg.severity, '%s', msg.text)
)
if swarming.task_template_canary_percentage.value > 100:
ctx.error('task_template_canary_percentage.value must must be in [0, 100]')
builder_defaults = copy.copy(swarming.builder_defaults)
builder_defaults.swarming_host = (
builder_defaults.swarming_host or swarming.hostname
)
should_try_merge = mixins_are_valid
if swarming.HasField('builder_defaults'):
with ctx.prefix('builder_defaults: '):
if builder_defaults.name:
ctx.error('name: not allowed')
subctx = make_subctx()
validate_builder_cfg(builder_defaults, mixins, False, subctx)
if subctx.result().has_errors:
should_try_merge = False
seen = set()
for i, b in enumerate(swarming.builders):
with ctx.prefix('builder %s: ' % (b.name or '#%s' % (i + 1))):
# Validate b before merging, otherwise merging will fail.
subctx = make_subctx()
validate_builder_cfg(b, mixins, False, subctx)
if subctx.result().has_errors or not should_try_merge:
# Do no try to merge invalid configs.
continue
merged = copy.deepcopy(b)
flatten_swarmingcfg.flatten_builder(merged, builder_defaults, mixins)
if merged.name in seen:
ctx.error('name: duplicate')
else:
seen.add(merged.name)
validate_builder_cfg(merged, mixins, True, ctx)
def _validate_package(package, ctx, allow_predicate=True):
if not package.package_name:
ctx.error('package_name is required')
if not package.version:
ctx.error('version is required')
if allow_predicate:
_validate_builder_predicate(package.builders, ctx)
elif package.HasField('builders'): # pragma: no cover
ctx.error('builders is not allowed')
def _validate_builder_predicate(predicate, ctx):
for regex in predicate.regex:
with ctx.prefix('regex %r: ', regex):
_validate_regex(regex, ctx)
for regex in predicate.regex_exclude:
with ctx.prefix('regex_exclude %r: ', regex):
_validate_regex(regex, ctx)
def _validate_regex(regex, ctx):
try:
re.compile(regex)
except re.error as ex:
ctx.error('invalid: %s', ex)
def validate_service_cfg(swarming, ctx):
with ctx.prefix('milo_hostname: '):
_validate_hostname(swarming.milo_hostname, ctx)
# Validate packages.
for i, p in enumerate(swarming.user_packages):
with ctx.prefix('user_package[%d]: ' % i):
_validate_package(p, ctx)
with ctx.prefix('bbagent_package: '):
_validate_package(swarming.bbagent_package, ctx)
with ctx.prefix('kitchen_package: '):
_validate_package(swarming.kitchen_package, ctx, allow_predicate=False)