blob: 4fa0ac311727b236265715baad1baa3950d5cf8a [file] [log] [blame]
# Copyright 2018 The LUCI Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Definitions imported by all other luci/**/*.star modules.
Should not import other LUCI modules to avoid dependency cycles.
"""
load("@stdlib//internal/error.star", "error")
load("@stdlib//internal/graph.star", "graph")
load("@stdlib//internal/sequence.star", "sequence")
load("@stdlib//internal/validate.star", "validate")
# Node edges (parent -> child):
# luci.project: root
# luci.project -> [luci.realm]
# luci.project -> [luci.custom_role]
# luci.realm -> [luci.realm]
# luci.custom_role -> [luci.custom_role]
# luci.bindings_root -> [luci.binding]
# luci.realm -> [luci.binding]
# luci.custom_role -> [luci.binding]
# luci.project -> luci.logdog
# luci.project -> luci.milo
# luci.project -> luci.cq
# luci.project -> [luci.bucket]
# luci.project -> [luci.milo_view]
# luci.project -> [luci.cq_group]
# luci.project -> [luci.notifiable]
# luci.project -> [luci.notifier_template]
# luci.bucket -> [luci.builder]
# luci.bucket -> [luci.gitiles_poller]
# luci.builder_ref -> luci.builder
# luci.builder -> [luci.triggerer]
# luci.builder -> luci.executable
# luci.gitiles_poller -> [luci.triggerer]
# luci.triggerer -> [luci.builder_ref]
# luci.milo_entries_root -> [luci.list_view_entry]
# luci.milo_entries_root -> [luci.console_view_entry]
# luci.milo_view -> luci.list_view
# luci.list_view -> [luci.list_view_entry]
# luci.list_view_entry -> list.builder_ref
# luci.milo_view -> luci.console_view
# luci.milo_view -> luci.external_console_view
# luci.console_view -> [luci.console_view_entry]
# luci.console_view_entry -> list.builder_ref
# luci.cq_verifiers_root -> [luci.cq_tryjob_verifier]
# luci.cq_group -> [luci.cq_tryjob_verifier]
# luci.cq_tryjob_verifier -> luci.builder_ref
# luci.cq_tryjob_verifier -> luci.cq_equivalent_builder
# luci.cq_equivalent_builder -> luci.builder_ref
# luci.notifiable -> [luci.builder_ref]
# luci.notifiable -> luci.notifier_template
def _namespaced_key(*pairs):
"""Returns a key namespaced to the current project.
Args:
*pairs: key path inside the current project namespace.
Returns:
graph.key.
"""
return graph.key(kinds.LUCI_NS, "", *pairs)
def _project_scoped_key(kind, attr, ref):
"""Returns a key either by grabbing it from the keyset or constructing.
Args:
kind: kind of the key to return.
attr: field name that supplies 'ref', for error messages.
ref: either a keyset or a string "<name>".
Returns:
graph.key of the requested kind.
"""
if graph.is_keyset(ref):
return ref.get(kind)
return graph.key(kinds.LUCI_NS, "", kind, validate.string(attr, ref))
def _bucket_scoped_key(kind, attr, ref, allow_external = False):
"""Returns either a bucket-scoped or a project-scoped key of the given kind.
Args:
kind: kind of the key to return.
attr: field name that supplies 'ref', for error messages.
ref: either a keyset, a string "<bucket>/<name>" or just "<name>". If it
is a string, it may optionally be prefixed with "<project>:" prefix to
indicate that this is a reference to a node in another project.
allow_external: True if it is OK to return a key from another project.
Returns:
graph.key of the requested kind.
"""
if graph.is_keyset(ref):
key = ref.get(kind)
if key.root.kind != kinds.LUCI_NS:
fail("unexpected key %s" % key)
if key.root.id and not allow_external:
fail("reference to %s in external project %r is not allowed here" % (kind, key.root.id))
return key
chunks = validate.string(attr, ref).split(":", 1)
if len(chunks) == 2:
proj, ref = chunks[0], chunks[1]
if proj:
if not allow_external:
fail("reference to %s in external project %r is not allowed here" % (kind, proj))
if proj == "*":
fail("buildbot builders no longer allowed here")
else:
proj, ref = "", chunks[0]
chunks = ref.split("/", 1)
if len(chunks) == 1:
return graph.key(kinds.LUCI_NS, proj, kind, chunks[0])
return graph.key(kinds.LUCI_NS, proj, kinds.BUCKET, chunks[0], kind, chunks[1])
def _builder_ref_key(ref, attr = "triggers", allow_external = False):
"""Returns luci.builder_ref key, auto-instantiating external nodes."""
if allow_external:
_maybe_add_external_builder(ref)
return _bucket_scoped_key(kinds.BUILDER_REF, attr, ref, allow_external = allow_external)
# Kinds is a enum-like struct with node kinds of various LUCI config nodes.
kinds = struct(
# This kind is used to namespace nodes from different projects:
# * ("@luci.ns", "", ...) - keys of nodes in the current project.
# * ("@luci.ns", "<name>", ...) - keys of nodes in another project.
LUCI_NS = "@luci.ns",
# Publicly declarable nodes.
PROJECT = "luci.project",
REALM = "luci.realm",
CUSTOM_ROLE = "luci.custom_role",
BINDING = "luci.binding",
LOGDOG = "luci.logdog",
BUCKET = "luci.bucket",
EXECUTABLE = "luci.executable",
BUILDER = "luci.builder",
GITILES_POLLER = "luci.gitiles_poller",
MILO = "luci.milo",
LIST_VIEW = "luci.list_view",
LIST_VIEW_ENTRY = "luci.list_view_entry",
CONSOLE_VIEW = "luci.console_view",
CONSOLE_VIEW_ENTRY = "luci.console_view_entry",
EXTERNAL_CONSOLE_VIEW = "luci.external_console_view",
CQ = "luci.cq",
CQ_GROUP = "luci.cq_group",
CQ_TRYJOB_VERIFIER = "luci.cq_tryjob_verifier",
NOTIFY = "luci.notify",
NOTIFIABLE = "luci.notifiable", # either luci.notifier or luci.tree_closer
NOTIFIER_TEMPLATE = "luci.notifier_template",
SHADOWED_BUCKET = "luci.shadowed_bucket",
SHADOW_OF = "luci.shadow_of",
# Internal nodes (declared internally as dependency of other nodes).
BUILDER_REF = "luci.builder_ref",
TRIGGERER = "luci.triggerer",
BINDINGS_ROOT = "luci.bindings_root",
MILO_ENTRIES_ROOT = "luci.milo_entries_root",
MILO_VIEW = "luci.milo_view",
CQ_VERIFIERS_ROOT = "luci.cq_verifiers_root",
CQ_EQUIVALENT_BUILDER = "luci.cq_equivalent_builder",
)
# Keys is a collection of key constructors for various LUCI config nodes.
keys = struct(
# Publicly declarable nodes.
project = lambda: _namespaced_key(kinds.PROJECT, "..."),
realm = lambda ref: _project_scoped_key(kinds.REALM, "realm", ref),
custom_role = lambda ref: _project_scoped_key(kinds.CUSTOM_ROLE, "role", ref),
logdog = lambda: _namespaced_key(kinds.LOGDOG, "..."),
bucket = lambda ref: _project_scoped_key(kinds.BUCKET, "bucket", ref),
executable = lambda ref: _project_scoped_key(kinds.EXECUTABLE, "executable", ref),
# TODO(vadimsh): Make them accept keysets if necessary. These currently
# require strings, not keysets. They are currently not directly used by
# anything, only through 'builder_ref' and 'triggerer' nodes. As such, they
# are never consumed via keysets.
builder = lambda bucket, name: _namespaced_key(kinds.BUCKET, bucket, kinds.BUILDER, name),
gitiles_poller = lambda bucket, name: _namespaced_key(kinds.BUCKET, bucket, kinds.GITILES_POLLER, name),
milo = lambda: _namespaced_key(kinds.MILO, "..."),
list_view = lambda ref: _project_scoped_key(kinds.LIST_VIEW, "list_view", ref),
console_view = lambda ref: _project_scoped_key(kinds.CONSOLE_VIEW, "console_view", ref),
external_console_view = lambda ref: _project_scoped_key(kinds.EXTERNAL_CONSOLE_VIEW, "external_console_view", ref),
cq = lambda: _namespaced_key(kinds.CQ, "..."),
cq_group = lambda ref: _project_scoped_key(kinds.CQ_GROUP, "cq_group", ref),
notify = lambda: _namespaced_key(kinds.NOTIFY, "..."),
notifiable = lambda ref: _project_scoped_key(kinds.NOTIFIABLE, "notifies", ref),
notifier_template = lambda ref: _project_scoped_key(kinds.NOTIFIER_TEMPLATE, "template", ref),
shadowed_bucket = lambda bucket_key: _namespaced_key(kinds.SHADOWED_BUCKET, bucket_key.id),
shadow_of = lambda bucket_key: _namespaced_key(kinds.SHADOW_OF, bucket_key.id),
# Internal nodes (declared internally as dependency of other nodes).
builder_ref = _builder_ref_key,
triggerer = lambda ref, attr = "triggered_by": _bucket_scoped_key(kinds.TRIGGERER, attr, ref),
bindings_root = lambda: _namespaced_key(kinds.BINDINGS_ROOT, "..."),
milo_entries_root = lambda: _namespaced_key(kinds.MILO_ENTRIES_ROOT, "..."),
milo_view = lambda name: _namespaced_key(kinds.MILO_VIEW, name),
cq_verifiers_root = lambda: _namespaced_key(kinds.CQ_VERIFIERS_ROOT, "..."),
# Generates a key of the given kind and name within some auto-generated
# unique container key.
#
# Used with LIST_VIEW_ENTRY, CONSOLE_VIEW_ENTRY and CQ_TRYJOB_VERIFIER
# helper nodes. They don't really represent any "external" entities, and
# their names don't really matter, other than for error messages.
#
# Note that IDs of keys whose kind stars with '_' (like '_UNIQUE' here),
# are skipped when printing the key in error messages. Thus the meaningless
# confusing auto-generated part of the key isn't showing up in error
# messages.
unique = lambda kind, name: graph.key("_UNIQUE", str(sequence.next(kind)), kind, name),
)
################################################################################
## builder_ref implementation.
def _builder_ref_add(target):
"""Adds two builder_ref nodes that have 'target' as a child.
Builder refs are named pointers to builders. Each builder has two such refs:
a bucket-scoped one (for when the builder is referenced using its full name
"<bucket>/<name>"), and a global one (when the builder is referenced just as
"<name>").
Global refs can have more than one child (which means there are multiple
builders with the same name in different buckets). Such refs are reported as
ambiguous by builder_ref.follow(...).
Args:
target: a graph.key (of BUILDER kind) to setup refs for.
Returns:
graph.key of added bucket-scoped BUILDER_REF node ("<bucket>/<name>" one).
"""
if target.kind != kinds.BUILDER:
fail("got %s, expecting a builder key" % (target,))
bucket = target.container.id # name of the bucket, as string
builder = target.id # name of the builder, as string
short = keys.builder_ref(builder)
graph.add_node(short, idempotent = True) # there may be such node already
graph.add_edge(short, target)
full = keys.builder_ref("%s/%s" % (bucket, builder))
graph.add_node(full)
graph.add_edge(full, target)
return full
def _builder_ref_follow(ref_node, context_node):
"""Given a BUILDER_REF node, returns a BUILDER graph.node the ref points to.
Emits an error and returns None if the reference is ambiguous (i.e.
'ref_node' has more than one child). Such reference can't be used to refer
to a single builder.
Note that the emitted error doesn't stop the generation phase, but marks it
as failed. This allows to collect more errors before giving up.
Args:
ref_node: graph.node with the ref.
context_node: graph.node where this ref is used, for error messages.
Returns:
graph.node of BUILDER kind.
"""
if ref_node.key.kind != kinds.BUILDER_REF:
fail("%s is not builder_ref" % ref_node)
# builder_ref nodes are always linked to something, see _builder_ref_add.
variants = graph.children(ref_node.key)
if not variants:
fail("%s is unexpectedly unconnected" % ref_node)
# No ambiguity.
if len(variants) == 1:
return variants[0]
error(
"ambiguous reference %r in %s, possible variants:\n %s",
ref_node.key.id,
context_node,
"\n ".join([str(v) for v in variants]),
trace = context_node.trace,
)
return None
# Additional API for dealing with builder_refs.
builder_ref = struct(
add = _builder_ref_add,
follow = _builder_ref_follow,
)
################################################################################
## Rudimentary hacky support for externally-defined builders.
def _maybe_add_external_builder(ref):
"""If 'ref' points to an external builder, defines corresponding nodes.
Kicks in if 'ref' is a string that has form "<project>:<bucket>/<name>".
Declares necessary luci.builder(...) and luci.builder_ref(...) nodes
namespaced to the corresponding external project.
"""
if type(ref) != "string" or ":" not in ref:
return
proj, rest = ref.split(":", 1)
if "/" not in rest:
return
bucket, name = rest.split("/", 1)
# TODO(vadimsh): This is very tightly coupled to the implementation of
# luci.builder(...) rule. It basically sets up same structure of nodes,
# except it doesn't fully populate luci.builder props (because we don't know
# them), and doesn't link the builder node to the project root (because we
# don't want to generate cr-buildbucket.cfg entry for it).
builder = graph.key(kinds.LUCI_NS, proj, kinds.BUCKET, bucket, kinds.BUILDER, name)
graph.add_node(builder, idempotent = True, props = {
"name": name,
"bucket": bucket,
"project": proj,
})
# This is roughly what _builder_ref_add does, except namespaced to 'proj'.
refs = [
graph.key(kinds.LUCI_NS, proj, kinds.BUILDER_REF, name),
graph.key(kinds.LUCI_NS, proj, kinds.BUCKET, bucket, kinds.BUILDER_REF, name),
]
for ref in refs:
graph.add_node(ref, idempotent = True)
graph.add_edge(ref, builder)
################################################################################
## triggerer implementation.
def _triggerer_add(owner, idempotent = False):
"""Adds two 'triggerer' nodes that have 'owner' as a parent.
Triggerer nodes are essentially nothing more than a way to associate a node
of arbitrary kind ('owner') to a list of builder_ref's of builders it
triggers (children of added 'triggerer' node).
We need this indirection to make 'triggered_by' relation work: when a
builder 'B' is triggered by something named 'T', we don't know whether 'T'
is another builder or a gitiles poller ('T' may not even be defined yet).
So instead all things that can trigger builders have an associated
'triggerer' node and 'T' names such a node.
To allow omitting bucket name when it is not important, each triggering
entity actually defines two 'triggerer' nodes: a bucket-scoped one and a
global one. During the graph traversal phase, children of both nodes are
merged.
If a globally named 'triggerer' node has more than one parent, it means
there are multiple things in different buckets that have the same name.
Using such references in 'triggered_by' relation is ambiguous. This
situation is detected during the graph traversal phase, see
triggerer.targets(...).
Args:
owner: a graph.key to setup triggerers for.
idempotent: if True, allow the triggerer node to be redeclared.
Returns:
graph.key of added bucket-scoped TRIGGERER node ("<bucket>/<name>" one).
"""
if not owner.container or owner.container.kind != kinds.BUCKET:
fail("got %s, expecting a bucket-scoped key" % (owner,))
bucket = owner.container.id # name of the bucket, as string
name = owner.id # name of the builder or poller, as string
# Short (not scoped to a bucket) keys are not unique, even for nodes that
# are marked as idempotent=False. Make sure it is OK to re-add them by
# marking them as idempotent. Dups are checked in triggerer.targets(...).
short = keys.triggerer(name)
graph.add_node(short, idempotent = True)
graph.add_edge(owner, short)
full = keys.triggerer("%s/%s" % (bucket, name))
graph.add_node(full, idempotent = idempotent)
graph.add_edge(owner, full)
return full
def _triggerer_targets(root):
"""Collects all BUILDER nodes triggered by the given node.
Enumerates all TRIGGERER children of 'root', and collects all BUILDER nodes
they refer to, following BUILDER_REF references.
Various ambiguities are reported as errors (which marks the generation
phase as failed). Corresponding nodes are skipped, to collect as many
errors as possible before giving up.
Args:
root: a graph.node that represents the triggering entity: something that
has a triggerer as a child.
Returns:
List of graph.node of BUILDER kind, sorted by key.
"""
out = set()
for t in graph.children(root.key, kinds.TRIGGERER):
parents = graph.parents(t.key)
for ref in graph.children(t.key, kinds.BUILDER_REF):
# Resolve builder_ref to a concrete builder. This may return None
# if the ref is ambiguous.
builder = builder_ref.follow(ref, root)
if not builder:
continue
# If 't' has multiple parents, it can't be used in 'triggered_by'
# relations, since it is ambiguous. Report this situation.
if len(parents) != 1:
error(
"ambiguous reference %r in %s, possible variants:\n %s",
t.key.id,
builder,
"\n ".join([str(v) for v in parents]),
trace = builder.trace,
)
else:
out = out.union([builder])
return graph.sorted_nodes(out)
triggerer = struct(
add = _triggerer_add,
targets = _triggerer_targets,
)
################################################################################
## Helpers for defining milo views (lists or consoles).
def _view_add_view(key, entry_kind, entry_ctor, entries, props):
"""Adds a *_view node, ensuring it doesn't clash with some other view.
Args:
key: a key of *_view node to add.
entry_kind: a kind of corresponding *_view_entry node.
entry_ctor: a corresponding *_view_entry rule.
entries: a list of *_view_entry, builder refs or dict.
props: properties for the added node.
Returns:
A keyset with added keys.
"""
graph.add_node(key, props)
# If there's some other view of different kind with the same name this will
# cause an error.
milo_view_key = keys.milo_view(key.id)
graph.add_node(milo_view_key)
# project -> milo_view -> *_view. Indirection through milo_view allows to
# treat different kinds of views uniformly.
graph.add_edge(keys.project(), milo_view_key)
graph.add_edge(milo_view_key, key)
# 'entry' is either a *_view_entry keyset, a builder_ref (perhaps given
# as a string) or a dict with parameters to *_view_entry rules. In the
# latter two cases, we add a *_view_entry for it automatically. This allows
# to somewhat reduce verbosity of list definitions.
for entry in validate.list("entries", entries):
if type(entry) == "dict":
entry = entry_ctor(**entry)
elif type(entry) == "string" or (graph.is_keyset(entry) and entry.has(kinds.BUILDER_REF)):
entry = entry_ctor(builder = entry)
graph.add_edge(key, entry.get(entry_kind))
return graph.keyset(key, milo_view_key)
def _view_add_entry(kind, view, builder, props = None):
"""Adds *_view_entry node.
Common implementation for list_view_node and console_view_node. Allows
referring to builders defined in other projects.
Args:
kind: a kind of the node to add (e.g. LIST_VIEW_ENTRY).
view: a key of the parent *_view to add the entry to, if known.
builder: a reference to builder.
props: properties for the added node.
Returns:
A keyset with the added key.
"""
if builder == None:
fail("'builder' is required")
builder = keys.builder_ref(builder, attr = "builder", allow_external = True)
# Note: name of this node is important only for error messages. It isn't
# showing up in any generated files and by construction it can't
# accidentally collide with some other name.
key = keys.unique(kind, builder.id)
graph.add_node(key, props)
if view != None:
graph.add_edge(parent = view, child = key)
if builder != None:
graph.add_edge(parent = key, child = builder)
# This is used to detect *_view_entry nodes that aren't connected to any
# *_view. Such orphan nodes aren't allowed.
graph.add_node(keys.milo_entries_root(), idempotent = True)
graph.add_edge(parent = keys.milo_entries_root(), child = key)
return graph.keyset(key)
view = struct(
add_view = _view_add_view,
add_entry = _view_add_entry,
)
################################################################################
## Helpers for defining luci.notifiable nodes.
def _notifiable_add(name, props, template, notified_by):
"""Adds a luci.notifiable node.
This is a shared portion of luci.notifier and luci.tree_closer
implementation. Nodes defined here are traversed by gen_notify_cfg in
generators.star.
Args:
name: name of the notifier or the tree closer.
props: a dict with node props.
template: an optional reference to a luci.notifier_template to link to.
notified_by: builders to link to.
Returns:
A keyset with the luci.notifiable key.
"""
key = keys.notifiable(validate.string("name", name))
graph.add_node(key, idempotent = True, props = props)
graph.add_edge(keys.project(), key)
for b in validate.list("notified_by", notified_by):
graph.add_edge(
parent = key,
child = keys.builder_ref(b, attr = "notified_by"),
title = "notified_by",
)
if template != None:
graph.add_edge(key, keys.notifier_template(template))
return graph.keyset(key)
notifiable = struct(
add = _notifiable_add,
)