| # Copyright 2022 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Utility library for working with lucicfg graph nodes.""" |
| |
| load("@stdlib//internal/graph.star", "graph") |
| load("@stdlib//internal/sequence.star", "sequence") |
| load("@stdlib//internal/luci/common.star", "builder_ref", "keys", "kinds") |
| |
| _CHROMIUM_NS_KIND = "@chromium" |
| |
| def _create_singleton_node_type(kind): |
| """Create a singleton node type. |
| |
| Singleton nodes types only allow for a single node of the type to exist. |
| This can be used for creating configuration nodes for generators since |
| generators are unable to access lucicfg vars. |
| |
| Args: |
| kind: (str) An identifier for the kind of the node. Must be unique |
| within the chromium namespace. |
| |
| Returns: |
| A node type that can be used for creating and getting a node of |
| the given kind. |
| |
| The type has the following properties: |
| * kind: The kind of node of the type. |
| |
| The node type has the following methods: |
| * key(): Creates a key for the node. |
| * add(**kwargs): Adds a node with a key created via `key()`. |
| `graph.add_node` will be called with the key and `**kwargs`. |
| Returns the key. |
| * get(): Gets the node with the key given by |
| `key(bucket_name, key_value)`. |
| """ |
| |
| def key(): |
| return graph.key(_CHROMIUM_NS_KIND, "", kind, "") |
| |
| def add(**kwargs): |
| k = key() |
| graph.add_node(k, **kwargs) |
| return k |
| |
| def get(): |
| return graph.node(key()) |
| |
| return struct( |
| kind = kind, |
| key = key, |
| add = add, |
| get = get, |
| ) |
| |
| _ANONYMOUS_PREFIX = "<anonymous>" |
| |
| def _create_unscoped_node_type(kind, allow_empty_id = False): |
| """Create an unscoped node type. |
| |
| Unscoped node types only allow for one node to exist with a given key_value. |
| Key values are arbitrary, it is up to the calling code to assign meaning to |
| the key values and enforce validity. |
| |
| Args: |
| kind: (str) An identifier for the kind of the node. Must be unique |
| within the chromium namespace. |
| allow_empty_id: (bool) Whether or not to allow the creation of nodes |
| without providing an ID value. This can allow for creating resources |
| that are defined within the definition of other resources without |
| requiring assigning an ID upfront. Instead, a unique ID will be |
| generated. |
| |
| Returns: |
| A node type that can be used for creating and getting nodes of |
| the given kind. |
| |
| The type has the following properties: |
| * kind: The kind of nodes of the type. |
| |
| The node types has the following methods: |
| * key(key_id_or_keyset): Gets a key of kind. key_id_or_keyset can either |
| be the ID value for the key or it can be a keyset, in which case the |
| key of kind will be extracted from the keyset. |
| * add(key_id, **kwargs): Adds a node with a key created via |
| `key(key_id)`. `graph.add_node` will be called with the key and |
| `**kwargs`. Returns the key. If allow_empty_id is True, key_id will |
| have the defult value of None and a None value for key_id will |
| create a node with a key that is unique within the lucicfg run. |
| * get(key_id): Gets the node with key given by `key(key_id)`. |
| """ |
| |
| def key(key_id_or_keyset): |
| if graph.is_keyset(key_id_or_keyset): |
| return key_id_or_keyset.get(kind) |
| return graph.key(_CHROMIUM_NS_KIND, "", kind, key_id_or_keyset) |
| |
| if allow_empty_id: |
| def add(key_id = None, **kwargs): |
| if key_id == None: |
| key_id = "{}:{}".format(_ANONYMOUS_PREFIX, sequence.next(kind)) |
| elif key_id.startswith(_ANONYMOUS_PREFIX): |
| fail("cannot specify a key ID with prefix \"{}\"".format(_ANONYMOUS_PREFIX)) |
| k = key(key_id) |
| graph.add_node(k, **kwargs) |
| return k |
| else: |
| def add(key_id, **kwargs): |
| k = key(key_id) |
| graph.add_node(k, **kwargs) |
| return k |
| |
| def get(key_id): |
| return graph.node(key(key_id)) |
| |
| return struct( |
| kind = kind, |
| key = key, |
| add = add, |
| get = get, |
| ) |
| |
| def _create_scoped_node_type(kind, scope_kind): |
| """Create a node type scoped to another kind. |
| |
| Scoped node types allow for a node to exist with a given key value per key |
| value of the scope kind. Key values are arbitrary, it is up to the calling |
| code to assign meaning to the key values and enforce validity. |
| |
| Args: |
| kind: (str) An identifier for the kind of the node. Must be unique within |
| the chromium namespace. |
| scope_kind: (str) An identifier for the scope kind. |
| |
| Returns: |
| A node type that can be used for creating and getting nodes of |
| the given kind. |
| |
| The type has the following properties: |
| * kind: The kind of nodes of the type. |
| |
| The node types has the following methods: |
| * key(scope_key_value, key_value): Creates a key with the given scope |
| value as the container and key_value as the ID. |
| * add(scope_key_value, key_value, **kwargs): Adds a node with a key |
| created via `key(scope_key_value, key_value)`. `graph.add_node` will |
| be called with the key and `**kwargs`. Returns the key. |
| * get(scope_key_value, key_value): Gets the node with the key given by |
| `key(scope_key_value, key_value)`. |
| """ |
| |
| def key(scope_key_value, key_value): |
| return graph.key(_CHROMIUM_NS_KIND, "", scope_kind, scope_key_value, kind, key_value) |
| |
| def add(scope_key_value, key_value, **kwargs): |
| k = key(scope_key_value, key_value) |
| graph.add_node(k, **kwargs) |
| return k |
| |
| def get(scope_key_value, key_value): |
| return graph.node(key(scope_key_value, key_value)) |
| |
| return struct( |
| kind = kind, |
| key = key, |
| add = add, |
| get = get, |
| ) |
| |
| def _create_node_type_with_builder_ref(kind): |
| """Create a node type that allows reference via builder name. |
| |
| Node types created by this function result in the creation of 2 different |
| kinds: the target kind (`kind`) and a ref kind. One node of the target kind |
| can exist for each builder defined in the project. Additionally, 2 nodes of |
| the ref kind will exist that have edges to the node of the target kind. The |
| ref nodes provide the means of linking to the node via a ref. A ref is |
| either a bucket-scoped builder name (e.g. "ci/linux-builder") or simple |
| builder name (e.g. "linux-builder") if the simple builder name is |
| unambiguous. |
| |
| Args: |
| kind: (str) An identifier for the kind of the node. Must be unique |
| within the chromium namespace. |
| |
| Returns: |
| A node type that can be used for creating and getting nodes of |
| the given kind. |
| |
| The type has the following properties: |
| * kind: The kind of nodes of the type. |
| * ref_kind: The kind of ref nodes of the type. |
| |
| The node types has the following methods: |
| * key(bucket_name, builder_name): Creates a key for the target kind with |
| the given bucket and builder. |
| * ref_key(ref): Creates a key with the ref kind for the given ref. |
| * add(bucket_name, builder_name, **kwargs): Adds a node with a key |
| created via `key(bucket_name, builder_name)`. `graph.add_node` will |
| be called with the key and `**kwargs`. Additionally, two ref nodes |
| will be created as parents of the target node, one with the |
| bucket-scoped builder name and one with the simple builder name. |
| Returns the target key. |
| * get(builder_name): Gets the node with the key given by |
| `key(bucket_name, builder_name)`. |
| * add_ref(key, ref): Add an edge from an arbitrary node identified by |
| `key` to one of this node type's ref nodes identified by `ref`. |
| * follow_ref(ref_node, context_node): Get the target node that is the |
| child of `ref_node`, which is a node of this node type's ref kind. |
| In the event that the ref node has multiple children (i.e. a simple |
| builder name that is ambiguous) `context_node` will be used in the |
| failure message to identify the source of the reference. |
| """ |
| ref_kind = kind + " ref" |
| |
| def key(bucket_name, builder_name): |
| return graph.key(_CHROMIUM_NS_KIND, "", kinds.BUCKET, bucket_name, kind, builder_name) |
| |
| def ref_key(ref): |
| chunks = ref.split(":", 1) |
| if len(chunks) != 1: |
| fail("reference to builder in external project '{}' is not allowed here" |
| .format(chunks[0])) |
| chunks = ref.split("/", 1) |
| if len(chunks) == 1: |
| return graph.key("@chromium", "", ref_kind, ref) |
| return graph.key("@chromium", "", kinds.BUCKET, chunks[0], ref_kind, chunks[1]) |
| |
| def add(bucket_name, builder_name, **kwargs): |
| k = key(bucket_name, builder_name) |
| graph.add_node(k, **kwargs) |
| for ref in (builder_name, "{}/{}".format(bucket_name, builder_name)): |
| rk = ref_key(ref) |
| graph.add_node(rk, idempotent = True) |
| graph.add_edge(rk, k) |
| return k |
| |
| def get(bucket_name, builder_name): |
| return graph.node(key(bucket_name, builder_name)) |
| |
| def add_ref(key, ref): |
| rk = ref_key(ref) |
| graph.add_edge(key, rk) |
| |
| def follow_ref(ref_node, context_node): |
| if ref_node.key.kind != ref_kind: |
| fail("{} is not {}".format(ref_node, ref_kind)) |
| |
| variants = graph.children(ref_node.key, kind) |
| if not variants: |
| fail("{} is unexpectedly unconnected".format(ref_node)) |
| |
| if len(variants) == 1: |
| return variants[0] |
| |
| fail( |
| "ambiguous reference '{}' in {}, possible variants:\n {}".format( |
| ref_node.key.id, |
| context_node, |
| "\n ".join([str(v) for v in variants]), |
| ), |
| trace = context_node.trace, |
| ) |
| |
| return struct( |
| kind = kind, |
| ref_kind = ref_kind, |
| add = add, |
| get = get, |
| add_ref = add_ref, |
| follow_ref = follow_ref, |
| ) |
| |
| # A node-type for access to lucicfg standard builder nodes. It doesn't provide |
| # full access because it doesn't allow for the creation of new nodes, but it can |
| # be used as a node type when creating link node types. |
| _BUILDER = struct( |
| kind = kinds.BUILDER, |
| ref_kind = kinds.BUILDER_REF, |
| add_ref = lambda key, ref: graph.add_edge(key, keys.builder_ref(ref)), |
| follow_ref = builder_ref.follow, |
| ) |
| |
| def _create_link_node_type(kind, parent_node_type, child_node_type): |
| """Create a link node type. |
| |
| A link node type allows for creating nodes that express a relationship |
| between two nodes. The nodes themselves only matter for providing a |
| connection between the related nodes and as such, no means are provided for |
| interacting directly with the nodes of this kind. The specific meaning of |
| the relationship is determined by the caller. Nodes of this type would be |
| necessary when there are potentially multiple relationships that could exist |
| between given nodes. |
| |
| Args: |
| kind: (str) An identifier for the kind of the node. Must be unique. |
| parent_node_type: (node type) The node type of the nodes that will be |
| the parent in the link relationship. |
| child_node_type: (node type) The node type of the nodes that will be the |
| child in the link relationship. |
| |
| Returns: |
| A node type that can be used for linking related nodes and retrieving |
| the parents or children of the relationship. |
| |
| The node types has the following methods: |
| * link(parent_key, child_key): Create a link between the nodes |
| identified by `parent_key` and `child_key`. The name is an arbitrary |
| name that will appear in error messages if there are issues with the |
| node. |
| * children(parent_key): Get the nodes that are linked by nodes of this |
| type as children of the node identified by `parent_key`. |
| * parents(child_key): Get the nodes that are linked by nodes of this |
| type as parents of the node identified by `child_key`. |
| |
| If the child_node_type was created by |
| `create_node_type_with_builder_ref`, the returned node type changes in |
| the following way: |
| * link instead has the signature link(name, parent_key, child_ref). A |
| link will be created between the node identified by `parent_key` and |
| the ref node for the reference `child_ref`. |
| * children will get the target node type by calling `follow_ref`, so |
| links to ambiguous refs will cause failures. |
| """ |
| |
| # The keys are not actually used after creation of the link, but may appear |
| # in error messages, so we create a unique key with the given name |
| def key(name): |
| return keys.unique(kind, name) |
| |
| parent_kind = parent_node_type.kind |
| child_kind = child_node_type.kind |
| |
| if hasattr(child_node_type, "ref_kind"): |
| def link(parent_key, child_ref): |
| k = key(child_ref) |
| graph.add_node(k) |
| graph.add_edge(parent_key, k) |
| child_node_type.add_ref(k, child_ref) |
| |
| def children(parent_key): |
| if parent_key.kind != parent_kind: |
| fail("kind of {} is not {}".format(parent_key, parent_kind)) |
| parent_node = graph.node(parent_key) |
| children = [] |
| |
| # The unique keys use the string representation of an incrementing |
| # number, which makes the ordering of them seem somewhat chaotic, so |
| # get the link nodes in definition order |
| for link_node in graph.children(parent_key, kind, graph.DEFINITION_ORDER): |
| for ref_node in graph.children(link_node.key, child_node_type.ref_kind): |
| children.append(child_node_type.follow_ref(ref_node, parent_node)) |
| return children |
| |
| def parents(child_key): |
| if child_key.kind != child_kind: |
| fail("kind of {} is not {}".format(child_key, child_kind)) |
| parents = [] |
| for ref_node in graph.parents(child_key, child_node_type.ref_kind): |
| # The unique keys use the string representation of an |
| # incrementing number, which makes the ordering of them seem |
| # somewhat chaotic, so get the link nodes in definition order |
| for link_node in graph.parents(ref_node.key, kind, graph.DEFINITION_ORDER): |
| parents.extend(graph.parents(link_node.key, parent_kind)) |
| return parents |
| |
| else: |
| def link(parent_key, child_key): |
| k = key(child_key.id) |
| graph.add_node(k) |
| graph.add_edge(parent_key, k) |
| graph.add_edge(k, child_key) |
| |
| def children(parent_key): |
| if parent_key.kind != parent_kind: |
| fail("kind of {} is not {}".format(parent_key, parent_kind)) |
| children = [] |
| |
| # The unique keys use the string representation of an incrementing |
| # number, which makes the ordering of them seem somewhat chaotic, so |
| # get the link nodes in definition order |
| for link in graph.children(parent_key, kind, graph.DEFINITION_ORDER): |
| children.extend(graph.children(link.key, child_kind)) |
| return children |
| |
| def parents(child_key): |
| if child_key.kind != child_kind: |
| fail("kind of {} is not {}".format(child_key, child_kind)) |
| parents = [] |
| |
| # The unique keys use the string representation of an incrementing |
| # number, which makes the ordering of them seem somewhat chaotic, so |
| # get the link nodes in definition order |
| for link in graph.parents(child_key, kind, graph.DEFINITION_ORDER): |
| parents.extend(graph.parents(link.key, parent_kind)) |
| return parents |
| |
| return struct( |
| link = link, |
| children = children, |
| parents = parents, |
| ) |
| |
| nodes = struct( |
| BUILDER = _BUILDER, |
| create_singleton_node_type = _create_singleton_node_type, |
| create_unscoped_node_type = _create_unscoped_node_type, |
| create_scoped_node_type = _create_scoped_node_type, |
| create_bucket_scoped_node_type = lambda kind: _create_scoped_node_type(kind, kinds.BUCKET), |
| create_node_type_with_builder_ref = _create_node_type_with_builder_ref, |
| create_link_node_type = _create_link_node_type, |
| ) |