| // Copyright 2025 The Chromium Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| syntax = "proto3"; |
| |
| package turboci.graph.orchestrator.v1; |
| |
| import "google/protobuf/timestamp.proto"; |
| import "turboci/graph/ids/v1/identifier.proto"; |
| import "turboci/graph/orchestrator/v1/check_kind.proto"; |
| import "turboci/graph/orchestrator/v1/check_state.proto"; |
| import "turboci/graph/orchestrator/v1/edge.proto"; |
| import "turboci/graph/orchestrator/v1/field_options.proto"; |
| import "turboci/graph/orchestrator/v1/stage.proto"; |
| import "turboci/graph/orchestrator/v1/stage_attempt_execution_policy.proto"; |
| import "turboci/graph/orchestrator/v1/stage_execution_policy.proto"; |
| import "turboci/graph/orchestrator/v1/transaction_details.proto"; |
| import "turboci/graph/orchestrator/v1/value_write.proto"; |
| |
| option go_package = "go.chromium.org/turboci/proto/go/graph/orchestrator/v1;orchestratorpb"; |
| option java_multiple_files = true; |
| |
| // Request message for TurboCIOrchestrator.WriteNodes. |
| // |
| // Allows atomically writing to multiple nodes (Checks, Stages) in a single |
| // transaction. |
| message WriteNodesRequest { |
| // An in-line representation of Dependencies.Group. |
| // |
| // The Orchestrator will deduplicate edges into Dependencies.edges in the |
| // Check or Stage. |
| // |
| // Otherwise this has the same meaning as Dependencies.Group. |
| message DependencyGroup { |
| // Singular edges in this group. |
| repeated Edge edges = 1; |
| |
| // Sub-groups in this group. |
| // |
| // May not contain empty groups. |
| repeated DependencyGroup groups = 2; |
| |
| // Number of edges and/or groups which need to be satisfied for this |
| // DependencyGroup to be satisfied. |
| // |
| // See `Dependencies.Group.threshold`. |
| optional int32 threshold = 3; |
| } |
| |
| // A single Stage.Attempt.Progress write message. |
| message StageAttemptProgress { |
| // Low-effort/human-readable progress information. |
| // |
| // Better than nothing, but not good for machines to parse. |
| optional string message = 1; |
| |
| // Machine-readable details for this progress item. |
| repeated ValueWrite details = 2; |
| |
| // An optional field for preventing duplicate progress messages. |
| // |
| // If you don't care about duplicate progress messages on retried RPCs, you |
| // can ignore this field. |
| // |
| // Otherwise, if set, this field must be unique per progress message, per |
| // Stage Attempt. |
| // |
| // Writing multiple progress messages with the same idempotency_key, but |
| // different `message` or `details` fields is an error. |
| // |
| // It is recommended to set this field like "<thread_id>/<unique>" where |
| // <thread_id> identifies a single actor within the Executor, in the |
| // context of the current stage attempt token, and <unique> is some unique |
| // value for that actor, for the Stage Attempt. |
| // |
| // Examples: |
| // * server/<purpose> - messages emitted by the server on behalf of a |
| // working bot with some specific purpose (e.g. "server/final"). |
| // * bot/<sequence> etc. - messages emitted by the bot from a single |
| // process on a machine somewhere. The process keeps <sequence> as a |
| // simple counter in-memory. |
| // * <uuid> - 36 bytes of stringy goodness. Hopefully your progress |
| // messages are much longer than this :) |
| // |
| // This solves the problem where: |
| // * you do a write with a progress message (e.g. "hello"), but fail to |
| // get the OK response (network, cosmic rays, etc.). |
| // * you then retry the RPC. |
| // |
| // If the WriteNodesRequest has a TransactionDetails, the retry will fail, |
| // and after getting the current Stage Attempt state, you would need to |
| // scan through the existing Progress messages to see if the message |
| // "hello" is present (assuming that "hello" is actually a unique progress |
| // message in the first place). |
| // |
| // If the WriteNodesRequest does NOT have a TransactionDetails, the new |
| // message will just be appended silently, and you will end up with two |
| // "hello" messages. |
| // |
| // Setting the idempotency_key to a per-Attempt-unique value will prevent |
| // this; this Progress message will be recorded at most once on the |
| // Attempt, and on the retry, the orchestrator will know that this message |
| // was already recorded and will be able to return OK without duplicating |
| // it. |
| optional string idempotency_key = 3; |
| } |
| |
| // The token which constrains this write to a specific Workplan (and possibly |
| // to a Stage Attempt in that Workplan). |
| // |
| // A suitable token is provided to the Executor of a Stage or to the creator |
| // of an empty Workplan. |
| // |
| // This is in addition to regular RPC authorization. |
| // |
| // If missing, this Orchestrator will check that the caller is permitted to |
| // act "externally" in the work plan, see http://go/turbo-ci-acls (Googlers |
| // only) for details. |
| optional string token = 1; |
| |
| // The reason that the caller is performing this write. |
| // |
| // Includes a low-effort English display message plus zero or more detail |
| // messages. |
| message Reason { |
| // A 'low-effort' reason for this write. |
| // |
| // This is required, and is intended to be shown as a quick English summary |
| // when displaying Check and Stage Edits in debugging tools and UIs. This |
| // should explain in simple terms why this write is happening. |
| // |
| // This should NOT be made to be parsable/machine-readable (use |
| // reason_details for this). |
| // |
| // This will be reflected in the CheckEdit or StageEdit logs for all |
| // Checks and/or Stages affected by this write, and thus readable under ALL |
| // realms implied by this set of Checks and/or Stages. |
| optional string message = 1; |
| |
| // Machine-readable reason(s) for this edit. |
| // |
| // These are optional, but are meant to be a way to communicate |
| // machine-readable context for this write. |
| // |
| // These must be unique by (type_url, realm), and they will be |
| // stably-sorted by type_url in the generated Edit messages (e.g. for pairs |
| // of (type_url, realm), [(a, R2), (b, R3), (a, R1)] would be sorted to |
| // [(a, R2), (a, R1), (b, R3)]). This order is compatible with the |
| // expectations of the `Find` and `Lookup` methods in |
| // go.chromium.org/luci/turboci/value. |
| // |
| // By convention, these should be ordered from most to least specific, so |
| // if a client has permission for multiple details of the same type, the |
| // first in this list should be a superset of the others. |
| repeated ValueWrite details = 2; |
| } |
| // Required reason for this write. |
| // |
| // At minimum, reason.message must be supplied. |
| optional Reason reason = 2; |
| |
| // Set if the caller wants to make this write transactional with a subset of |
| // the graph at a particular snapshot. |
| // |
| // If this is unset, then this is an 'oblivious write' and any valid writes |
| // will apply/overwrite the current database state, assuming they are |
| // semantically valid (i.e. it will still not be possible to change a Check's |
| // realm, or roll a Check's state backwards, etc.). |
| // |
| // There are common cases where this will be left unset, for example when a |
| // Stage Attempt updates its own progress based on its own internal state, or |
| // when a Stage Attempt records results for a Check off of some internal |
| // computation. It's allowed for a Stage Attempt to tie this progress update |
| // to the state of the graph, but it's not expected or required. |
| optional TransactionDetails txn = 3; |
| |
| // A description of modifications to make to a single Check. |
| // |
| // All fields annotated as `(turboci).creation_only = true` can be set only |
| // once in a sense that if a CheckWrite updates an existing check, values |
| // of all these fields (if they are set) must match existing check values. |
| message CheckWrite { |
| // The check to write to. |
| // |
| // The work plan in the ID can either be unset or be equal to the work plan |
| // the WriteNodesRequest's token is associated with. If there's no token, |
| // the work plan in the ID must be set. |
| // |
| // Required. |
| optional ids.v1.Check identifier = 1; |
| |
| // Realm to assign to this Check. |
| // |
| // If provided, must be in the absolute form "<project>:<name>" or be one |
| // of special values: |
| // * "$from_token" - see [ValueWrite] for how it works. |
| // * "$from_container" means "the same realm as the WorkPlan". |
| // |
| // Required when inserting a new check. |
| optional string realm = 2 [(turboci).creation_only = true]; |
| |
| // Kind to assign to this check. |
| // |
| // Required when inserting a new check. |
| optional CheckKind kind = 3 [(turboci).creation_only = true]; |
| |
| // The display name of this Check. |
| // |
| // Optional. If set, overrides the existing value. |
| optional string display_name = 9 [(turboci).check.editable = CHECK_STATE_PLANNING]; |
| |
| // The list of Options to write/overwrite. |
| // |
| // Must be unique on `ValueWrite.data.type_url`. Changing realms of existing |
| // options is not allowed. |
| repeated ValueWrite options = 4 [(turboci).check.editable = CHECK_STATE_PLANNING]; |
| |
| // Dependency predicate for this Check. |
| // |
| // If set, used to populate the dependencies.edges and |
| // dependencies.predicate fields in the target Check. |
| // |
| // To clear dependencies, set this to an empty DependencyGroup. |
| optional DependencyGroup dependencies = 5 [ |
| (turboci).id.allowed = IDENTIFIER_KIND_CHECK, |
| (turboci).check.editable = CHECK_STATE_PLANNING |
| ]; |
| |
| // Write data to a Result for this Check. |
| // |
| // The Result to write to is automatically keyed on: |
| // * The Stage Attempt (if `token` is provided) |
| // * The caller's identity (if `token` is absent) |
| // |
| // If the given keyed Result does not exist, it will be automatically |
| // created. Multiple calls to WriteNodes from this same Stage Attempt or |
| // service account will update the same Result, and the Result will be |
| // automatically finalized when this Stage Attempt ends (if a WriteNodes |
| // with `finalize_results` is not called before then). |
| // |
| // The data here will overwrite existing data of the same type in the |
| // selected Result for this Stage Attempt. |
| // |
| // Must be unique on `ValueWrite.data.type_url`. |
| repeated ValueWrite result_data = 6 [ |
| (turboci).check.editable = CHECK_STATE_PLANNED, |
| (turboci).check.editable = CHECK_STATE_WAITING |
| ]; |
| |
| // If set, finalize the Check.Result. |
| // |
| // No more data may be written to the Result from the caller after this is |
| // set. |
| optional bool finalize_results = 7 [(turboci).check.editable = CHECK_STATE_WAITING]; |
| |
| // The new state of this Check. |
| // |
| // If set, must be equal to, or greater than, the current state of the |
| // Check. |
| optional CheckState state = 8; |
| } |
| // Write to zero or more Checks. |
| repeated CheckWrite checks = 4; |
| |
| // A description of modifications to make to a single Stage. |
| // |
| // Note that the `state` of a Stage is managed entirely by the Orchestrator |
| // itself. If you are a Stage implementation and need to manage the state of |
| // your own Stage Attempt, see CurrentStageWrite. |
| // |
| // There are only two sorts of writes currently allowed: inserting a new stage |
| // or cancelling an existing stage. |
| // |
| // All fields annotated as `(turboci).creation_only = true` can be set only |
| // once in a sense that if a StageWrite overwrites an existing stage, values |
| // of all these fields (if they are set) must match existing stage values. |
| // |
| // All these fields are also optional when cancelling a stage. If they are |
| // set, they'll act as a precondition for cancellation, i.e. the stage will be |
| // cancelled only if its existing values match the provided ones. |
| message StageWrite { |
| // The stage to write to. |
| // |
| // The work plan in the ID can either be unset or be equal to the work plan |
| // the WriteNodesRequest's token is associated with. If there's no token, |
| // the work plan in the ID must be set. |
| // |
| // When inserting a new stage, if `identifier.is_worknode` is true, `args` |
| // must be type.googleapis.com/wireless.android.launchcontrol.WorkNodeStage |
| // (only vice versa: if a WorkNodeStage is given as `args, the ID must be |
| // `is_worknode`). Such writes can be used to insert legacy work nodes using |
| // Turbo CI APIs. IDs for such stages must be preallocated via |
| // AllocateWorkNodeIDs RPC. |
| // |
| // Required. |
| optional ids.v1.Stage identifier = 1; |
| |
| // The arguments of the Stage when inserting a new stage. |
| // |
| // Required when inserting a new stage. The realm must match `realm` (or |
| // be "$from_container", which will make it match the stage realm). |
| optional ValueWrite args = 2 [(turboci).creation_only = true]; |
| |
| // Realm to assign to this Stage when inserting it. |
| // |
| // If provided, must be in the absolute form "<project>:<name>" or be one |
| // of special values: |
| // * "$from_token" - see [ValueWrite] for how it works. |
| // * "$from_container" means "the same realm as the WorkPlan". |
| // * "$legacy_worknode" means to use legacy WorkPlan API ACLs. Can only be |
| // used if `identifier.is_worknode` is true. |
| // |
| // When inserting a new stage defaults to: |
| // * "$from_token" if `identifier.is_worknode` is false. |
| // * "$legacy_worknode" if `identifier.is_worknode` is true. |
| optional string realm = 3 [(turboci).creation_only = true]; |
| |
| // The display name of this Stage. |
| // |
| // Optional. |
| optional string display_name = 8 [(turboci).creation_only = true]; |
| |
| // Dependency predicate for this Stage. |
| // |
| // If set, used to populate the dependencies.edges and |
| // dependencies.predicate fields in the target Check. |
| // |
| // NOTE: Currently Stages in this group must only point to Stages created by |
| // the Stage performing this write. In theory, this should help prevent |
| // excessive coupling between different, unrelated, stage implementations. |
| // |
| // If arbitrary stage dependencies are allowed, it could cause errors when |
| // an upstream stage changes it's implementation and no longer produces the |
| // stages the downstream one expects. Instead, the upstream stage should |
| // create a Check which can remain stable across the implementation |
| // versions, and the downstream stages should depend on that. |
| // |
| // However, if the current writer is the one creating the stages, then there |
| // is no implementation risk. |
| optional DependencyGroup dependencies = 4 [ |
| (turboci).id.allowed = IDENTIFIER_KIND_STAGE, |
| (turboci).id.allowed = IDENTIFIER_KIND_CHECK, |
| (turboci).creation_only = true |
| ]; |
| |
| // The requested execution policy of the Stage. |
| // |
| // Optional. If omitted when creating a stage, the executor will decide what |
| // policy to enforce in its ValidateStage RPC. |
| optional StageExecutionPolicy requested_stage_execution_policy = 5 [(turboci).creation_only = true]; |
| |
| // The Check assignments of the Stage. |
| // |
| // Currently not implemented. |
| repeated Stage.Assignment assignments = 6 [(turboci).creation_only = true]; |
| |
| // If true, ensures that this Stage is marked for cancellation. |
| // |
| // See Stage.cancelled_by for how cancellation affects stages in different |
| // lifecycle states. |
| // |
| // Use the top-level `reason` field to provide the cancellation reason. |
| optional bool cancelled = 7; |
| } |
| // Write to zero or more Stages. |
| repeated StageWrite stages = 5; |
| |
| // Internal writes for the Stage Attempt indicated by the token. |
| // |
| // These aspects come from either the Executor which owns this Stage Attempt, |
| // or the running Stage Attempt process. |
| message CurrentAttemptWrite { |
| // Sets details in the Stage Attempt.details field. |
| // |
| // Details are stored in an append-only map "type URL -> ValueRef". |
| // Overwrites of existing details are ignored (see ProgressIgnoredDetail |
| // message). |
| repeated ValueWrite details = 1; |
| |
| // Progress messages to add to the current Stage Attempt. |
| // |
| // See StageAttemptProgress.idempotency_key to prevent duplicate appends on |
| // network flake. |
| repeated StageAttemptProgress progress = 2; |
| |
| // Indicate how the Orchestrator should transition to a given state. |
| message StateTransition { |
| // Throttled indicates that the Executor can run this Attempt, just not |
| // right now. |
| // |
| // Can only be used for attempts in PENDING or THROTTLED state. If the |
| // attempt is already running, use Incomplete instead (since some work was |
| // done, but it was incomplete) with `throttle_next_attempt_until` field |
| // set. |
| message Throttled { |
| // Specifies that the Stage Attempt should not be made PENDING again |
| // (and thus sent to the Executor via RunStage) until this time. |
| // |
| // Specifying a time in the past will cause the write to be a no-op |
| // (which will leave the Attempt in the PENDING state). |
| // |
| // Required. |
| optional google.protobuf.Timestamp until = 1; |
| } |
| |
| // Scheduled indicates that the Executor has accepted the Attempt, and |
| // will run it at a later time. |
| message Scheduled { |
| // The execution policy for this Stage Attempt. |
| // |
| // Optional. |
| optional StageAttemptExecutionPolicy attempt_execution_policy = 1; |
| } |
| |
| // Running indicates that the Executor is actively running this Attempt. |
| message Running { |
| // The execution policy for this Stage Attempt. |
| // |
| // This should only be used if it wasn't already set in SCHEDULED. |
| // If it's set for both SCHEDULED and RUNNING, it must be the same |
| // value. |
| // |
| // Optional. |
| optional StageAttemptExecutionPolicy attempt_execution_policy = 1; |
| |
| // Sets the Stage.Attempt.process_uid field. |
| // |
| // If the field is already populated in the Stage Attempt, this value |
| // must match - otherwise the write is rejected with an error detail |
| // of StageAttemptClaimedFailure. |
| // |
| // Refer to Stage.Attempt.process_uid. |
| // |
| // Required. |
| optional string process_uid = 2; |
| } |
| |
| // TearingDown indicates that the Executor has either finished RUNNING |
| // the Attempt, or the Stage was cancelled and the Executor is doing |
| // its final cleanup for a previously-RUNNING Attempt. |
| message TearingDown { |
| // Nothing here for now. |
| } |
| |
| // Complete indicates that the Executor has finished all work on this |
| // Attempt, and the Stage can now move into its next state. |
| // |
| // NOTE: See STAGE_ATTEMPT_COMPLETE. Completing a Stage does not imply |
| // workflow-level success like "the code compiled successfully" or "the |
| // tests all passed". It simply means that e.g. the requested build ran |
| // to completion, or the test harness executed all the tests. |
| // Workflow-level records like compilation success/failure or test |
| // execution success/failure should be recorded in the Checks for those |
| // things as specific Result data. |
| message Complete { |
| // Nothing here for now. |
| } |
| |
| // Incomplete indicates an Executor-level failure which prevented the |
| // stage from running to completion. |
| // |
| // By convention, the reason for this should be reflected in the final |
| // Progress message (which, unlike the write reasons, will not be |
| // garbage collected after some amount of time). |
| // |
| // NOTE: See STAGE_ATTEMPT_INCOMPLETE. Similar to COMPLETE, INCOMPLETE |
| // is used to indicate failure of the stage to complete it's desired |
| // task (e.g. compiler segfaulted, test harness lost contact with a DUT |
| // during test execution (and such a failure may not be due to the test |
| // itself), etc.) |
| message Incomplete { |
| // Controls if this Stage should make another Attempt. |
| // |
| // If true, then no new Attempt is made, even if policy would permit |
| // one. |
| // |
| // Optional. |
| optional bool block_new_attempts = 1; |
| |
| // The earliest time the next attempt should start running. |
| // |
| // If set, a new Attempt (if allowed by the policy at all) will be |
| // created in THROTTLED state and will start running no sooner |
| // than the given timestamp. |
| // |
| // If unset, a new Attempt will be created in AWAITING_RETRY state and |
| // it will start running after some delay picked by the Orchestrator |
| // itself (using randomized exponential backoff based on the number of |
| // previous incomplete attempts). |
| optional google.protobuf.Timestamp throttle_next_attempt_until = 2; |
| } |
| |
| // Required - the state that this StateTransition wants to move |
| // to. |
| oneof desired_state { |
| // Move to the THROTTLED state. |
| // |
| // Current state must be PENDING. |
| Throttled throttled = 1; |
| |
| // Move to the SCHEDULED state. |
| // |
| // Current state must be PENDING. |
| Scheduled scheduled = 2; |
| |
| // Move to the RUNNING state. |
| // |
| // Current state must be PENDING or SCHEDULED. |
| Running running = 3; |
| |
| // Move to the TEARING_DOWN state. |
| // |
| // Current state must be RUNNING or CANCELLING. |
| TearingDown tearing_down = 4; |
| |
| // Move to the COMPLETE state. |
| // |
| // Current state must be PENDING, SCHEDULED, RUNNING or TEARING_DOWN. |
| Complete complete = 5; |
| |
| // Move to the INCOMPLETE state. |
| // |
| // Current state must be PENDING, SCHEDULED, RUNNING or TEARING_DOWN. |
| Incomplete incomplete = 6; |
| } |
| } |
| // A state transition to make in this write. |
| optional StateTransition state_transition = 3; |
| } |
| // State for the current Stage Attempt as indicated by `token`. |
| // |
| // It is invalid to set this without also setting `token`. |
| // |
| // A WriteNodes call with `current_attempt` set (even when it is empty) acts |
| // as a heartbeat for the current Stage Attempt indicated by the token. If |
| // you need to implement the 'simplest heartbeat', you can make a WriteNodes |
| // call with just the token and an empty `current_attempt {}`. |
| // |
| // WriteNodes calls that don't have `current_attempt` set do not affect the |
| // heartbeat timer. This makes such calls less likely to hit transaction |
| // collisions (since they don't try to concurrently modify the heartbeat |
| // timer). This may be useful if you want to make concurrent WriteNodes calls |
| // from the same Stage Attempt, or even closely-sequential WriteNodes calls |
| // (to non-overlapping portions of the graph) from the same Stage Attempt. |
| // |
| // All else being equal, it's better to do fewer, larger, WriteNodes calls |
| // than many small WriteNodes calls. |
| optional CurrentAttemptWrite current_attempt = 6; |
| |
| // Internal writes for the Stage indicated by the token. |
| message CurrentStageWrite { |
| // Continuation Group predicate for this Stage. |
| // |
| // If set, used to populate the continuation_group.edges and |
| // continuation_group.predicate fields in this Stage. |
| // |
| // Edges here must only point to Stages which were created_by this Stage. |
| // |
| // If continuation_group is already set, this will overwrite the existing |
| // continuation_group. |
| optional DependencyGroup continuation_group = 1 [(turboci).id.allowed = IDENTIFIER_KIND_STAGE]; |
| } |
| // State for the current Stage as indicated by `token`. |
| // |
| // It is invalid to set this without also setting `token`. |
| // |
| // This does not impact the heartbeat timer for the current attempt. |
| optional CurrentStageWrite current_stage = 7; |
| } |
