cv/internal/run/storage.proto

// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package cv.internal.run;

option go_package = "go.chromium.org/luci/cv/internal/run;run";

import "google/protobuf/timestamp.proto";

import "go.chromium.org/luci/cv/api/config/v2/config.proto";
import "go.chromium.org/luci/cv/internal/gerrit/storage.proto";
import "go.chromium.org/luci/cv/internal/run/eventpb/submission.proto";
import "go.chromium.org/luci/cv/internal/tryjob/storage.proto";
import "go.chromium.org/luci/cv/internal/tryjob/task.proto";

// Status describes the status of a CV Run.
enum Status {
  // Unspecified status.
  STATUS_UNSPECIFIED = 0;
  // Run is pending to start.
  //
  // It is either because Run Manager hasn't processed the StartEvent yet or
  // the RunOwner has exhausted all the quota and waiting for new quota to
  // be available.
  PENDING = 1;
  // Run is running.
  RUNNING = 2;
  // Run is waiting for submission.
  //
  // Run is in this status if one of the following scenario is true:
  //   1. Tree is closed at the time Run attempts to submit.
  //   2. There is another Run in the same LUCI Project that is currently
  //      submitting.
  //   3. The submission is rate limited according to the submit option in
  //      Project Config.
  //
  // This status is cancellable.
  WAITING_FOR_SUBMISSION = 4;
  // Run is submitting.
  //
  // A Run can't be cancelled while submitting. A Run may transition from
  // this status to either `WAITING_FOR_SUBMISSION` status or a non-cancelled
  // terminal status.
  SUBMITTING = 5;

  // End of non-terminal status; MUST have value less than `ENDED_MASK`.

  /////////////////////////////////////////////////////////////////////////////
  // Terminal Status

  // ENDED_MASK can be used as a bitmask to check if a Run has ended.
  // This MUST NOT be used as the status of a Run.
  ENDED_MASK = 64;
  // Run ends successfully.
  SUCCEEDED = 65;
  // Run ends unsuccessfully.
  FAILED = 66;
  // Run is cancelled.
  CANCELLED = 67;
}

// Trigger describes who/how CV was triggered on a specific CL.
message Trigger {
  reserved 5; // additional_label
  google.protobuf.Timestamp time = 1;
  // Mode is string value of run.Mode.
  string mode = 2;
  // ModeDefinition is the definition of the mode in case `mode` is not standard
  // modes supported by LUCI CV (e.g. DRY_RUN, FULL_RUN, NEW_PATCHSET_RUN) but
  // is provided through project configuration.
  //
  // This field is empty if `mode` is standard mode in LUCI CV.
  cv.config.Mode mode_definition = 6;
  // Triggering user email. Always known for Runs created since July 2021.
  //
  // Gerrit doesn't guarantee that every user has set their preferred email,
  // but LUCI ACLs are based entirely on user emails. Thus, Project Manager will
  // refuse starting Runs for users without the email.
  string email = 3;
  // Gerrit account ID. Always known.
  int64 gerrit_account_id = 4;
}

// Triggers describes the triggers associated with a CL.
message Triggers {
  // CQVoteTrigger is the trigger that corresponds to a vote on the
  // `Commit-Queue` Gerrit label.
  Trigger cq_vote_trigger = 1;

  // NewPatchsetRunTrigger is the trigger that is created automatically by
  // uploading a new patchset.
  //
  // This is only populated if the CL matches a config group that allows this.
  Trigger new_patchset_run_trigger = 2;
}

// Submission describes the state of Run submission.
message Submission {
  // The deadline of this submission.
  //
  // If the deadline is not set or has already expired, a RunManager task
  // can claim the exclusive privilege by setting the deadline to a future
  // timestamp (generally, end of task deadline).
  google.protobuf.Timestamp deadline = 1;
  // ID of the task that executes this submission.
  string task_id = 2;
  // IDs of all CLs that should be submitted in this submission.
  //
  // Must be ordered in submission order.
  repeated int64 cls = 3;
  // IDs of all CLs that have been submitted successfully already.
  repeated int64 submitted_cls = 4;
  // IDs of all CLs that fails to submit if any.
  //
  // CLs that are neither in this list nor in the `submitted_cls` should be
  // treated as if CV has never attempted to submit them.
  //
  // This could be empty even when the entire submission fails, which would be
  // typically caused by faulty infrastructure (e.g. Task Queue not executing
  // a Run Manager task before the whole submission timeout is reached).
  repeated int64 failed_cls = 5;
  // If True, Tree is currently in open state.
  bool tree_open = 10;
  // The timestamp when the last attempt to fetch the Tree status occurred.
  google.protobuf.Timestamp last_tree_check_time = 11;
  // The timestamp when an attempt to fetch the Tree status first resulted in
  // an error.
  google.protobuf.Timestamp tree_error_since = 12;

}

// Options are Run-specific additions on top of LUCI project config.
message Options {
  // If true, submitting the Run isn't blocked on open tree.
  //
  // If false (default), respects project configuration.
  bool skip_tree_checks = 1;

  // If true, `builders.equivalent_to{...}` sections are ignored when triggering
  // tryjobs.
  //
  // If false (default), respects project configuration.
  bool skip_equivalent_builders = 2;

  // If true, no longer useful tryjobs won't be cancelled.
  //
  // If false (default), respects project configuration.
  bool avoid_cancelling_tryjobs = 3;

  // If true, no tryjobs will be triggered except "presubmit" regardless of
  // project configuration.
  //
  // "presubmit" builders are legacy which are currently configured with
  // "disable_reuse: true" in project config. To skip triggering them,
  // skip_presubmit must be set to true.
  // TODO(https://crbug.com/950074): ignore.
  //
  // If false (default), respects project configuration.
  bool skip_tryjobs = 4;
  // Deprecated per https://crbug.com/950074.
  // See skip_tryjobs doc.
  bool skip_presubmit = 5;

  // Contains the directives to include specific builder in the Run.
  //
  // Its elements are strings of the form:
  // project/bucket:builder1,builder2;project2/bucket:builder3
  //
  // Note that there may be duplication in the directives, it's up to the
  // consumer of this information to handle it.
  //
  // Mutually exclusive with `overridden_tryjobs` option.
  // This option is ignored if `skip_tryjobs` is true.
  repeated string included_tryjobs = 6;

  // Overrides all the Tryjobs triggered for this Run regardless of Project
  // configuration.
  //
  // Its elements are strings of the form:
  // project/bucket:builder1,builder2;project2/bucket:builder3
  //
  // Note that there may be duplication in the directives, it's up to the
  // consumer of this information to handle it.
  //
  // Mutually exclusive with `included_tryjobs` option.
  // This option is ignored if `skip_tryjobs` is true.
  repeated string overridden_tryjobs = 8;

  // Contains the custom Tryjob tags that should be added when launching
  // Tryjobs for this Run.
  //
  // Each element SHOULD be in the format of "$tag_key:$tag_value" and the
  // character set for tag key is [a-z0-9_\-].
  repeated string custom_tryjob_tags = 7;
}

// LogEntries contains 1+ LogEntry ordered from logically oldest to newest.
message LogEntries {
  repeated LogEntry entries = 1;
}

// LogEntry records what changed in a Run.
message LogEntry {
  // Next tag: 7.

  // Time is when something was changed.
  google.protobuf.Timestamp time = 1;
  oneof kind {
    // Run was created.
    Created created = 2;
    // Run was started.
    Started started = 6;
    // Run updated to a new project config version.
    ConfigChanged config_changed = 3;
    // Tryjobs requirement was (re-)computed.
    TryjobsRequirementUpdated tryjobs_requirement_updated = 4;
    // Applicable tryjobs were updated.
    TryjobsUpdated tryjobs_updated = 5;
    // TODO(crbug/1232158): add & implement events related to Submission and
    // ending of the Run.

    // Intended for informational logs (E.g. temporary/during migration)
    Info info = 7;
    // The tree is configured and was checked.
    TreeChecked tree_checked = 8;
    // The run has been added to the submit queue's waitlist.
    Waitlisted waitlisted = 9;
    // The run is current on the queue.
    AcquiredSubmitQueue acquired_submit_queue = 10;
    ReleasedSubmitQueue released_submit_queue = 11;
    // CL(s) submitted successfully.
    CLSubmitted cl_submitted = 12;
    // Submission failed.
    SubmissionFailure submission_failure = 13;
    RunEnded run_ended = 14;
  }

  message Created {
    string config_group_id = 1;
  }
  message Started {
  }
  message ConfigChanged {
    string config_group_id = 1;
  }
  message TryjobsRequirementUpdated {
    // TODO(crbug/1227363): define a Tryjobs.Requirement diff.
  }
  message Info {
    // If you have the need to add fields here, consider instead adding a new
    // dedicated kind
    string label = 1;
    string message = 2;
  }
  message TryjobsUpdated {
    // Which tryjobs had a meaningful change (e.g. change of status).
    repeated Tryjob tryjobs = 2;
  }
  message TreeChecked{
    bool open = 1;
  }
  message Waitlisted{
  }
  message AcquiredSubmitQueue{
  }
  message ReleasedSubmitQueue{
  }
  message CLSubmitted{
    // The CLs that were submitted in this event.
    repeated int64 newly_submitted_cls = 1;
    // The number of CLs submitted for this run, so far.
    int64 total_submitted = 2;
  }
  message SubmissionFailure{
    cv.internal.run.eventpb.SubmissionCompleted event = 1;
  }
  message RunEnded{
  }
}

// Tryjobs is the state of Run's tryjobs.
message Tryjobs {
  // Requirement is what has to happen to verify a given Run.
  cv.internal.tryjob.Requirement requirement = 1;
  // StagingRequirement will be promoted to requirement.
  //
  // It is typically set when the existing requirement is executing and waiting
  // to be cancelled. For example:
  //  T0: Run starts and computes the requirement. A long op task is executing
  //      the requirement.
  //  T1: A new config is ingested and results in a new requirement. RM
  //      requests a cancellation for the long op task and set the new
  //      requirement to this field.
  //  T2: The long op task is successfully cancelled. RM promotes the
  //      staging requirement to requirement and enqueue a new long op task to
  //      execute the new requirement.
  cv.internal.tryjob.Requirement staging_requirement = 4;
  // RequirementVersion increments by 1 every time this requirement changes.
  //
  // Starts with 1.
  int32 requirement_version = 6;
  // The timestamp when the requirement is last computed.
  //
  // Every requirement computation will update this field even if the result
  // requirement is the same as the existing one.
  google.protobuf.Timestamp requirement_computed_at = 7;
  // Tryjobs tracks tryjobs of a Run.
  //
  // It may contain Tryjobs which are no longer required.
  // It does contain all Tryjobs which weren't reused even if no longer
  // required.
  //
  // TODO(crbug/1227363): replace this field in favor of `state`
  repeated Tryjob tryjobs = 2 [deprecated = true];
  // The timestamp of the CQDaemon report last incorporated into `tryjobs`.
  //
  // TODO(crbug/1227523): delete this field.
  google.protobuf.Timestamp cqd_update_time = 3;
  // State is the latest state reported by Tryjob Executor task.
  //
  // Once the Run is ended, this state is finalized.
  cv.internal.tryjob.ExecutionState state = 5;
}

// Tryjob represents a Run's view of a tryjob.
message Tryjob {
  cv.internal.tryjob.Definition definition = 1;

  // ID is a CV internal Tryjob ID, corresponding to a Datastore entity.
  //
  // During migration from CQDaemon, the ID may be not set but then ExternalID
  // is set.
  // TODO(crbug/1227523): make this field required.
  int64 id = 2;

  // EVersion of the Tryjob entity last observed by this Run.
  int64 eversion = 3;

  // ExternalID is the external job ID.
  //
  // It's kept here for ease of URL generation and to ease migration from
  // CQDaemon.
  // TODO(crbug/1227523): update comment above after CQDaemon migration.
  string external_id = 4;

  // Status of the Tryjob.
  cv.internal.tryjob.Status status = 5;

  // Reused is true, if this tryjob wasn't triggered by CV for this Run.
  //
  // In other words, either:
  //  * tryjob was triggered by CV for a previous Run
  //  * tryjob was triggered by non-CV.
  bool reused = 6;

  // If true, indicates this tryjob must pass in order for the Run to be
  // considered successful.
  //
  // It is typically true when the tryjob is NOT experimental or triggered
  // because of `Cq-Include-TryBot` git footer.
  bool critical = 9;

  // Result of the tryjob.
  cv.internal.tryjob.Result result = 7;

  // If true, this Tryjob was computed based on CQDaemon's input.
  //
  // TODO(crbug/1227523): delete after CQDaemon migration.
  bool cqd_derived = 8;
}

// OngoingLongOps tracks ongoing long operations.
message OngoingLongOps {
  message Op {
    reserved 4; // cancel_triggers
    // Deadline best-effort limits the lifetime of this work.
    //
    // Run Manager expects to receive a LongOpCompleted event by this deadline.
    google.protobuf.Timestamp deadline = 1;
    // If true, signals to the Long Op handler that it should stop as soon
    // possible.
    bool cancel_requested = 2;

    message ResetTriggers {
      // Request is to request resetting the trigger of a CL.
      message Request {
        // internal CLID.
        int64 clid = 1;
        // Message explains why trigger is reset. Will be posted to the CL.
        string message = 2;
        // Whom to notify.
        repeated gerrit.Whom notify = 3;
        // Whom to add to the attention set.
        repeated gerrit.Whom add_to_attention = 4;
        // Reason explains the reason of attention set change.
        string add_to_attention_reason = 5;
      }
      repeated Request requests = 1;
      // The status Run will transition to if all triggers are successfully
      // cancelled.
      //
      // Must be one of the terminal statues.
      Status run_status_if_succeeded = 2;
    }

    // ExecutePostActionPayload is the payload of a long-op task that executes
    // a given post action.
    message ExecutePostActionPayload {
      reserved 1;  // action

      // Name of the post action.
      //
      // For config action, it should be the same as the name in the config
      // action.
      string name = 2;

      message CreditRunQuota {}

      oneof kind {
        // ConfigAction is the post action defined in the config group.
        cv.config.ConfigGroup.PostAction config_action = 3;
        // CreditRunQuota credits the quota back to the run creator and ask
        // next run that is blocked on the quota to start.
        CreditRunQuota credit_run_quota = 4;
      }
    }

    // PostGerritMessage is the payload for posting a bot message within
    // a run's gerrit CLs.
    message PostGerritMessage {
      // Message to post to gerrit. The current implementation treats the
      // message as a unique key to achieve idempotence. If multiple payloads
      // with the same message is triggered for the same run, only one would
      // be posted.
      string message = 1;
    }

    oneof work {
      // If true, posts a start message.
      bool post_start_message = 3;
      ResetTriggers reset_triggers = 6;
      cv.internal.tryjob.ExecuteTryjobsPayload execute_tryjobs = 5;
      // The PostActions from different runs can race each other in case
      // the actions from an earlier Run transiently failed and are being
      // retried. However, the resolution depends on the type of the post
      // action.
      //
      // If the action is to export the Run records to BQ, each of the Run
      // records should be exported, and CV should still attempt to execute post
      // actions for all the Runs ever triggered.
      //
      // In contrast, if the action is to vote a label, the post action handler
      // should check if the originated patchset is still the latest at the time
      // of the long-op processing time, and skip the action, if the patchset
      // is no longer the latest.
      ExecutePostActionPayload execute_post_action = 7;

      // Post a custom message to all the CLs linked to a run.
      PostGerritMessage post_gerrit_message = 8;
    }
  }
  // Ops map operation ID to details.
  map<string, Op> ops = 1;
}