components/optimization_guide/proto/features/actions_data.proto

// Copyright 2025 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

edition = "2023";

package optimization_guide.proto;

import "components/optimization_guide/proto/features/common_quality_data.proto";

option java_outer_classname = "ActionsProto";

option java_package = "org.chromium.components.optimization_guide.features.proto";

option optimize_for = LITE_RUNTIME;

// DO NOT EDIT THIS FILE DIRECTLY!
//
// This file is generated in g3 and then synced to Chrome. Instead, please refer to
// http://go/chrome-mqls-onboarding (Google-internal link), and then changes will
// be synced with Chrome automatically.

// This API should eventually converge with go/unified-browser-control-api.
// Please check with erikchen@ before making modifications.

// The target of an action.
// Next ID: 4
message ActionTarget {
  // Id of the content node within the frame. Same as
  // ContentNode.common_ancestor_dom_node_id.
  int32 content_node_id = 1 [features = { field_presence: EXPLICIT }];

  Coordinate coordinate = 2 [features = { field_presence: EXPLICIT }];

  // Which document in a frame the action should be taken in.
  DocumentIdentifier document_identifier = 3 [features = { field_presence: EXPLICIT }];
}

// Mouse click action.
// Next ID: 5
message ClickAction {
  ActionTarget target = 1 [features = { field_presence: EXPLICIT }];

  ClickType click_type = 2 [features = { field_presence: EXPLICIT }];

  ClickCount click_count = 3 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 4 [features = { field_presence: EXPLICIT }];

  enum ClickType {
    // Unknown click type. This should not be used.
    UNKNOWN_CLICK_TYPE = 0;

    // Left mouse click.
    LEFT = 1;

    // Right mouse click.
    RIGHT = 2;
  }

  // Represents the number of times the target was clicked, for now only
  // supports single and double clicks.
  enum ClickCount {
    // Unknown click count. This should not be used.
    UNKNOWN_CLICK_COUNT = 0;

    // Single click.
    SINGLE = 1;

    // Double click.
    DOUBLE = 2;
  }
}

// Type text action.
// Next ID: 6
message TypeAction {
  ActionTarget target = 1 [features = { field_presence: EXPLICIT }];

  // The text to type.
  string text = 2 [features = { field_presence: EXPLICIT }];

  // After Typing the text, press enter.
  bool follow_by_enter = 3 [features = { field_presence: EXPLICIT }];

  TypeMode mode = 4 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 5 [features = { field_presence: EXPLICIT }];

  // How the text should be inserted into the target.
  enum TypeMode {
    UNKNOWN_TYPE_MODE = 0;

    DELETE_EXISTING = 1;

    PREPEND = 2;

    APPEND = 3;
  }
}

// Scroll action.
// Next ID: 5
message ScrollAction {
  ActionTarget target = 1 [features = { field_presence: EXPLICIT }];

  ScrollDirection direction = 2 [features = { field_presence: EXPLICIT }];

  // The distance to scroll in pixels.
  float distance = 3 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 4 [features = { field_presence: EXPLICIT }];

  enum ScrollDirection {
    // Unknown scroll direction. This should not be used.
    UNKNOWN_SCROLL_DIRECTION = 0;

    // Scroll left.
    LEFT = 1;

    // Scroll right.
    RIGHT = 2;

    // Scroll up.
    UP = 3;

    // Scroll down.
    DOWN = 4;
  }
}

// Move mouse action.
// This is considered a low-level building block for more complex actions, e.g.
// scroll_at can be mouse_move + scroll action.
// This action also covers hovering, equivalent to "hovering at" when used in
// isolation.
// Next ID: 3
message MoveMouseAction {
  ActionTarget target = 1 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 2 [features = { field_presence: EXPLICIT }];
}

// Drag and release mouse action.
// Next ID: 4
message DragAndReleaseAction {
  ActionTarget from_target = 1 [features = { field_presence: EXPLICIT }];

  ActionTarget to_target = 2 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 3 [features = { field_presence: EXPLICIT }];
}

// Select dropdown action.
// Next ID: 4
message SelectAction {
  ActionTarget target = 1 [features = { field_presence: EXPLICIT }];

  // The value of the option to select.
  string value = 2 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 3 [features = { field_presence: EXPLICIT }];
}

// Navigate the page to a URL action.
// Next ID: 3
message NavigateAction {
  string url = 1 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 2 [features = { field_presence: EXPLICIT }];
}

// Wait action waits the specified milliseconds.
// Next ID: 2
message WaitAction {
  int32 wait_time_ms = 1 [features = { field_presence: EXPLICIT }];
}

// Navigate the page Back action.
// Next ID: 2
message HistoryBackAction {
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// Navigate the page Forward action.
// Next ID: 2
message HistoryForwardAction {
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// Creates a new tab
// Next ID: 3
message CreateTabAction {
  // The id of the window to create the tab in. Passing in invalid window id
  // will cause the action to fail.
  int32 window_id = 1 [features = { field_presence: EXPLICIT }];

  // When set to true, the new tab will be visible within the window, although
  // the window itself may not be visible.
  bool foreground = 2 [features = { field_presence: EXPLICIT }];
}

// Closes an existing tab.
// Next ID: 2
message CloseTabAction {
  // The id of the tab to close. Passing in invalid tab id will cause the action
  // to fail.
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// Activates an existing tab. This does not affect window visibility or
// activation.
// Next ID: 2
message ActivateTabAction {
  // The id of the tab to activate. Passing in invalid tab id will cause the
  // action to fail.
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// Creates a new window.
// Next ID: 1
message CreateWindowAction {
}

// Closes an existing window.
// Next ID: 2
message CloseWindowAction {
  // The id of the window to close. Passing in invalid window id will cause the
  // action to fail.
  int32 window_id = 1 [features = { field_presence: EXPLICIT }];
}

// Activates an existing window. This does not move the window, but it brings it
// to the front of the z-order, and gives it OS-activation, which means the
// window will receive keyboard events.
// Next ID: 2
message ActivateWindowAction {
  // The id of the window to activate. Passing in invalid window id will cause
  // the action to fail.
  int32 window_id = 1 [features = { field_presence: EXPLICIT }];
}

// Yields control to the user.
// Next ID: 1
message YieldToUserAction {
}

// Attempts to log in to the website.
// Next ID: 2
message AttemptLoginAction {
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// Scrolls the DOM element into viewport.
// Next ID: 5
message ScrollToAction {
  ActionTarget target = 3 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 4 [features = { field_presence: EXPLICIT }];

  reserved 1, 2;
}

// Executes a script tool.
// Next ID: 5
message ScriptToolAction {
  // The Document which provides the associated Tool.
  DocumentIdentifier document_identifier = 1 [features = { field_presence: EXPLICIT }];

  int32 tab_id = 2 [features = { field_presence: EXPLICIT }];

  // The name of the tool to execute.
  string tool_name = 3 [features = { field_presence: EXPLICIT }];

  // A serialized JSON string containing the arguments to pass to the tool.
  string input_arguments = 4 [features = { field_presence: EXPLICIT }];
}

// All of the actions that can be taken.
// Only one of these is allowed per action.
// Next ID: 21
message Action {
  oneof action {
    ClickAction click = 1;

    TypeAction type = 2;

    ScrollAction scroll = 3;

    MoveMouseAction move_mouse = 4;

    DragAndReleaseAction drag_and_release = 5;

    SelectAction select = 6;

    NavigateAction navigate = 7;

    HistoryBackAction back = 8;

    HistoryForwardAction forward = 9;

    WaitAction wait = 10;

    CreateTabAction create_tab = 11;

    CloseTabAction close_tab = 12;

    ActivateTabAction activate_tab = 13;

    CreateWindowAction create_window = 14;

    CloseWindowAction close_window = 15;

    ActivateWindowAction activate_window = 16;

    YieldToUserAction yield_to_user = 17;

    AttemptLoginAction attempt_login = 18;

    ScrollToAction scroll_to = 19;

    ScriptToolAction script_tool = 20;
  }
}

// Deprecated. Use Actions instead.
// A single set of actions to be taken by Chrome before responding.
// Next ID: 4
message BrowserAction {
  // The actions to be taken by Chrome.
  repeated Action actions = 1;

  // The task id of the action.
  int32 task_id = 2 [features = { field_presence: EXPLICIT }];

  // The id of the tab the action will be taken in.
  // TODO(crbug.com/421441072): Remove this field once all models are using tab_id in
  // actions.
  int32 tab_id = 3 [features = { field_presence: EXPLICIT }];
}

// Deprecated. Use ActionsResult instead.
// The state of the browser after an action is taken.
// Currently only used for extension APIs, but should directly reflect the
// production state of the browser returned in glic.mojom.
// Next ID: 7
message BrowserActionResult {
  // The page content at the time after the action.
  AnnotatedPageContent annotated_page_content = 1 [features = { field_presence: EXPLICIT }];

  // The screenshot of the page at the time after the action.
  bytes screenshot = 2 [features = { field_presence: EXPLICIT }];

  // The mime type of the |screenshot|.
  string screenshot_mime_type = 3 [features = { field_presence: EXPLICIT }];

  // The result of the action.
  int32 action_result = 4 [features = { field_presence: EXPLICIT }];

  // The task id of the action.
  int32 task_id = 5 [features = { field_presence: EXPLICIT }];

  // The id of the tab the action was taken in.
  int32 tab_id = 6 [features = { field_presence: EXPLICIT }];
}

// A single set of actions to be taken by Chrome. The response is ActionsResult.
// Next ID: 3
message Actions {
  // The actions to be taken by Chrome.
  repeated Action actions = 1;

  // The task id of the action.
  int32 task_id = 2 [features = { field_presence: EXPLICIT }];
}

// Information about a chrome window.
// Next ID: 5
message WindowObservation {
  // Unique id
  int32 id = 1 [features = { field_presence: EXPLICIT }];

  // id of the activated tab. There may be multiple tabs visible at the same
  // time.
  int32 activated_tab_id = 2 [features = { field_presence: EXPLICIT }];

  // The active window is defined as the most recently used chrome browser
  // window.
  bool active = 3 [features = { field_presence: EXPLICIT }];

  // All tabs. The order is not meaningful.
  repeated int32 tab_ids = 4;
}

// Information about a chrome tab.
// Next ID: 5
message TabObservation {
  // Unique id
  int32 id = 1 [features = { field_presence: EXPLICIT }];

  // The page content at the observation time.
  AnnotatedPageContent annotated_page_content = 2 [features = { field_presence: EXPLICIT }];

  // The screenshot of the page at the time after the action.
  bytes screenshot = 3 [features = { field_presence: EXPLICIT }];

  // The mime type of the |screenshot|.
  string screenshot_mime_type = 4 [features = { field_presence: EXPLICIT }];
}

// All relevant state in chrome after the actions are taken or an error has
// occurred.
// Next ID: 7
message ActionsResult {
  // The result of the actions. See
  // https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/actor.mojom
  // for definition.
  int32 action_result = 1 [features = { field_presence: EXPLICIT }];

  // If a specific action failed, the index of the failed action. -1 for no
  // failure.
  int32 index_of_failed_action = 2 [features = { field_presence: EXPLICIT }];

  // Observations after all actions are finished or an error has occurred. The
  // client will determine the relevant windows and tabs.
  repeated WindowObservation windows = 3;

  // Observations after all actions are finished or an error has occurred. The
  // client will determine the relevant windows and tabs.
  repeated TabObservation tabs = 4;

  // The results of executing script tools.
  repeated ScriptToolResult script_tool_results = 5;

  LatencyInformation latency_information = 6 [features = { field_presence: EXPLICIT }];

  // Chrome reported information about the latency of steps while performing the
  // actions.
  // Next ID: 4
  message LatencyInformation {
    // All steps that Chrome reported latency for. These steps can overlap due
    // to parallel execution.
    repeated LatencyStep latency_steps = 1;

    // Latency of a single step.
    // Next ID: 7
    message LatencyStep {
      oneof step {
        ActionLatency action = 3;

        PageStabilizationLatency page_stabilization = 4;

        AnnotatedPageContentLatency annotated_page_content = 5;

        ScreenshotLatency screenshot = 6;
      }

      // The latency since the time that Chrome received the Actions request.
      // Time from request to step start.
      int32 latency_start_ms = 1 [features = { field_presence: EXPLICIT }];

      // Time from request to step stop.
      int32 latency_stop_ms = 2 [features = { field_presence: EXPLICIT }];

      message ActionLatency {
        // The index of the action that triggered the latency step.
        int32 action_index = 1 [features = { field_presence: EXPLICIT }];
      }

      message PageStabilizationLatency {
        // The index of the action that triggered the page stabilization.
        int32 action_index = 1 [features = { field_presence: EXPLICIT }];
      }

      // The latency of the annotated page content step.
      message AnnotatedPageContentLatency {
        // The id of the tab that was observed.
        int32 id = 1 [features = { field_presence: EXPLICIT }];
      }

      // The latency of the screenshot step.
      message ScreenshotLatency {
        // The id of the tab that was observed.
        int32 id = 1 [features = { field_presence: EXPLICIT }];
      }
    }
  }
}

// A request to start a task.
// Next ID: 1

message BrowserStartTask {
  // The id of the tab the task should be started in. If not set, the task
  // will be started in a new tab.
  // TODO(crbug.com/421441072): Remove this field. This is only needed in the short term
  // since bluedog needs a mechanism to start tasks in a new tab, and the model
  // has not yet been trained to use CreateTabAction.
  int32 tab_id = 1 [features = { field_presence: EXPLICIT }];
}

// The response to a request to start a task.
// Next ID: 4
message BrowserStartTaskResult {
  // The id of the task that was created.
  int32 task_id = 1 [features = { field_presence: EXPLICIT }];

  // The id of the tab the action will be taken in.
  int32 tab_id = 2 [features = { field_presence: EXPLICIT }];

  // The status of the start task request. If not SUCCESS, the task_id and
  // tab_id may not be set.
  StartTaskStatus status = 3 [features = { field_presence: EXPLICIT }];

  enum StartTaskStatus {
    // Unknown start task status. This should not be used.
    START_TASK_STATUS_UNKNOWN = 0;

    // The task was started successfully.
    SUCCESS = 1;

    // The task was not started because there were too many tasks already
    // running.
    OVER_TASK_LIMIT = 2;
  }
}