components/optimization_guide/proto/features/model_prototyping.proto

// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";

package optimization_guide.proto;

import "components/optimization_guide/proto/features/forms_classifications.proto";

import "components/optimization_guide/proto/features/forms_predictions.proto";

import "components/optimization_guide/proto/features/common_quality_data.proto";

import "components/optimization_guide/proto/common_types.proto";

import "components/optimization_guide/proto/model_quality_metadata.proto";

option optimize_for = LITE_RUNTIME;
option java_package = "org.chromium.components.optimization_guide.features.proto";

option java_outer_classname = "ModelPrototypingProto";

// DO NOT EDIT THIS FILE DIRECTLY!
//
// This file is generated in g3 and then synced to Chrome. Instead, please
// refer to http://go/chrome-intelligence-feature-protos (Google-internal link),
// and then changes will be synced with Chrome automatically.

message ModelPrototypingLoggingData {
  ModelPrototypingRequest request = 1;

  ModelPrototypingResponse response = 2;

  ModelPrototypingMetadata metadata = 3;

  ModelExecutionInfo model_execution_info = 6;
}

message ModelPrototypingRequest {
  ModelingInputs modeling_inputs = 1;

  // The series of prompts to send to the model(s). The calls are run in series
  // and the responses can be used in future calls allowing piping the output of
  // one query into the input of the next.
  repeated PrototypingPrompt prototyping_prompts = 2;

  // The responses from previous calls to the model. Can be used in future
  // prompts. Syntax for accessing them is golang text/templates
  // e.g., something like {{index .GetModelResponses 0}}.
  repeated string model_responses = 3;

  // Defines a single prompt to be sent to the model.
  message PrototypingPrompt {
    // Prompt variables that can be used in the rest of the prompt. These are in
    // addition to any prompt variables defined in the prompt template in the
    // config for the model sequence. Prompt variables are helper functions that
    // can be used in the prompt. For example, a prompt variable could be
    // something like:
    // {{ $funVar := "1" }}
    // This would define a function that can be used in the prompt as
    // {{$funVar}}. The value of the function is "1".
    string prompt_variables = 1;

    // The prompt is composed by inserting the following roles into the prompt
    // template in the order they are defined.
    // Role system is generally the instructions for the model to follow.
    string system_instructions_template = 2;

    // Role context is the information around the user interaction such as page
    // state.
    string context_area_template = 3;

    // Role user is the information from the user such as a user input they
    // typed.
    string user_input_template = 4;

    // Information about the model to use.
    ModelInformation model_information = 5;

    // The image to use in the prompt.
    string image_template = 6;

    // The pre-existing conversation between the user and the model.
    Conversation conversation = 7;

    message ModelInformation {
      ModelEnum model_enum = 1;

      bool skip_model = 2;

      enum ModelEnum {
        MODEL_ENUM_UNSPECIFIED = 0;

        MODEL_ENUM_EVERGREEN_GEMINI_V2 = 1;

        MODEL_ENUM_SUP_GEMINI_V2 = 2;

        MODEL_ENUM_EVERGREEN_GEMINI_V2_MULTI_MODAL = 3;

        MODEL_ENUM_EVERGREEN_GEMINI_V2_MULTI_MODAL_MPP = 4;

        MODEL_ENUM_EVERGREEN_GEMINI_V3 = 5;

        MODEL_ENUM_EVERGREEN_GEMINI_NANO_V2_MULTI_MODAL = 6;

        MODEL_ENUM_EVERGREEN_GEMINI_V3_XS = 7;
      }
    }
  }

  // Data specific to the feature.
  message ModelingInputs {
    BrowserCollectedInformation browser_collected_information = 1;

    string user_input = 2;

    ExtensionAddedInformation extension_added_information = 3;
  }
}

// A representation of an ongoing conversation.
message Conversation {
  // The turns of various messages in the conversation. Media turns are not
  // included in this list.
  repeated ConversationTurn conversation_turns = 1;
}

// A single turn in a conversation.
message ConversationTurn {
  // The text of the conversation turn.
  string conversation_text = 1;

  // Which participant in the conversation this turn is from.
  string role = 2;

  // The mime type of the conversation turn.
  string mime_type = 3;
}

// All the information collected from the browser along with the user input
// (for features like Compose).
// NEXT ID: 18
message BrowserCollectedInformation {
  // The page context of the page the model is acting on.
  PageContext page_context = 1;

  // Deprecated, but still sent by older clients. Use page_context instead.
  string inner_text = 2;

  // Deprecated, but still sent by older clients. Use page_context instead.
  uint64 inner_text_offset = 3;

  // Custom text that a prototyper can inject into prompts. If the browser
  // collected information is not sufficient, an early stage prototype can
  // build a string in Chrome/colab to be used in the prompt. This allows
  // separation of prompt definition and call specific data.
  repeated string custom_data = 4;

  // The form to predict values for. It is expected for the values for each
  // field to be empty.
  FormData form_data = 5;

  // The current entries in the user annotations database helpful for filling
  // out this form.
  repeated UserAnnotationsEntry entries = 6;

  // The ungrouped tabs that are requested to be organized.
  repeated Tab tabs = 7;

  // The tab that was active at the time the user requested tab organization.
  int64 active_tab_id = 8;

  // All pre-existing tab groups.
  repeated TabGroup pre_existing_tab_groups = 9;

  // The site engagement information for the user.
  SiteEngagement site_engagement = 10;

  // The recent historical page data.
  repeated HistoryQueryResult history_query_result = 11;

  // Forms predictions metadata used for data collection/evaluations.
  FormsPredictionsRequest forms_predictions_request = 12;

  FormsPredictionsResponse forms_predictions_response = 13;

  // Specifies which data is included in this request to the browser.
  // For example, some requests only include foreground tab data,
  // whereas others include all profile data.
  ModelPrototypingCollectionSpecifier collection_specifier = 14;

  // Autofill AI type request and response.
  AutofillAiTypeRequest autofill_ai_type_request = 15;

  AutofillAiTypeResponse autofill_ai_type_response = 16;

  // The annotated page content with actionable elements for the foreground tab.
  // This is intentionally not part of the PageContext as it should not be used
  // for production but is needed for  prototyping. See crbug.com/403632369 for more
  // details.
  AnnotatedPageContent action_annotated_page_content = 17;
}

message HistoryQueryResult {
  // The history data.
  HistoryVisitData history_data = 1;

  // The query that generated this history data.
  HistoryQuery query = 2;
}

// The requested history data.
message HistoryVisitData {
  repeated HistoryVisitItem visit_item = 1;
}

// The data for a single history item.
message HistoryVisitItem {
  string page_title = 1;

  string page_url = 2;

  Timestamp visit_time = 3;

  repeated string passages = 4;
}

message HistoryQuery {
  // The text used to find matching history items.
  // If empty returns all entries matching the other query parameters.
  string query = 1;

  // The number of history items to fetch.
  int32 num_history_visits = 2;

  // The start and end time to fetch history items.
  DateTimeRange history_search_time_range = 3;
}

message DateTimeRange {
  // Both fields are inclusive.
  Timestamp start_time = 1;

  Timestamp end_time = 2;
}

message ModelPrototypingCollectionSpecifier {
  // The feature data collection specifier.
  FeatureDataCollectionSpecifier feature_data_collection_specifier = 1;

  // The browser data collection specifier.
  BrowserDataCollectionSpecifier browser_data_collection_specifier = 2;
}

// Used to specify the feature protos that should be collected.
message FeatureDataCollectionSpecifier {
  // Whether to collect the forms request.
  bool collect_forms_request = 1;

  // Whether to collect the compose request.
  bool collect_compose_request = 2;

  // Whether to collect the tab organization request.
  bool collect_tab_organization_request = 3;
}

// Input used to tell Chrome what data to collect.
message BrowserDataCollectionSpecifier {
  // The page context specifier for the foreground tab.
  PageContextSpecifier foreground_tab_page_context_specifier = 1;

  // The tabs context specifier for the foreground window's tab.
  TabsContextSpecifier tabs_context_specifier = 2;

  // Whether to collect the site engagement information for the user.
  bool site_engagement = 3;

  // Whether to collect the tab groups information for the user.
  bool tab_groups = 4;

  // Whether to collect the user annotations.
  bool user_annotations = 5;

  // Which history queries to collect data for.
  HistoryQuerySpecifiers history_query_specifiers = 6;
}

// Specifies the context to collect for a single page.
message PageContextSpecifier {
  // Send the default data (a subset of the data below) in addition to any
  // other fields set to true below.
  bool default_data = 1;

  // Whether to collect the inner text of the page.
  bool inner_text = 2;

  // The DOM node ID of the inner text where the "focus" is.
  int64 inner_text_dom_node_id = 3;

  // Whether to collect the tab screenshot.
  bool tab_screenshot = 4;

  // Whether to collect the accessibility tree.
  bool ax_tree = 5;

  // Whether to collect the PDF data.
  bool pdf_data = 6;

  // Whether to collect the last FormsPredictions request and response.
  bool forms_prediction = 7;

  // Whether to collect the annotated page content.
  bool annotated_page_content = 8;

  // The global ID of the field whose form should be extract (if any).
  AutofillFieldGlobalId field_global_id = 9;
}

message AutofillFieldGlobalId {
  string frame_token = 1;

  uint64 renderer_id = 2;
}

// Used to specify the context to collect for a set of tabs.
message TabsContextSpecifier {
  oneof specifier {
    PerTabSpecifiers per_tab_specifiers = 1;

    GeneralTabSpecifier general_tab_specifier = 2;
  }
}

// Repeated version of PerTabSpecifier for oneof.
message PerTabSpecifiers {
  // The specifiers for each tab.
  repeated PerTabSpecifier per_tab_specifier = 1;
}

// Used to specify the context to collect for a single tab.
message PerTabSpecifier {
  // The tab ID to collect the context for. Corresponds to the index in the
  // foreground window's tabs.
  int64 tab_id = 1;

  // The page context specifiers to use for this tab.
  PageContextSpecifier page_context_specifier = 2;
}

// Used to specify the context to collect for a set of tabs, generally limited
// to the first N tabs.
message GeneralTabSpecifier {
  // The page context specifiers to use for the tabs.
  PageContextSpecifier page_context_specifier = 1;

  // The maximum number of tabs to collect the context for (collects tab index
  // 0,...,n-1).
  int64 tab_limit = 2;
}

message HistoryQuerySpecifiers {
  repeated HistoryQuery history_queries = 1;
}

message ExtensionAddedInformation {
  // Free form data that a prototyper can inject into prompts.
  repeated string free_form_data = 1;

  // The local annotated page content as a json string.
  string local_annotated_page_content_as_json = 2;

  // The local annotated page content as a markdown string.
  string local_annotated_page_content_as_markdown = 3;

  // The local annotated page content as a list of passages.
  string local_annotated_page_content_as_passages = 4;
}

message ModelPrototypingResponse {
  // The series of prompts sent to the model corresponding to the
  // |prototyping_prompts| in the request.
  repeated string model_prompts = 1;

  // The responses from the model corresponding to |model_prompts|.
  repeated string model_responses = 2;

  // Ordered conversation between the user and the model.
  repeated Conversation conversations = 3;
}

message ModelPrototypingMetadata {
  // TODO: crbug.com/411195355 - Wire up IFTTT to the enums in the extension.

  // An optional tag to use for querying the record from MQLS logs.
  string logging_tag = 1;

  // An optional description to provide more context about the data uploaded by
  // the user.
  string logging_description = 2;

  // The email address of the user who uploaded the data. This will only be
  // populated for the TVC data upload.
  string user_email = 3;

  // A session id, used for connecting multiple uploads across a single user
  // session. The ids are generated client-side (in the extension) as a random
  // UUID.
  string session_id = 4;

  // When this data was collected. Recorded by the client, so timestamps might
  // not be consistent across data from different users. Combined with the
  // `session_id` field, this field can be used to stitch together data from
  // different uploads that belong to the same user session.
  Timestamp collection_time = 5;

  // The user-specified page annotation for the current page (if any).
  // Note: This field is commonly empty, as most pages have no relevant
  // annotation.
  // Note: Some pages may have multiple annotations. For example, a shopping
  // flow might combine the "view cart" and "checkout" steps.
  // Note: DataPol annotation is set to ST_NOT_REQUIRED because, although this
  // content is user-generated, it is fundamentally page metadata rather than
  // user content.
  repeated PageAnnotation page_annotation = 6;

  // If a task was completed (or aborted), the user-annotated status of the
  // task. For sessions spanning multiple uploads, this will only be set for
  // the final upload in the session.
  // Note: This field is commonly empty, as many data collection trajectories do
  // not ask the user to annotate the task completion status.
  TaskCompletionStatus task_completion_status = 7;

  enum PageAnnotation {
    PAGE_ANNOTATION_UNSPECIFIED = 0;

    // Used for any page representing a shopping cart.
    PAGE_ANNOTATION_SHOPPING_CART = 1;

    // Used for any page representing a checkout or review order page.
    PAGE_ANNOTATION_SHOPPING_CHECKOUT = 2;

    // Used for any page representing a successful order / confirmation page.
    PAGE_ANNOTATION_SHOPPING_SUCCESS = 3;
  }

  enum TaskCompletionStatus {
    TASK_COMPLETION_STATUS_UNSPECIFIED = 0;

    TASK_COMPLETION_STATUS_SUCCESS = 1;

    TASK_COMPLETION_STATUS_FAILURE_UNCATEGORIZED = 2;

    TASK_COMPLETION_STATUS_FAILURE_INFEASIBLE_TASK = 3;

    TASK_COMPLETION_STATUS_FAILURE_AMBIGUOUS_TASK = 4;

    TASK_COMPLETION_STATUS_FAILURE_SHOPPING_OUT_OF_STOCK = 5;
  }
}