components/optimization_guide/proto/model_quality_metadata.proto

// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";

option optimize_for = LITE_RUNTIME;
option java_package = "org.chromium.components.optimization_guide.proto";

option java_outer_classname = "ModelQualityMetadataProto";

package optimization_guide.proto;

import "components/optimization_guide/proto/model_execution.proto";
import "third_party/metrics_proto/system_profile.proto";

// Contains metadata about a specific feature's AiLoggingData (e.g. training
// opt-out, etc.).
message LoggingMetadata {
  metrics.SystemProfileProto system_profile = 1;

  // Auxiliary system profile information that is not captured in system_profile
  // but is critical for understanding on-device model execution.
  OnDeviceSystemProfile on_device_system_profile = 2;

  // The client's unique identifier. Each ModelExecutionFeature will have a
  // different ID every day. It should only be used for model quality logging,
  // and should not be linked to any other data sources.
  int64 client_id = 3;
}

message OnDeviceSystemProfile {
  // The performance class determined for the device based on benchmarking at
  // the start of the session.
  PerformanceClass performance_class = 1;
}

enum PerformanceClass {
  PERFORMANCE_CLASS_UNSPECIFIED = 0;
  PERFORMANCE_CLASS_VERY_LOW = 1;
  PERFORMANCE_CLASS_LOW = 2;
  PERFORMANCE_CLASS_MEDIUM = 3;
  PERFORMANCE_CLASS_HIGH = 4;
  PERFORMANCE_CLASS_VERY_HIGH = 5;
}

// Data about model execution. This includes information about whether it was
// done on a server or on a device. This includes an ID that can be mapped to
// the ModelExecutionService logs on the server.
message ModelExecutionInfo {
  string server_execution_id = 1;

  // Information for how the on-device model execution was done.
  OnDeviceModelExecutionInfo on_device_model_execution_info = 2;

  // The error code returned by the model execution server, if any.
  ErrorResponse error_response = 3;
}

message OnDeviceModelExecutionInfo {
  reserved 1;

  // List of internal requests/responses that were performed to fulfill the
  // on-device model execution request.
  repeated InternalOnDeviceModelExecutionInfo execution_infos = 2;

  // Versions of the model being used for this execution.
  OnDeviceModelVersions model_versions = 3;
}

message OnDeviceModelVersions {
  // The model version for the on-device model.
  OnDeviceModelServiceVersion on_device_model_service_version = 1;

  // The model version for the text-safety model.
  int64 text_safety_model_version = 2;
}

message OnDeviceModelServiceVersion {
  // The version of the component used.
  string component_version = 1;
}

// InternalOnDeviceModelExecutionInfo is a request/response pair from a call to
// a model used to fulfill the on-device execution request.
message InternalOnDeviceModelExecutionInfo {
  InternalOnDeviceRequest request = 1;
  InternalOnDeviceResponse response = 2;
}

message InternalOnDeviceRequest {
  oneof request {
    // The request made to the on-device model service.
    OnDeviceModelServiceRequest on_device_model_service_request = 1;
    // The request made to the text safety model.
    TextSafetyModelRequest text_safety_model_request = 2;
  }
}

message InternalOnDeviceResponse {
  oneof response {
    // The response returned from the on-device model service.
    OnDeviceModelServiceResponse on_device_model_service_response = 1;
    // The response returned from the text safety model.
    TextSafetyModelResponse text_safety_model_response = 2;
  }
}

message OnDeviceModelServiceRequest {
  // The full input context constructed to be sent to the model.
  //
  // To better understand what was used as the input context, this string should
  // be truncated by `input_context_num_tokens_processed`.
  string input_context_string = 1;

  // The number of tokens processed for the input context.
  uint32 input_context_num_tokens_processed = 2;

  // The full execution string constructed to be sent to the model.
  //
  // To better understand what was used as the full string executed, this
  // string, truncated by `executed_num_tokens_processed`, should be
  // concatenated with the input context.
  string execution_string = 3;

  // The number of tokens processed for the execution string.
  uint32 execution_num_tokens_processed = 4;

  // The time in milliseconds from the context started being processed to the
  // request being initiated.
  int64 time_from_input_context_processed_to_request_initiated_millis = 5;
}

message OnDeviceModelServiceResponse {
  // The last output string sent to the calling code.
  string output_string = 1;

  // The time in milliseconds from Execute being called to the first response
  // tokens being output.
  int64 time_to_first_response_millis = 2;
  // The time in milliseconds from Execute being called to response completion.
  int64 time_to_completion_millis = 3;

  // The status of the completed response.
  OnDeviceModelServiceResponseStatus status = 4;

  // Whether the response was stopped because of repeating text.
  bool has_repeats = 5;
}

enum OnDeviceModelServiceResponseStatus {
  ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_UNSPECIFIED = 0;
  // The response completed.
  ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_SUCCESS = 1;
  // The response is retracted by the service.
  ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_RETRACTED = 2;
}

message TextSafetyModelRequest {
  // The text sent to the text safety model for evaluation.
  string text = 1;
}

message TextSafetyModelResponse {
  // The scores output by the model.
  repeated float scores = 1;

  // Whether the output was deemed unsafe.
  bool is_unsafe = 2;
}