components/optimization_guide/proto/model_quality_metadata.proto - chromium/src.git - Git at Google

 // Copyright 2023 The Chromium Authors
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 syntax = "proto3";

 option optimize_for = LITE_RUNTIME;
 option java_package = "org.chromium.components.optimization_guide.proto";

 option java_outer_classname = "ModelQualityMetadataProto";

 package optimization_guide.proto;

 import "components/optimization_guide/proto/model_execution.proto";
 import "components/optimization_guide/proto/on_device_base_model_metadata.proto";
 import "third_party/metrics_proto/system_profile.proto";

 // Contains metadata about a specific feature's AiLoggingData (e.g. training
 // opt-out, etc.).
 message LoggingMetadata {
   metrics.SystemProfileProto system_profile = 1;

   // Auxiliary system profile information that is not captured in system_profile
   // but is critical for understanding on-device model execution.
   OnDeviceSystemProfile on_device_system_profile = 2;

   // The client's unique identifier. Each ModelExecutionFeature will have a
   // different ID every day. It should only be used for model quality logging,
   // and should not be linked to any other data sources.
   int64 client_id = 3;

   // Field id 4 is intentionally reserved for internal use.
   reserved 4;

   // Whether this Chrome installation (most likely) has a Google-managed dogfood
   // account. Note that this will be `true` whenever at least one Chrome profile
   // is signed into an @google.com account, regardless of whether that profile
   // is currently active. Precisely, the reported value matches the return value
   // of `VariationsService::IsLikelyDogfoodClient()`:
   // https://source.chromium.org/chromium/chromium/src/+/main:components/variations/service/variations_service.h?q=symbol:%5Cbvariations::VariationsService::IsLikelyDogfoodClient%5Cb
   bool is_likely_dogfood_client = 5;
 }

 message OnDeviceSystemProfile {
   // The performance class determined for the device based on benchmarking at
   // the start of the session.
   PerformanceClass performance_class = 1;
 }

 enum PerformanceClass {
   PERFORMANCE_CLASS_UNSPECIFIED = 0;
   PERFORMANCE_CLASS_VERY_LOW = 1;
   PERFORMANCE_CLASS_LOW = 2;
   PERFORMANCE_CLASS_MEDIUM = 3;
   PERFORMANCE_CLASS_HIGH = 4;
   PERFORMANCE_CLASS_VERY_HIGH = 5;
 }

 // Data about model execution. This includes information about whether it was
 // done on a server or on a device. This includes an ID that can be mapped to
 // the ModelExecutionService logs on the server.
 message ModelExecutionInfo {
   reserved 1, 5, 6;

   // Information for how the on-device model execution was done.
   OnDeviceModelExecutionInfo on_device_model_execution_info = 2;

   // The error code returned by the model execution server, if any.
   ErrorResponse error_response = 3;

   // The model execution error enum. It offers more granularity over
   // `error_state` in `error_response`. This corresponds to the error enum:
   // OptimizationGuideModelExecutionError::ModelExecutionError. See more in
   // components/optimization_guide/core/model_execution/optimization_guide_model_execution_error.h
   uint32 model_execution_error_enum = 7;

   // The ID for the execution.
   //
   // If on-device, this will be prefixed with on-device:. Otherwise, this is the
   // server-side id as received from the server response used for joining with
   // ModelExecutionService logs when server-side execution is used.
   string execution_id = 4;
 }

 message OnDeviceModelExecutionInfo {
   reserved 1;

   // List of internal requests/responses that were performed to fulfill the
   // on-device model execution request.
   repeated InternalOnDeviceModelExecutionInfo execution_infos = 2;

   // Versions of the model being used for this execution.
   OnDeviceModelVersions model_versions = 3;
 }

 message OnDeviceModelVersions {
   // The model version for the on-device model.
   OnDeviceModelServiceVersion on_device_model_service_version = 1;

   // The model version for the text-safety model.
   int64 text_safety_model_version = 2;
 }

 message OnDeviceModelServiceVersion {
   // The version of the component used.
   string component_version = 1;

   // The version metadata of the base model used.
   OnDeviceBaseModelMetadata on_device_base_model_metadata = 2;

   // The version of the model adaptation downloaded from the server, if the
   // feature requires model adaptation.
   int64 model_adaptation_version = 3;
 }

 // InternalOnDeviceModelExecutionInfo is a request/response pair from a call to
 // a model used to fulfill the on-device execution request.
 message InternalOnDeviceModelExecutionInfo {
   InternalOnDeviceRequest request = 1;
   InternalOnDeviceResponse response = 2;
 }

 message InternalOnDeviceRequest {
   oneof request {
     // The request made to the on-device model service.
     OnDeviceModelServiceRequest on_device_model_service_request = 1;
     // The request made to the text safety model.
     TextSafetyModelRequest text_safety_model_request = 2;
   }
 }

 message InternalOnDeviceResponse {
   oneof response {
     // The response returned from the on-device model service.
     OnDeviceModelServiceResponse on_device_model_service_response = 1;
     // The response returned from the text safety model.
     TextSafetyModelResponse text_safety_model_response = 2;
   }
 }

 message OnDeviceModelServiceRequest {
   // The full input context constructed to be sent to the model.
   //
   // To better understand what was used as the input context, this string should
   // be truncated by `input_context_num_tokens_processed`.
   string input_context_string = 1;

   // The number of tokens processed for the input context.
   uint32 input_context_num_tokens_processed = 2;

   // The full execution string constructed to be sent to the model.
   //
   // To better understand what was used as the full string executed, this
   // string, truncated by `executed_num_tokens_processed`, should be
   // concatenated with the input context.
   string execution_string = 3;

   // The number of tokens processed for the execution string.
   uint32 execution_num_tokens_processed = 4;

   // The time in milliseconds from the context started being processed to the
   // request being initiated.
   int64 time_from_input_context_processed_to_request_initiated_millis = 5;
 }

 message OnDeviceModelServiceResponse {
   // The last output string sent to the calling code.
   string output_string = 1;

   // The time in milliseconds from Execute being called to the first response
   // tokens being output.
   int64 time_to_first_response_millis = 2;
   // The time in milliseconds from Execute being called to response completion.
   int64 time_to_completion_millis = 3;

   // The status of the completed response.
   OnDeviceModelServiceResponseStatus status = 4;

   // Whether the response was stopped because of repeating text.
   bool has_repeats = 5;
 }

 enum OnDeviceModelServiceResponseStatus {
   ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_UNSPECIFIED = 0;
   // The response completed.
   ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_SUCCESS = 1;
   // The response is retracted by the service.
   ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_RETRACTED = 2;
 }

 message TextSafetyModelRequest {
   // The text sent to the text safety model for evaluation.
   string text = 1;

   // The URL sent to the text safety model for evaluation, if any.
   string url = 2;
 }

 message TextSafetyModelResponse {
   // The scores output by the model.
   repeated float scores = 1;

   // Whether the output was deemed unsafe.
   bool is_unsafe = 2;

   // The ID for the execution of the text safety model if it leveraged the
   // server for text safety evaluation.
   string server_execution_id = 3;

   // The language code of the detected language. If detection was indeterminate,
   // this is "und" per ISO 639-2.
   string language_code = 4;

   // The confidence score for the detected language. Will be a value from
   // [0.0, 1.0].
   float language_confidence = 5;
 }
	// Copyright 2023 The Chromium Authors
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	syntax = "proto3";

	option optimize_for = LITE_RUNTIME;
	option java_package = "org.chromium.components.optimization_guide.proto";

	option java_outer_classname = "ModelQualityMetadataProto";

	package optimization_guide.proto;

	import "components/optimization_guide/proto/model_execution.proto";
	import "components/optimization_guide/proto/on_device_base_model_metadata.proto";
	import "third_party/metrics_proto/system_profile.proto";

	// Contains metadata about a specific feature's AiLoggingData (e.g. training
	// opt-out, etc.).
	message LoggingMetadata {
	metrics.SystemProfileProto system_profile = 1;

	// Auxiliary system profile information that is not captured in system_profile
	// but is critical for understanding on-device model execution.
	OnDeviceSystemProfile on_device_system_profile = 2;

	// The client's unique identifier. Each ModelExecutionFeature will have a
	// different ID every day. It should only be used for model quality logging,
	// and should not be linked to any other data sources.
	int64 client_id = 3;

	// Field id 4 is intentionally reserved for internal use.
	reserved 4;

	// Whether this Chrome installation (most likely) has a Google-managed dogfood
	// account. Note that this will be `true` whenever at least one Chrome profile
	// is signed into an @google.com account, regardless of whether that profile
	// is currently active. Precisely, the reported value matches the return value
	// of `VariationsService::IsLikelyDogfoodClient()`:
	// https://source.chromium.org/chromium/chromium/src/+/main:components/variations/service/variations_service.h?q=symbol:%5Cbvariations::VariationsService::IsLikelyDogfoodClient%5Cb
	bool is_likely_dogfood_client = 5;
	}

	message OnDeviceSystemProfile {
	// The performance class determined for the device based on benchmarking at
	// the start of the session.
	PerformanceClass performance_class = 1;
	}

	enum PerformanceClass {
	PERFORMANCE_CLASS_UNSPECIFIED = 0;
	PERFORMANCE_CLASS_VERY_LOW = 1;
	PERFORMANCE_CLASS_LOW = 2;
	PERFORMANCE_CLASS_MEDIUM = 3;
	PERFORMANCE_CLASS_HIGH = 4;
	PERFORMANCE_CLASS_VERY_HIGH = 5;
	}

	// Data about model execution. This includes information about whether it was
	// done on a server or on a device. This includes an ID that can be mapped to
	// the ModelExecutionService logs on the server.
	message ModelExecutionInfo {
	reserved 1, 5, 6;

	// Information for how the on-device model execution was done.
	OnDeviceModelExecutionInfo on_device_model_execution_info = 2;

	// The error code returned by the model execution server, if any.
	ErrorResponse error_response = 3;

	// The model execution error enum. It offers more granularity over
	// `error_state` in `error_response`. This corresponds to the error enum:
	// OptimizationGuideModelExecutionError::ModelExecutionError. See more in
	// components/optimization_guide/core/model_execution/optimization_guide_model_execution_error.h
	uint32 model_execution_error_enum = 7;

	// The ID for the execution.
	//
	// If on-device, this will be prefixed with on-device:. Otherwise, this is the
	// server-side id as received from the server response used for joining with
	// ModelExecutionService logs when server-side execution is used.
	string execution_id = 4;
	}

	message OnDeviceModelExecutionInfo {
	reserved 1;

	// List of internal requests/responses that were performed to fulfill the
	// on-device model execution request.
	repeated InternalOnDeviceModelExecutionInfo execution_infos = 2;

	// Versions of the model being used for this execution.
	OnDeviceModelVersions model_versions = 3;
	}

	message OnDeviceModelVersions {
	// The model version for the on-device model.
	OnDeviceModelServiceVersion on_device_model_service_version = 1;

	// The model version for the text-safety model.
	int64 text_safety_model_version = 2;
	}

	message OnDeviceModelServiceVersion {
	// The version of the component used.
	string component_version = 1;

	// The version metadata of the base model used.
	OnDeviceBaseModelMetadata on_device_base_model_metadata = 2;

	// The version of the model adaptation downloaded from the server, if the
	// feature requires model adaptation.
	int64 model_adaptation_version = 3;
	}

	// InternalOnDeviceModelExecutionInfo is a request/response pair from a call to
	// a model used to fulfill the on-device execution request.
	message InternalOnDeviceModelExecutionInfo {
	InternalOnDeviceRequest request = 1;
	InternalOnDeviceResponse response = 2;
	}

	message InternalOnDeviceRequest {
	oneof request {
	// The request made to the on-device model service.
	OnDeviceModelServiceRequest on_device_model_service_request = 1;
	// The request made to the text safety model.
	TextSafetyModelRequest text_safety_model_request = 2;
	}
	}

	message InternalOnDeviceResponse {
	oneof response {
	// The response returned from the on-device model service.
	OnDeviceModelServiceResponse on_device_model_service_response = 1;
	// The response returned from the text safety model.
	TextSafetyModelResponse text_safety_model_response = 2;
	}
	}

	message OnDeviceModelServiceRequest {
	// The full input context constructed to be sent to the model.
	//
	// To better understand what was used as the input context, this string should
	// be truncated by `input_context_num_tokens_processed`.
	string input_context_string = 1;

	// The number of tokens processed for the input context.
	uint32 input_context_num_tokens_processed = 2;

	// The full execution string constructed to be sent to the model.
	//
	// To better understand what was used as the full string executed, this
	// string, truncated by `executed_num_tokens_processed`, should be
	// concatenated with the input context.
	string execution_string = 3;

	// The number of tokens processed for the execution string.
	uint32 execution_num_tokens_processed = 4;

	// The time in milliseconds from the context started being processed to the
	// request being initiated.
	int64 time_from_input_context_processed_to_request_initiated_millis = 5;
	}

	message OnDeviceModelServiceResponse {
	// The last output string sent to the calling code.
	string output_string = 1;

	// The time in milliseconds from Execute being called to the first response
	// tokens being output.
	int64 time_to_first_response_millis = 2;
	// The time in milliseconds from Execute being called to response completion.
	int64 time_to_completion_millis = 3;

	// The status of the completed response.
	OnDeviceModelServiceResponseStatus status = 4;

	// Whether the response was stopped because of repeating text.
	bool has_repeats = 5;
	}

	enum OnDeviceModelServiceResponseStatus {
	ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_UNSPECIFIED = 0;
	// The response completed.
	ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_SUCCESS = 1;
	// The response is retracted by the service.
	ON_DEVICE_MODEL_SERVICE_RESPONSE_STATUS_RETRACTED = 2;
	}

	message TextSafetyModelRequest {
	// The text sent to the text safety model for evaluation.
	string text = 1;

	// The URL sent to the text safety model for evaluation, if any.
	string url = 2;
	}

	message TextSafetyModelResponse {
	// The scores output by the model.
	repeated float scores = 1;

	// Whether the output was deemed unsafe.
	bool is_unsafe = 2;

	// The ID for the execution of the text safety model if it leveraged the
	// server for text safety evaluation.
	string server_execution_id = 3;

	// The language code of the detected language. If detection was indeterminate,
	// this is "und" per ISO 639-2.
	string language_code = 4;

	// The confidence score for the detected language. Will be a value from
	// [0.0, 1.0].
	float language_confidence = 5;
	}