extensions/common/api/media_perception_private.idl

// Copyright 2017 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Private API for receiving real-time media perception information.
[platforms=("chromeos")]
namespace mediaPerceptionPrivate {
  enum Status {
    // The media analytics process is waiting to be launched.
    UNINITIALIZED,

    // The analytics process is running and the media processing pipeline is
    // started, but it is not yet receiving image frames. This is a
    // transitional state between <code>SUSPENDED</code> and
    // <code>RUNNING</code> for the time it takes to warm up the media
    // processing pipeline, which can take anywhere from a few seconds to a
    // minute.
    // Note: <code>STARTED</code> is the initial reply to SetState
    // <code>RUNNING</code>.
    STARTED,

    // The analytics process is running and the media processing pipeling is
    // injesting image frames. At this point, MediaPerception signals should
    // be coming over D-Bus.
    RUNNING,

    // Analytics process is running and the media processing pipeline is ready
    // to be set to state <code>RUNNING</code>. The D-Bus communications
    // are enabled but the media processing pipeline is suspended.
    SUSPENDED,

    // Enum for restarting the media analytics process using Upstart.
    // Calling setState <code>RESTARTING</code> will restart the media process
    // to the <code>SUSPENDED</code> state. The app has to set the state to
    // <code>RUNNING</code> in order to start receiving media perception
    // information again.
    RESTARTING,

    // Stops the media analytics process via Upstart.
    STOPPED,

    // Indicates that a ServiceError has occurred.
    SERVICE_ERROR
  };

  enum ServiceError {
    // The media analytics process could not be reached. This is likely due to
    // a faulty comms configuration or that the process crashed.
    SERVICE_UNREACHABLE,

    // The media analytics process is not running. The MPP API knows that the
    // process has not been started yet.
    SERVICE_NOT_RUNNING,

    // The media analytics process is busy launching. Wait for setState
    // <code>RUNNING</code> or setState <code>RESTARTING</code> callback.
    SERVICE_BUSY_LAUNCHING,

    // The component is not installed properly.
    SERVICE_NOT_INSTALLED,

    // Failed to establish a Mojo connection to the service.
    MOJO_CONNECTION_FAILURE
  };

  enum Feature {
    AUTOZOOM,
    HOTWORD_DETECTION,
    OCCUPANCY_DETECTION,
    EDGE_EMBEDDINGS,
    SOFTWARE_CROPPING
  };

  dictionary NamedTemplateArgument {
    DOMString? name;
    (DOMString or double)? value;
  };

  enum ComponentType {
    // The smaller component with limited functionality (smaller size and
    // limited models).
    LIGHT,
    // The fully-featured component with more functionality (larger size and
    // more models).
    FULL
  };

  // The status of the media analytics process component on the device.
  enum ComponentStatus {
    UNKNOWN,
    // The component is successfully installed and the image is mounted.
    INSTALLED,
    // The component failed to download, install or load.
    FAILED_TO_INSTALL
  };

  // Error code associated with a failure to install the media analytics
  // component.
  enum ComponentInstallationError {
    // Component requested does not exist.
    UNKNOWN_COMPONENT,

    // The update engine fails to install component.
    INSTALL_FAILURE,

    // Component can not be mounted.
    MOUNT_FAILURE,

    // The component is not compatible with the device.
    COMPATIBILITY_CHECK_FAILED,

    // The component was not found - reported for load requests with kSkip
    // update policy.
    NOT_FOUND
  };

  dictionary Component {
    ComponentType type;
  };

  // The state of the media analytics downloadable component.
  dictionary ComponentState {
    ComponentStatus status;

    // The version string for the current component.
    DOMString? version;

    // If the component installation failed, the encountered installation
    // error. Not set if the component installation succeeded.
    ComponentInstallationError? installationErrorCode;
  };

  // ------------------- Start of process management definitions. ------------
  // New interface for managing the process state of the media perception
  // service with the intention of eventually phasing out the setState() call.
  enum ProcessStatus {
    // The component process state is unknown, for example, if the process is
    // waiting to be launched. This is the initial state before
    // $(ref:setComponentProcessState) is first called.
    UNKNOWN,

    // The component process has been started.
    // This value can only be passed to $(ref:setComponentProcessState) if the
    // process is currently in state <code>STOPPED</code> or
    // <code>UNKNOWN</code>.
    STARTED,

    // The component process has been stopped.
    // This value can only be passed to $(ref:setComponentProcessState) if the
    // process is currently in state <code>STARTED</code>.
    // Note: the process is automatically stopped when the Chrome process
    // is closed.
    STOPPED,

    // Indicates that a ServiceError has occurred.
    SERVICE_ERROR
  };

  dictionary ProcessState {
    ProcessStatus? status;

    // Return parameter for $(ref:setComponentProcessState) that
    // specifies the error type for failure cases.
    ServiceError? serviceError;
  };
  // ------------------- End of process management definitions. --------------

  // The parameters for processing a particular video stream.
  dictionary VideoStreamParam {
    // Identifies the video stream described by these parameters.
    DOMString? id;

    // Frame width in pixels.
    long? width;

    // Frame height in pixels.
    long? height;

    // The frame rate at which this video stream would be processed.
    long? frameRate;
  };

  dictionary Point {
    // The horizontal distance from the top left corner of the image.
    double? x;

    // The vertical distance from the top left corner of the image.
    double? y;
  };

  // The parameters for a whiteboard in the image frame. Corners are given in
  // pixel coordinates normalized to the size of the image frame (i.e. in the
  // range [(0.0, 0.0), (1.0, 1.0)]. The aspectRatio is the physical aspect
  // ratio of the whiteboard (e.g. for a 1m high and 2m wide whiteboard, the
  // aspect ratio would be 2).
  dictionary Whiteboard {
    // The top left corner of the whiteboard in the image frame.
    Point? topLeft;

    // The top right corner of the whiteboard in the image frame.
    Point? topRight;

    // The bottom left corner of the whiteboard in the image frame.
    Point? bottomLeft;

    // The bottom right corner of the whiteboard in the image frame.
    Point? bottomRight;

    // The physical aspect ratio of the whiteboard.
    double? aspectRatio;
  };

  // The system and configuration state of the analytics process.
  dictionary State {
    Status status;

    // Optional $(ref:setState) parameter. Specifies the video device the media
    // analytics process should open while the media processing pipeline is
    // starting. To set this parameter, status has to be <code>RUNNING</code>.
    DOMString? deviceContext;

    // Return parameter for $(ref:setState) or $(ref:getState) that
    // specifies the error type for failure cases.
    ServiceError? serviceError;

    // A list of video streams processed by the analytics process. To set this
    // parameter, status has to be <code>RUNNING</code>.
    VideoStreamParam[]? videoStreamParam;

    // Media analytics configuration. It can only be used when setting state to
    // RUNNING.
    DOMString? configuration;

    // Corners and aspect ratio of the whiteboard in the image frame. Should
    // only be set when setting state to <code>RUNNING</code> and configuration
    // to whiteboard.
    Whiteboard? whiteboard;

    // A list of enabled media perception features.
    Feature[]? features;

    // A list of named parameters to be substituted at start-up. Will
    // only have effect when setting state to <code>RUNNING</code>.
    NamedTemplateArgument[]? namedTemplateArguments;
  };

  dictionary BoundingBox {
    // Specifies whether the points are normalized to the size of the image.
    boolean? normalized;

    // The two points that define the corners of a bounding box.
    Point? topLeft;
    Point? bottomRight;
  };

  enum DistanceUnits {
    UNSPECIFIED,
    METERS,
    PIXELS
  };

  // Generic dictionary to encapsulate a distance magnitude and units.
  dictionary Distance {
    // This field provides flexibility to report depths or distances of
    // different entity types with different units.
    DistanceUnits? units;

    double? magnitude;
  };

  enum EntityType {
    UNSPECIFIED,
    FACE,
    PERSON,
    MOTION_REGION,
    LABELED_REGION
  };

  enum FramePerceptionType {
    UNKNOWN_TYPE,
    FACE_DETECTION,
    PERSON_DETECTION,
    MOTION_DETECTION
  };

  dictionary Entity {
    // A unique id associated with the detected entity, which can be used to
    // track the entity over time.
    long? id;

    EntityType? type;

    // Label for this entity.
    DOMString? entityLabel;

    // Minimum box which captures entire detected entity.
    BoundingBox? boundingBox;

    // A value for the quality of this detection.
    double? confidence;

    // The estimated depth of the entity from the camera.
    Distance? depth;
  };

  dictionary PacketLatency {
    // Label for this packet.
    DOMString? packetLabel;

    // Packet processing latency in microseconds.
    long? latencyUsec;
  };

  // Type of lighting conditions.
  enum LightCondition {
    UNSPECIFIED,

    // No noticeable change occurred.
    NO_CHANGE,

    // Light was switched on in the room.
    TURNED_ON,

    // Light was switched off in the room.
    TURNED_OFF,

    // Light gradually got dimmer (for example, due to a sunset).
    DIMMER,

    // Light gradually got brighter (for example, due to a sunrise).
    BRIGHTER,

    // Black frame was detected - the current frame contains only noise.
    BLACK_FRAME
  };

  // Detection of human presence close to the camera.
  dictionary VideoHumanPresenceDetection {
    // Indicates a probability in [0, 1] interval that a human is present in
    // the video frame.
    double? humanPresenceLikelihood;

    // Indicates a probability in [0, 1] that motion has been detected in the
    // video frame.
    double? motionDetectedLikelihood;

    // Indicates lighting condition in the video frame.
    LightCondition? lightCondition;

    // Indicates a probablity in [0, 1] interval that
    // <code>lightCondition</code> value is correct.
    double? lightConditionLikelihood;
  };

  // The set of computer vision metadata for an image frame.
  dictionary FramePerception {
    long? frameId;

    long? frameWidthInPx;
    long? frameHeightInPx;

    // The timestamp associated with the frame (when its recieved by the
    // analytics process).
    double? timestamp;

    // The list of entities detected in this frame.
    Entity[]? entities;

    // Processing latency for a list of packets.
    PacketLatency[]? packetLatency;

    // Human presence detection results for a video frame.
    VideoHumanPresenceDetection? videoHumanPresenceDetection;

    // Indicates what types of frame perception were run.
    FramePerceptionType[]? framePerceptionTypes;
  };

  // An estimate of the direction that the sound is coming from.
  dictionary AudioLocalization {
    // An angle in radians in the horizontal plane. It roughly points to the
    // peak in the probability distribution of azimuth defined below.
    double? azimuthRadians;

    // A probability distribution for the current snapshot in time that shows
    // the likelihood of a sound source being at a particular azimuth. For
    // example, <code>azimuthScores = [0.1, 0.2, 0.3, 0.4]</code> means that
    // the probability that the sound is coming from an azimuth of 0, pi/2, pi,
    // 3*pi/2 is 0.1, 0.2, 0.3 and 0.4, respectively.
    double[]? azimuthScores;
  };

  // Spectrogram of an audio frame.
  dictionary AudioSpectrogram {
    double[]? values;
  };

  // Detection of human presence close to the microphone.
  dictionary AudioHumanPresenceDetection {
    // Indicates a probability in [0, 1] interval that a human has caused a
    // sound close to the microphone.
    double? humanPresenceLikelihood;

    // Estimate of the noise spectrogram.
    AudioSpectrogram? noiseSpectrogram;

    // Spectrogram of an audio frame.
    AudioSpectrogram? frameSpectrogram;
  };

  enum HotwordType {
    UNKNOWN_TYPE,
    OK_GOOGLE
  };

  // A hotword detected in the audio stream.
  dictionary Hotword {
    // Unique identifier for the hotword instance. Note that a single hotword
    // instance can span more than one audio frame. In that case a single
    // hotword instance can be reported in multiple Hotword or HotwordDetection
    // results. Hotword results associated with the same hotword instance will
    // have the same <code>id</code>.
    long? id;

    // Indicates the type of this hotword.
    HotwordType? type;

    // Id of the audio frame in which the hotword was detected.
    long? frameId;

    // Indicates the start time of this hotword in the audio frame.
    long? startTimestampMs;

    // Indicates the end time of this hotword in the audio frame.
    long? endTimestampMs;

    // Indicates a probability in [0, 1] interval that this hotword is present
    // in the audio frame.
    double? confidence;
  };

  // Detection of hotword in the audio stream.
  dictionary HotwordDetection {
    Hotword[]? hotwords;
  };

  // Audio perception results for an audio frame.
  dictionary AudioPerception {
    // A timestamp in microseconds attached when this message was generated.
    double? timestampUs;

    // Audio localization results for an audio frame.
    AudioLocalization? audioLocalization;

    // Audio human presence detection results for an audio frame.
    AudioHumanPresenceDetection? audioHumanPresenceDetection;

    // Hotword detection results.
    HotwordDetection? hotwordDetection;
  };

  // Detection of human presence based on both audio and video inputs.
  dictionary AudioVisualHumanPresenceDetection {
    // Indicates a probability in [0, 1] interval that a human is present.
    double? humanPresenceLikelihood;
  };

  // Perception results based on both audio and video inputs.
  dictionary AudioVisualPerception {
    // A timestamp in microseconds attached when this message was generated.
    double? timestampUs;

    // Human presence detection results.
    AudioVisualHumanPresenceDetection? audioVisualHumanPresenceDetection;
  };

  // Stores metadata such as version of media perception features.
  dictionary Metadata {
    DOMString? visualExperienceControllerVersion;
  };

  dictionary MediaPerception {
    // The time the media perception data was emitted by the media processing
    // pipeline. This value will be greater than the timestamp stored within
    // the FramePerception dictionary and the difference between them can be
    // viewed as the processing time for a single frame.
    double? timestamp;

    // An array of framePerceptions.
    FramePerception[]? framePerceptions;

    // An array of audio perceptions.
    AudioPerception[]? audioPerceptions;

    // An array of audio-visual perceptions.
    AudioVisualPerception[]? audioVisualPerceptions;

    // Stores metadata such as version of media perception features.
    Metadata? metadata;
  };

  enum ImageFormat {
    // Image represented by RGB data channels.
    RAW,
    PNG,
    JPEG
  };

  dictionary ImageFrame {
    long? width;
    long? height;

    ImageFormat? format;

    long? dataLength;

    // The bytes of the image frame.
    ArrayBuffer? frame;
  };

  dictionary PerceptionSample {
    // The video analytics FramePerception for the associated image frame
    // data.
    FramePerception? framePerception;

    // The image frame data for the associated FramePerception object.
    ImageFrame? imageFrame;

    // The audio perception results for an audio frame.
    AudioPerception? audioPerception;

    // Perception results based on both audio and video inputs.
    AudioVisualPerception? audioVisualPerception;

    // Stores metadata such as version of media perception features.
    Metadata? metadata;
  };

  dictionary Diagnostics {
    // Return parameter for $(ref:getDiagnostics) that specifies the error
    // type for failure cases.
    ServiceError? serviceError;

    // A buffer of image frames and the associated video analytics information
    // that can be used to diagnose a malfunction.
    PerceptionSample[]? perceptionSamples;
  };

  callback StateCallback = void(State state);

  callback DiagnosticsCallback = void(Diagnostics diagnostics);

  callback ComponentStateCallback = void(ComponentState componentState);

  callback ProcessStateCallback = void(ProcessState processState);

  interface Functions {
    // Gets the status of the media perception process.
    // |callback| : The current state of the system.
    static void getState(StateCallback callback);

    // Sets the desired state of the system.
    // |state| : A dictionary with the desired new state. The only settable
    // states are <code>RUNNING</code>, <code>SUSPENDED</code>, and
    // <code>RESTARTING</code>.
    // |callback| : Invoked with the State of the system after setting it. Can
    // be used to verify the state was set as desired.
    static void setState(
        State state,
        StateCallback callback);

    // Get a diagnostics buffer out of the video analytics process.
    // |callback| : Returns a Diagnostics dictionary object.
    static void getDiagnostics(DiagnosticsCallback callback);

    // Attempts to download and load the media analytics component. This
    // function should be called every time a client starts using this API. If
    // the component is already loaded, the callback will simply return that
    // information. The process must be <code>STOPPED</code> for this function
    // to succeed.
    // Note: If a different component type is desired, this function can
    // be called with the new desired type and the new component will be
    // downloaded and installed.
    // |component| : The desired component to install and load.
    // |callback| : Returns the state of the component.
    static void setAnalyticsComponent(
        Component component,
        ComponentStateCallback callback);

    // Manages the lifetime of the component process. This function should
    // only be used if the component is installed. It will fail if the
    // component is not installed.
    // |processState| : The desired state for the component process.
    // |callback| : Reports the new state of the process, which is expected to
    // be the same as the desired state, unless something goes wrong.
    static void setComponentProcessState(
        ProcessState processState,
        ProcessStateCallback callback);
  };

  interface Events {
    // Fired when media perception information is received from the media
    // analytics process.
    // |mediaPerception| : The dictionary which contains a dump of everything
    // the analytics process has detected or determined from the incoming media
    // streams.
    static void onMediaPerception(MediaPerception mediaPerception);
  };
};