media/video/video_encode_accelerator.h

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_
#define MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_

#include <stddef.h>
#include <stdint.h>

#include <memory>
#include <vector>

#include "base/callback_forward.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/single_thread_task_runner.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/media_export.h"
#include "media/base/video_bitrate_allocation.h"
#include "media/base/video_codecs.h"
#include "media/base/video_frame.h"
#include "media/video/h264_parser.h"
#include "media/video/video_encoder_info.h"

namespace media {

class BitstreamBuffer;
class VideoFrame;

//  Metadata for a VP8 bitstream buffer.
//  |non_reference| is true iff this frame does not update any reference buffer,
//                  meaning dropping this frame still results in a decodable
//                  stream.
//  |temporal_idx|  indicates the temporal index for this frame.
//  |layer_sync|    if true iff this frame has |temporal_idx| > 0 and does NOT
//                  reference any reference buffer containing a frame with
//                  temporal_idx > 0.
struct MEDIA_EXPORT Vp8Metadata final {
  Vp8Metadata();
  bool non_reference;
  uint8_t temporal_idx;
  bool layer_sync;
};

// Metadata for a VP9 bitstream buffer
// |has_reference|      is true iff this frame depends on previously coded frame
//                      on the same spatial layer.
// |temporal_up_switch| is true iff this frame only references TL0 frames.
// |tempora_idx|        indicates the temporal index for this frame.
// |p_diffs|            indicates the differences between the picture id of this
//                      frame and picture ids of reference frames.
// CAVEATS: This Vp9 metadata is for temporal layer bitstream and not sufficient
// for spatial layer bitstream.
struct MEDIA_EXPORT Vp9Metadata final {
  Vp9Metadata();
  ~Vp9Metadata();
  Vp9Metadata(const Vp9Metadata&);

  // Default values are for keyframe.
  bool has_reference = false;
  bool temporal_up_switch = false;
  uint8_t temporal_idx = 0;
  std::vector<uint8_t> p_diffs;
};

//  Metadata associated with a bitstream buffer.
//  |payload_size| is the byte size of the used portion of the buffer.
//  |key_frame| is true if this delivered frame is a keyframe.
//  |timestamp| is the same timestamp as in VideoFrame passed to Encode().
//  |vp8|, if set, contains metadata specific to VP8. See above.
struct MEDIA_EXPORT BitstreamBufferMetadata final {
  BitstreamBufferMetadata();
  BitstreamBufferMetadata(const BitstreamBufferMetadata& other);
  BitstreamBufferMetadata& operator=(const BitstreamBufferMetadata& other);
  BitstreamBufferMetadata(BitstreamBufferMetadata&& other);
  BitstreamBufferMetadata(size_t payload_size_bytes,
                          bool key_frame,
                          base::TimeDelta timestamp);
  ~BitstreamBufferMetadata();

  size_t payload_size_bytes;
  bool key_frame;
  base::TimeDelta timestamp;

  // Either |vp8| or |vp9| may be set, but not both of them. Presumably, it's
  // also possible for none of them to be set.
  base::Optional<Vp8Metadata> vp8;
  base::Optional<Vp9Metadata> vp9;
};

// Video encoder interface.
class MEDIA_EXPORT VideoEncodeAccelerator {
 public:
  // Specification of an encoding profile supported by an encoder.
  struct MEDIA_EXPORT SupportedProfile {
    SupportedProfile();
    SupportedProfile(VideoCodecProfile profile,
                     const gfx::Size& max_resolution,
                     uint32_t max_framerate_numerator = 0u,
                     uint32_t max_framerate_denominator = 1u);
    ~SupportedProfile();
    VideoCodecProfile profile;
    gfx::Size min_resolution;
    gfx::Size max_resolution;
    uint32_t max_framerate_numerator;
    uint32_t max_framerate_denominator;
  };
  using SupportedProfiles = std::vector<SupportedProfile>;
  using FlushCallback = base::OnceCallback<void(bool)>;

  // Enumeration of potential errors generated by the API.
  enum Error {
    // An operation was attempted during an incompatible encoder state.
    kIllegalStateError,
    // Invalid argument was passed to an API method.
    kInvalidArgumentError,
    // A failure occurred at the GPU process or one of its dependencies.
    // Examples of such failures include GPU hardware failures, GPU driver
    // failures, GPU library failures, GPU process programming errors, and so
    // on.
    kPlatformFailureError,
    kErrorMax = kPlatformFailureError
  };

  // A default framerate for all VEA implementations.
  enum { kDefaultFramerate = 30 };

  // Parameters required for VEA initialization.
  struct MEDIA_EXPORT Config {
    // Indicates if video content should be treated as a "normal" camera feed
    // or as generated (e.g. screen capture).
    enum class ContentType { kCamera, kDisplay };
    // Indicates the storage type of a video frame provided on Encode().
    // kShmem if a video frame is mapped in user space.
    // kDmabuf if a video frame is referred by dmabuf.
    enum class StorageType { kShmem, kDmabuf };

    struct MEDIA_EXPORT SpatialLayer {
      // The encoder dimension of the spatial layer.
      int32_t width = 0;
      int32_t height = 0;
      // The bitrate of encoded output stream of the spatial layer in bits per
      // second.
      uint32_t bitrate_bps = 0u;
      uint32_t framerate = 0u;
      // The recommended maximum qp value of the spatial layer. VEA can ignore
      // this value.
      uint8_t max_qp = 0u;
      // The number of temporal layers of the spatial layer. The detail of
      // the temporal layer structure is up to VideoEncodeAccelerator.
      uint8_t num_of_temporal_layers = 0u;
    };

    Config();
    Config(const Config& config);

    Config(VideoPixelFormat input_format,
           const gfx::Size& input_visible_size,
           VideoCodecProfile output_profile,
           uint32_t initial_bitrate,
           base::Optional<uint32_t> initial_framerate = base::nullopt,
           base::Optional<uint32_t> gop_length = base::nullopt,
           base::Optional<uint8_t> h264_output_level = base::nullopt,
           bool is_constrained_h264 = false,
           base::Optional<StorageType> storage_type = base::nullopt,
           ContentType content_type = ContentType::kCamera,
           const std::vector<SpatialLayer>& spatial_layers = {});

    ~Config();

    std::string AsHumanReadableString() const;

    bool HasTemporalLayer() const;
    bool HasSpatialLayer() const;

    // Frame format of input stream (as would be reported by
    // VideoFrame::format() for frames passed to Encode()).
    VideoPixelFormat input_format;

    // Resolution of input stream (as would be reported by
    // VideoFrame::visible_rect().size() for frames passed to Encode()).
    gfx::Size input_visible_size;

    // Codec profile of encoded output stream.
    VideoCodecProfile output_profile;

    // Initial bitrate of encoded output stream in bits per second.
    uint32_t initial_bitrate;

    // Initial encoding framerate in frames per second. This is optional and
    // VideoEncodeAccelerator should use |kDefaultFramerate| if not given.
    base::Optional<uint32_t> initial_framerate;

    // Group of picture length for encoded output stream, indicates the
    // distance between two key frames, i.e. IPPPIPPP would be represent as 4.
    base::Optional<uint32_t> gop_length;

    // Codec level of encoded output stream for H264 only. This value should
    // be aligned to the H264 standard definition of SPS.level_idc.
    // If this is not given, VideoEncodeAccelerator selects one of proper H.264
    // levels for |input_visible_size| and |initial_framerate|.
    base::Optional<uint8_t> h264_output_level;

    // Indicates baseline profile or constrained baseline profile for H264 only.
    bool is_constrained_h264;

    // The storage type of video frame provided on Encode().
    // If no value is set, VEA doesn't check the storage type of video frame on
    // Encode().
    // This is kShmem iff a video frame is mapped in user space.
    // This is kDmabuf iff a video frame has dmabuf.
    base::Optional<StorageType> storage_type;

    // Indicates captured video (from a camera) or generated (screen grabber).
    // Screen content has a number of special properties such as lack of noise,
    // burstiness of motion and requirements for readability of small text in
    // bright colors. With this content hint the encoder may choose to optimize
    // for the given use case.
    ContentType content_type;

    // The configuration for spatial layers. This is not empty if and only if
    // either spatial or temporal layer encoding is configured. When this is not
    // empty, VideoEncodeAccelerator should refer the width, height, bitrate and
    // etc. of |spatial_layers|.
    std::vector<SpatialLayer> spatial_layers;
  };

  // Interface for clients that use VideoEncodeAccelerator. These callbacks will
  // not be made unless Initialize() has returned successfully.
  class MEDIA_EXPORT Client {
   public:
    // Callback to tell the client what size of frames and buffers to provide
    // for input and output.  The VEA disclaims use or ownership of all
    // previously provided buffers once this callback is made.
    // Parameters:
    //  |input_count| is the number of input VideoFrames required for encoding.
    //  The client should be prepared to feed at least this many frames into the
    //  encoder before being returned any input frames, since the encoder may
    //  need to hold onto some subset of inputs as reference pictures.
    //  |input_coded_size| is the logical size of the input frames (as reported
    //  by VideoFrame::coded_size()) to encode, in pixels.  The encoder may have
    //  hardware alignment requirements that make this different from
    //  |input_visible_size|, as requested in Initialize(), in which case the
    //  input VideoFrame to Encode() should be padded appropriately.
    //  |output_buffer_size| is the required size of output buffers for this
    //  encoder in bytes.
    virtual void RequireBitstreamBuffers(unsigned int input_count,
                                         const gfx::Size& input_coded_size,
                                         size_t output_buffer_size) = 0;

    // Callback to deliver encoded bitstream buffers.  Ownership of the buffer
    // is transferred back to the VEA::Client once this callback is made.
    // Parameters:
    //  |bitstream_buffer_id| is the id of the buffer that is ready.
    //  |metadata| contains data such as payload size and timestamp. See above.
    virtual void BitstreamBufferReady(
        int32_t bitstream_buffer_id,
        const BitstreamBufferMetadata& metadata) = 0;

    // Error notification callback. Note that errors in Initialize() will not be
    // reported here, but will instead be indicated by a false return value
    // there.
    virtual void NotifyError(Error error) = 0;

    // Call VideoEncoderInfo of the VEA is changed.
    virtual void NotifyEncoderInfoChange(const VideoEncoderInfo& info);

   protected:
    // Clients are not owned by VEA instances and should not be deleted through
    // these pointers.
    virtual ~Client() {}
  };

  // Video encoder functions.

  // Returns a list of the supported codec profiles of the video encoder. This
  // can be called before Initialize().
  virtual SupportedProfiles GetSupportedProfiles() = 0;

  // Initializes the video encoder with specific configuration.  Called once per
  // encoder construction.  This call is synchronous and returns true iff
  // initialization is successful.
  // TODO(mcasas): Update to asynchronous, https://crbug.com/744210.
  // Parameters:
  //  |config| contains the initialization parameters.
  //  |client| is the client of this video encoder.  The provided pointer must
  //  be valid until Destroy() is called.
  // TODO(sheu): handle resolution changes.  http://crbug.com/249944
  virtual bool Initialize(const Config& config, Client* client) = 0;

  // Encodes the given frame.
  // The storage type of |frame| must be the |storage_type| if it is specified
  // in Initialize().
  // TODO(crbug.com/895230): Raise an error if the storage types are mismatch.
  // Parameters:
  //  |frame| is the VideoFrame that is to be encoded.
  //  |force_keyframe| forces the encoding of a keyframe for this frame.
  virtual void Encode(scoped_refptr<VideoFrame> frame, bool force_keyframe) = 0;

  // Send a bitstream buffer to the encoder to be used for storing future
  // encoded output.  Each call here with a given |buffer| will cause the buffer
  // to be filled once, then returned with BitstreamBufferReady().
  // Parameters:
  //  |buffer| is the bitstream buffer to use for output.
  virtual void UseOutputBitstreamBuffer(BitstreamBuffer buffer) = 0;

  // Request a change to the encoding parameters. This is only a request,
  // fulfilled on a best-effort basis.
  // Parameters:
  //  |bitrate| is the requested new bitrate, in bits per second.
  //  |framerate| is the requested new framerate, in frames per second.
  virtual void RequestEncodingParametersChange(uint32_t bitrate,
                                               uint32_t framerate) = 0;

  // Request a change to the encoding parameters. This is only a request,
  // fulfilled on a best-effort basis. If not implemented, default behavior is
  // to get the sum over layers and pass to version with bitrate as uint32_t.
  // Parameters:
  //  |bitrate| is the requested new bitrate, per spatial and temporal layer.
  //  |framerate| is the requested new framerate, in frames per second.
  virtual void RequestEncodingParametersChange(
      const VideoBitrateAllocation& bitrate,
      uint32_t framerate);

  // Destroys the encoder: all pending inputs and outputs are dropped
  // immediately and the component is freed.  This call may asynchronously free
  // system resources, but its client-visible effects are synchronous. After
  // this method returns no more callbacks will be made on the client. Deletes
  // |this| unconditionally, so make sure to drop all pointers to it!
  virtual void Destroy() = 0;

  // Flushes the encoder: all pending inputs will be encoded and all bitstreams
  // handed back to the client, and afterwards the |flush_callback| will be
  // called. The FlushCallback takes a boolean argument: |true| indicates the
  // flush is complete; |false| indicates the flush is cancelled due to errors
  // or destruction. The client should not invoke Flush() or Encode() while the
  // previous Flush() is not finished yet.
  virtual void Flush(FlushCallback flush_callback);

  // Returns true if the encoder support flush. This method must be called after
  // VEA has been initialized.
  virtual bool IsFlushSupported();

  // Returns true if the encoder supports automatic resize of GPU backed frames
  // to the size provided during encoder configuration.
  // This method must be called after VEA has been initialized.
  virtual bool IsGpuFrameResizeSupported();

 protected:
  // Do not delete directly; use Destroy() or own it with a scoped_ptr, which
  // will Destroy() it properly by default.
  virtual ~VideoEncodeAccelerator();
};

MEDIA_EXPORT bool operator==(const Vp8Metadata& l, const Vp8Metadata& r);
MEDIA_EXPORT bool operator==(const Vp9Metadata& l, const Vp9Metadata& r);
MEDIA_EXPORT bool operator==(const BitstreamBufferMetadata& l,
                             const BitstreamBufferMetadata& r);
MEDIA_EXPORT bool operator==(
    const VideoEncodeAccelerator::Config::SpatialLayer& l,
    const VideoEncodeAccelerator::Config::SpatialLayer& r);
MEDIA_EXPORT bool operator==(const VideoEncodeAccelerator::Config& l,
                             const VideoEncodeAccelerator::Config& r);
}  // namespace media

namespace std {

// Specialize std::default_delete so that
// std::unique_ptr<VideoEncodeAccelerator> uses "Destroy()" instead of trying to
// use the destructor.
template <>
struct MEDIA_EXPORT default_delete<media::VideoEncodeAccelerator> {
  void operator()(media::VideoEncodeAccelerator* vea) const;
};

}  // namespace std

#endif  // MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_