media/gpu/mac/video_toolbox_vp9_accelerator.cc

// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/gpu/mac/video_toolbox_vp9_accelerator.h"

#include <utility>
#include <vector>

#include "base/notreached.h"
#include "media/base/media_log.h"
#include "media/base/video_types.h"
#include "media/gpu/mac/vt_config_util.h"

namespace media {

VideoToolboxVP9Accelerator::VideoToolboxVP9Accelerator(
    std::unique_ptr<MediaLog> media_log,
    std::optional<gfx::HDRMetadata> hdr_metadata,
    DecodeCB decode_cb,
    OutputCB output_cb)
    : media_log_(std::move(media_log)),
      hdr_metadata_(std::move(hdr_metadata)),
      decode_cb_(std::move(decode_cb)),
      output_cb_(std::move(output_cb)) {
  DVLOG(1) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}

VideoToolboxVP9Accelerator::~VideoToolboxVP9Accelerator() {
  DVLOG(1) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}

scoped_refptr<VP9Picture> VideoToolboxVP9Accelerator::CreateVP9Picture() {
  DVLOG(4) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  return base::MakeRefCounted<VP9Picture>();
}

VideoToolboxVP9Accelerator::Status VideoToolboxVP9Accelerator::SubmitDecode(
    scoped_refptr<VP9Picture> pic,
    const Vp9SegmentationParams& segm_params,
    const Vp9LoopFilterParams& lf_params,
    const Vp9ReferenceFrameVector& reference_frames) {
  DVLOG(3) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  // `show_existing_frame` pictures go directly to OutputPicture().
  DCHECK(!pic->frame_hdr->show_existing_frame);

  if (!ProcessFrame(std::move(pic))) {
    return Status::kFail;
  }

  return Status::kOk;
}

bool VideoToolboxVP9Accelerator::OutputPicture(scoped_refptr<VP9Picture> pic) {
  DVLOG(3) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  DCHECK(pic->frame_hdr->show_frame || pic->frame_hdr->show_existing_frame);

  // `show_existing_frame` frames are not passed to SubmitDecode(), so handle
  // them as new frames.
  if (pic->frame_hdr->show_existing_frame) {
    if (!ProcessFrame(pic)) {
      return false;
    }
  }

  output_cb_.Run(std::move(pic));

  return true;
}

bool VideoToolboxVP9Accelerator::ProcessFrame(scoped_refptr<VP9Picture> pic) {
  DVLOG(4) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

  // Update `active_format_`.
  bool format_changed = false;
  if (!ProcessFormat(pic, &format_changed)) {
    return false;
  }

  if (format_changed && frame_data_) {
    // TODO(crbug.com/40227557): Consider dropping existing frame data. Doing so
    // probably requires handling output callbacks ourselves, so that we don't
    // have to figure out which ones are duplicates.
    // TODO(crbug.com/40227557): Add Reset() to VP9Accelerator for resetting
    // superframe state after Flush().
    MEDIA_LOG(WARNING, media_log_.get()) << "Format change inside superframe";
  }

  // If this is the first picture in the current superframe, create a buffer.
  if (!frame_data_) {
    OSStatus status = CMBlockBufferCreateEmpty(
        /*structureAllocator=*/kCFAllocatorDefault,
        /*subBlockCapacity=*/0,
        /*flags=*/0, frame_data_.InitializeInto());
    if (status != noErr) {
      OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
          << "CMBlockBufferCreateWithMemoryBlock()";
      return false;
    }
  }

  // Append this picture to the current superframe.
  AppendData(frame_data_.get(), pic->frame_hdr->data.data(),
             pic->frame_hdr->data.size());
  frame_sizes_.push_back(pic->frame_hdr->data.size());

  // If this is an output picture, submit the current superframe for decoding.
  if (pic->frame_hdr->show_frame || pic->frame_hdr->show_existing_frame) {
    if (!SubmitFrames(std::move(pic))) {
      return false;
    }
  }

  return true;
}

bool VideoToolboxVP9Accelerator::ProcessFormat(scoped_refptr<VP9Picture> pic,
                                               bool* format_changed) {
  DVLOG(4) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

  if (pic->frame_hdr->show_existing_frame) {
    *format_changed = false;
    return true;
  }

  // TODO(crbug.com/40227557): Consider merging with CreateFormatExtensions() to
  // avoid converting back and forth.
  VideoColorSpace color_space = pic->get_colorspace();

  VideoCodecProfile profile;
  switch (pic->frame_hdr->profile) {
    case 0:
      profile = VP9PROFILE_PROFILE0;
      break;
    case 1:
      profile = VP9PROFILE_PROFILE1;
      break;
    case 2:
      profile = VP9PROFILE_PROFILE2;
      break;
    case 3:
      profile = VP9PROFILE_PROFILE3;
      break;
    default:
      profile = VIDEO_CODEC_PROFILE_UNKNOWN;
      break;
  }

  std::optional<gfx::HDRMetadata> hdr_metadata = pic->hdr_metadata();
  if (!hdr_metadata) {
    hdr_metadata = hdr_metadata_;
  }

  gfx::Size coded_size(static_cast<int>(pic->frame_hdr->frame_width),
                       static_cast<int>(pic->frame_hdr->frame_height));
  if (coded_size.IsEmpty()) {
    DLOG(ERROR) << "Invalid frame size";
    return false;
  }

  // If the parameters have changed, generate a new format.
  if (color_space != active_color_space_ || profile != active_profile_ ||
      hdr_metadata != active_hdr_metadata_ ||
      coded_size != active_coded_size_) {
    active_format_.reset();

    base::apple::ScopedCFTypeRef<CFDictionaryRef> format_config =
        CreateFormatExtensions(kCMVideoCodecType_VP9, profile,
                               pic->frame_hdr->bit_depth, color_space,
                               hdr_metadata, std::nullopt);
    if (!format_config) {
      MEDIA_LOG(ERROR, media_log_.get())
          << "Failed to create format extensions";
      return false;
    }

    base::apple::ScopedCFTypeRef<CMFormatDescriptionRef> format;
    OSStatus status = CMVideoFormatDescriptionCreate(
        kCFAllocatorDefault, kCMVideoCodecType_VP9, coded_size.width(),
        coded_size.height(), format_config.get(),
        active_format_.InitializeInto());
    if (status != noErr) {
      OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
          << "CMVideoFormatDescriptionCreate()";
      return false;
    }

    active_color_space_ = color_space;
    active_profile_ = profile;
    active_hdr_metadata_ = hdr_metadata;
    active_coded_size_ = coded_size;

    session_metadata_ = VideoToolboxDecompressionSessionMetadata{
        /*allow_software_decoding=*/false,
        /*bit_depth=*/pic->frame_hdr->bit_depth,
        /*chroma_sampling=*/VideoChromaSampling::k420,
        /*has_alpha=*/false,
        /*visible_rect=*/pic->visible_rect()};

    *format_changed = true;
  } else {
    *format_changed = false;
  }

  return true;
}

bool VideoToolboxVP9Accelerator::SubmitFrames(
    scoped_refptr<VP9Picture> output_pic) {
  DVLOG(4) << __func__;
  DCHECK(frame_sizes_.size() >= 1);
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

  // Take the current superframe.
  base::apple::ScopedCFTypeRef<CMBlockBufferRef> frame_data;
  std::vector<size_t> frame_sizes;
  frame_data.swap(frame_data_);
  frame_sizes.swap(frame_sizes_);

  // If there is only one frame, it can be submitted directly. Otherwise append
  // a superframe trailer. See Annex B of the VP9 specification.
  if (frame_sizes.size() > 1) {
    if (frame_sizes_.size() > 8) {
      MEDIA_LOG(ERROR, media_log_.get()) << "Too many frames in superframe";
      return false;
    }

    constexpr uint8_t kSuperframeMarker = 0b110;
    constexpr size_t kBytesPerFrameSize = 4;
    constexpr size_t kSuperframeHeaderSize = 1;

    uint8_t header = (kSuperframeMarker << 5) |
                     ((kBytesPerFrameSize - 1) << 3) | (frame_sizes.size() - 1);

    size_t trailer_size = kSuperframeHeaderSize +
                          kBytesPerFrameSize * frame_sizes.size() +
                          kSuperframeHeaderSize;

    std::vector<uint8_t> trailer;
    trailer.reserve(trailer_size);
    trailer.push_back(header);
    for (size_t frame_size : frame_sizes) {
      trailer.push_back(frame_size & 0xff);
      frame_size >>= 8;
      trailer.push_back(frame_size & 0xff);
      frame_size >>= 8;
      trailer.push_back(frame_size & 0xff);
      frame_size >>= 8;
      trailer.push_back(frame_size & 0xff);
      frame_size >>= 8;
      DCHECK(frame_size == 0);
    }
    trailer.push_back(header);
    DCHECK_EQ(trailer.size(), trailer_size);

    AppendData(frame_data.get(), trailer.data(), trailer.size());
  }

  // Wrap the frame data in a sample.
  base::apple::ScopedCFTypeRef<CMSampleBufferRef> sample;
  size_t size = CMBlockBufferGetDataLength(frame_data.get());
  OSStatus status = CMSampleBufferCreate(
      /*allocator=*/kCFAllocatorDefault,
      /*dataBuffer=*/frame_data.get(),
      /*dataReady=*/true,
      /*makeDataReadyCallback=*/nullptr,
      /*makeDataReadyRefcon=*/nullptr,
      /*formatDescription=*/active_format_.get(),
      /*numSamples=*/1,
      /*numSampleTimingEntries=*/0,
      /*sampleTimingArray=*/nullptr,
      /*numSampleSizeEntries=*/1,
      /*sampleSizeArray=*/&size, sample.InitializeInto());
  if (status != noErr) {
    OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
        << "CMSampleBufferCreate()";
    return false;
  }

  // Submit for decoding.
  decode_cb_.Run(std::move(sample), session_metadata_, std::move(output_pic));
  return true;
}

bool VideoToolboxVP9Accelerator::AppendData(CMBlockBufferRef dest,
                                            const uint8_t* data,
                                            size_t data_size) {
  DVLOG(4) << __func__;
  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

  size_t offset = CMBlockBufferGetDataLength(dest);

  // Describe the required backing memory.
  OSStatus status = CMBlockBufferAppendMemoryBlock(
      /*theBuffer=*/dest,
      /*memoryBlock=*/nullptr,
      /*blockLength=*/data_size,
      /*blockAllocator=*/kCFAllocatorDefault,
      /*customBlockSource=*/nullptr,
      /*offsetToData=*/0,
      /*dataLength=*/data_size,
      /*flags=*/0);
  if (status != noErr) {
    OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
        << "CMBlockBufferAppendMemoryBlock()";
    return false;
  }

  // Actually allocate the backing memory.
  status = CMBlockBufferAssureBlockMemory(dest);
  if (status != noErr) {
    OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
        << "CMBlockBufferAssureBlockMemory()";
    return false;
  }

  // Copy the data.
  status = CMBlockBufferReplaceDataBytes(data, dest, offset, data_size);
  if (status != noErr) {
    OSSTATUS_MEDIA_LOG(ERROR, status, media_log_.get())
        << "CMBlockBufferReplaceDataBytes()";
    return false;
  }

  return true;
}

}  // namespace media