media/gpu/chromeos/image_processor_factory.cc

// Copyright 2018 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/gpu/chromeos/image_processor_factory.h"

#include <stddef.h>

#include "base/containers/contains.h"
#include "base/feature_list.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/memory/scoped_refptr.h"
#include "base/task/sequenced_task_runner.h"
#include "media/base/media_switches.h"
#include "media/base/video_types.h"
#include "media/gpu/buildflags.h"
#include "media/gpu/chromeos/gl_image_processor_backend.h"
#include "media/gpu/chromeos/image_processor.h"
#include "media/gpu/chromeos/libyuv_image_processor_backend.h"
#include "media/gpu/macros.h"

#if BUILDFLAG(USE_VAAPI)
#include "media/gpu/vaapi/vaapi_image_processor_backend.h"
#include "media/gpu/vaapi/vaapi_wrapper.h"
#elif BUILDFLAG(USE_V4L2_CODEC)
#include "media/gpu/v4l2/v4l2_device.h"
#include "media/gpu/v4l2/v4l2_image_processor_backend.h"
#include "media/gpu/v4l2/v4l2_vda_helpers.h"
#endif

namespace media {

namespace {

using PixelLayoutCandidate = ImageProcessor::PixelLayoutCandidate;

#if BUILDFLAG(USE_VAAPI)
std::unique_ptr<ImageProcessor> CreateVaapiImageProcessorWithInputCandidates(
    const std::vector<PixelLayoutCandidate>& input_candidates,
    const gfx::Rect& input_visible_rect,
    const gfx::Size& output_size,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    ImageProcessorFactory::PickFormatCB out_format_picker,
    ImageProcessor::ErrorCB error_cb) {
  std::vector<Fourcc> vpp_supported_formats =
      VaapiWrapper::GetVppSupportedFormats();
  absl::optional<PixelLayoutCandidate> chosen_input_candidate;
  for (const auto& input_candidate : input_candidates) {
    if (base::Contains(vpp_supported_formats, input_candidate.fourcc) &&
        VaapiWrapper::IsVppResolutionAllowed(input_candidate.size)) {
      chosen_input_candidate = input_candidate;
      break;
    }
  }
  if (!chosen_input_candidate)
    return nullptr;

  // Note that we pick the first input candidate as the preferred output format.
  // The reason is that in practice, the VaapiVideoDecoder will make
  // |input_candidates| either {NV12} or {P010} depending on the bitdepth. So
  // choosing the first (and only) element will keep the bitdepth of the frame
  // which is needed to display HDR content.
  auto chosen_output_format = out_format_picker.Run(
      /*candidates=*/vpp_supported_formats, input_candidates[0].fourcc);
  if (!chosen_output_format)
    return nullptr;

  // Note: the VaapiImageProcessorBackend doesn't use the ColorPlaneLayouts in
  // the PortConfigs, so we just pass an empty list of plane layouts.
  ImageProcessor::PortConfig input_config(
      chosen_input_candidate->fourcc, chosen_input_candidate->size,
      /*planes=*/{}, input_visible_rect,
      {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  ImageProcessor::PortConfig output_config(
      /*fourcc=*/*chosen_output_format, /*size=*/output_size, /*planes=*/{},
      /*visible_rect=*/gfx::Rect(output_size),
      {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  return ImageProcessor::Create(
      base::BindRepeating(&VaapiImageProcessorBackend::Create), input_config,
      output_config, ImageProcessor::OutputMode::IMPORT, VIDEO_ROTATION_0,
      std::move(error_cb), std::move(client_task_runner));
}

#elif BUILDFLAG(USE_V4L2_CODEC)

std::unique_ptr<ImageProcessor> CreateV4L2ImageProcessorWithInputCandidates(
    const std::vector<PixelLayoutCandidate>& input_candidates,
    const gfx::Rect& visible_rect,
    size_t num_buffers,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    ImageProcessorFactory::PickFormatCB out_format_picker,
    ImageProcessor::ErrorCB error_cb) {
  // Pick a renderable output format, and try each available input format.
  // TODO(akahuang): let |out_format_picker| return a list of supported output
  // formats, and try all combination of input/output format, if any platform
  // fails to create ImageProcessor via current approach.
  const std::vector<uint32_t> supported_output_formats =
      V4L2ImageProcessorBackend::GetSupportedOutputFormats();
  std::vector<Fourcc> supported_fourccs;
  for (const auto& format : supported_output_formats) {
    const auto fourcc = Fourcc::FromV4L2PixFmt(format);
    if (fourcc.has_value())
      supported_fourccs.push_back(*fourcc);
  }

  const auto output_fourcc = out_format_picker.Run(
      /*candidates=*/supported_fourccs, /*preferred_fourcc=*/absl::nullopt);
  if (!output_fourcc) {
#if DCHECK_IS_ON()
    std::string output_fourccs_string;
    for (const auto fourcc : supported_fourccs) {
      output_fourccs_string += fourcc.ToString();
      output_fourccs_string += " ";
    }
    DVLOGF(1) << "None of " << output_fourccs_string << "formats is supported.";
#endif
    return nullptr;
  }

  const auto supported_input_pixfmts =
      V4L2ImageProcessorBackend::GetSupportedInputFormats();
  for (const auto& input_candidate : input_candidates) {
    const Fourcc input_fourcc = input_candidate.fourcc;
    const gfx::Size& input_size = input_candidate.size;

    if (!base::Contains(supported_input_pixfmts, input_fourcc.ToV4L2PixFmt()))
      continue;

    // Ideally the ImageProcessor would be able to scale and crop |input_size|
    // to the |visible_rect| area of |output_size|. TryOutputFormat() below is
    // called to verify that a given combination of fourcc values and input/
    // output sizes are indeed supported (the driver can potentially return a
    // different supported |output_size|). Some Image Processors(e.g. MTK8183)
    // are not able to crop/scale correctly -- but TryOutputFormat()  doesn't
    // return a "corrected" |output_size|. To avoid troubles (and, in general,
    // low performance), we set |output_size| to be equal to |input_size|; the
    // |visible_rect| will carry the information of exactly what part of the
    // video frame contains valid pixels, and the media/compositor pipeline
    // will take care of it.
    gfx::Size output_size = input_size;
    size_t num_planes = 0;
    if (!V4L2ImageProcessorBackend::TryOutputFormat(
            input_fourcc.ToV4L2PixFmt(), output_fourcc->ToV4L2PixFmt(),
            input_size, &output_size, &num_planes)) {
      VLOGF(2) << "Failed to get output size and plane count of IP";
      continue;
    }
    // This is very restrictive because it assumes the IP has the same alignment
    // criteria as the video decoder that will produce the input video frames.
    // In practice, this applies to all Image Processors, i.e. Mediatek devices.
    DCHECK_EQ(input_size, output_size);
    // |visible_rect| applies equally to both |input_size| and |output_size|.
    DCHECK(gfx::Rect(output_size).Contains(visible_rect));

    return v4l2_vda_helpers::CreateImageProcessor(
        input_fourcc, *output_fourcc, input_size, output_size, visible_rect,
        VideoFrame::StorageType::STORAGE_GPU_MEMORY_BUFFER, num_buffers,
        V4L2Device::Create(), ImageProcessor::OutputMode::IMPORT,
        std::move(client_task_runner), std::move(error_cb));
  }
  return nullptr;
}

std::unique_ptr<ImageProcessor> CreateLibYUVImageProcessorWithInputCandidates(
    const std::vector<PixelLayoutCandidate>& input_candidates,
    const gfx::Rect& input_visible_rect,
    const gfx::Size& output_size,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    ImageProcessorFactory::PickFormatCB out_format_picker,
    ImageProcessor::ErrorCB error_cb) {
  if (input_candidates.size() != 1)
    return nullptr;

  if (input_candidates[0].fourcc != Fourcc(Fourcc::MM21))
    return nullptr;

  std::vector<Fourcc> supported_output_formats =
      LibYUVImageProcessorBackend::GetSupportedOutputFormats(
          Fourcc(Fourcc::MM21));
  auto output_format =
      out_format_picker.Run(supported_output_formats, Fourcc(Fourcc::NV12));

  if (!output_format)
    return nullptr;

  ImageProcessor::PortConfig input_config(
      Fourcc(Fourcc::MM21), input_candidates[0].size, /*planes=*/{},
      input_visible_rect, {VideoFrame::STORAGE_DMABUFS});
  ImageProcessor::PortConfig output_config(
      *output_format, output_size, /*planes=*/{}, gfx::Rect(output_size),
      {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  return ImageProcessor::Create(
      base::BindRepeating(&LibYUVImageProcessorBackend::Create), input_config,
      output_config, ImageProcessor::OutputMode::IMPORT, VIDEO_ROTATION_0,
      std::move(error_cb), std::move(client_task_runner));
}

std::unique_ptr<ImageProcessor> CreateGLImageProcessorWithInputCandidates(
    const std::vector<PixelLayoutCandidate>& input_candidates,
    const gfx::Rect& input_visible_rect,
    const gfx::Size& output_size,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    ImageProcessorFactory::PickFormatCB out_format_picker,
    ImageProcessor::ErrorCB error_cb) {
  if (input_candidates.size() != 1)
    return nullptr;

  if (input_candidates[0].fourcc != Fourcc(Fourcc::MM21))
    return nullptr;

  ImageProcessor::PortConfig input_config(
      Fourcc(Fourcc::MM21), input_candidates[0].size, /*planes=*/{},
      input_visible_rect, {VideoFrame::STORAGE_DMABUFS});
  ImageProcessor::PortConfig output_config(
      Fourcc(Fourcc::NV12), output_size, /*planes=*/{}, gfx::Rect(output_size),
      {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});

  if (!GLImageProcessorBackend::IsSupported(input_config, output_config,
                                            VIDEO_ROTATION_0)) {
    return nullptr;
  }

  return ImageProcessor::Create(
      base::BindRepeating(&GLImageProcessorBackend::Create), input_config,
      output_config, ImageProcessor::OutputMode::IMPORT, VIDEO_ROTATION_0,
      std::move(error_cb), std::move(client_task_runner));
}
#endif

}  // namespace

// static
std::unique_ptr<ImageProcessor> ImageProcessorFactory::Create(
    const ImageProcessor::PortConfig& input_config,
    const ImageProcessor::PortConfig& output_config,
    ImageProcessor::OutputMode output_mode,
    size_t num_buffers,
    VideoRotation relative_rotation,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    ImageProcessor::ErrorCB error_cb) {
  std::vector<ImageProcessor::CreateBackendCB> create_funcs;
#if BUILDFLAG(USE_VAAPI)
  create_funcs.push_back(
      base::BindRepeating(&VaapiImageProcessorBackend::Create));
#elif BUILDFLAG(USE_V4L2_CODEC)
  create_funcs.push_back(base::BindRepeating(
      &V4L2ImageProcessorBackend::Create, V4L2Device::Create(), num_buffers));
#endif
  create_funcs.push_back(
      base::BindRepeating(&LibYUVImageProcessorBackend::Create));

  std::unique_ptr<ImageProcessor> image_processor;
  for (auto& create_func : create_funcs) {
    image_processor = ImageProcessor::Create(
        std::move(create_func), input_config, output_config, output_mode,
        relative_rotation, error_cb, client_task_runner);
    if (image_processor)
      return image_processor;
  }
  return nullptr;
}

// static
std::unique_ptr<ImageProcessor>
ImageProcessorFactory::CreateWithInputCandidates(
    const std::vector<PixelLayoutCandidate>& input_candidates,
    const gfx::Rect& input_visible_rect,
    const gfx::Size& output_size,
    size_t num_buffers,
    scoped_refptr<base::SequencedTaskRunner> client_task_runner,
    PickFormatCB out_format_picker,
    ImageProcessor::ErrorCB error_cb) {
#if BUILDFLAG(USE_VAAPI)
  auto processor = CreateVaapiImageProcessorWithInputCandidates(
      input_candidates, input_visible_rect, output_size, client_task_runner,
      out_format_picker, error_cb);
  if (processor)
    return processor;
#elif BUILDFLAG(USE_V4L2_CODEC)
  if (base::FeatureList::IsEnabled(media::kPreferGLImageProcessor)) {
    auto processor = CreateGLImageProcessorWithInputCandidates(
        input_candidates, input_visible_rect, output_size, client_task_runner,
        out_format_picker, error_cb);
    if (processor)
      return processor;
  }

    auto processor = CreateLibYUVImageProcessorWithInputCandidates(
        input_candidates, input_visible_rect, output_size, client_task_runner,
        out_format_picker, error_cb);
    if (processor)
      return processor;

    processor = CreateV4L2ImageProcessorWithInputCandidates(
        input_candidates, input_visible_rect, num_buffers, client_task_runner,
        out_format_picker, error_cb);
    if (processor)
      return processor;

#endif

  // TODO(crbug.com/1004727): Implement LibYUVImageProcessorBackend. When doing
  // so, we must keep in mind that it might not be desirable to fallback to
  // libyuv if the hardware image processor fails (e.g., in the case of
  // protected content).
  return nullptr;
}

}  // namespace media