capture/video/chromeos/camera_app_device_impl.cc

// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/capture/video/chromeos/camera_app_device_impl.h"

#include <algorithm>
#include <cmath>

#include "base/task/bind_post_task.h"
#include "base/task/single_thread_task_runner.h"
#include "base/time/time.h"
#include "chromeos/ash/components/mojo_service_manager/connection.h"
#include "gpu/ipc/common/gpu_memory_buffer_impl.h"
#include "media/capture/video/chromeos/camera_app_device_bridge_impl.h"
#include "media/capture/video/chromeos/camera_device_context.h"
#include "media/capture/video/chromeos/camera_metadata_utils.h"
#include "media/capture/video/chromeos/mojom/document_scanner.mojom.h"
#include "mojo/public/cpp/bindings/remote.h"
#include "third_party/cros_system_api/mojo/service_constants.h"
#include "third_party/libyuv/include/libyuv.h"

namespace media {

namespace {

constexpr int kDetectionWidth = 256;
constexpr int kDetectionHeight = 256;

}  // namespace

class CameraAppDeviceImpl::DocumentScanner {
 public:
  using DetectCornersFromNV12ImageCallback =
      base::OnceCallback<void(bool success,
                              const std::vector<gfx::PointF>& results)>;

  DocumentScanner() {
    if (!ash::mojo_service_manager::IsServiceManagerBound()) {
      return;
    }
    ash::mojo_service_manager::GetServiceManagerProxy()->Request(
        chromeos::mojo_services::kCrosDocumentScanner, std::nullopt,
        document_scanner_remote_.BindNewPipeAndPassReceiver().PassPipe());
  }

  DocumentScanner(const DocumentScanner&) = delete;
  DocumentScanner& operator=(const DocumentScanner&) = delete;

  ~DocumentScanner() = default;

  void DetectCornersFromNV12Image(base::ReadOnlySharedMemoryRegion nv12_image,
                                  DetectCornersFromNV12ImageCallback callback) {
    document_scanner_remote_->DetectCornersFromNV12Image(
        std::move(nv12_image),
        base::BindOnce(
            [](DetectCornersFromNV12ImageCallback callback,
               cros::mojom::DetectCornersResultPtr detect_result) {
              std::move(callback).Run(detect_result->success,
                                      std::move(detect_result->corners));
            },
            std::move(callback)));
  }

 private:
  mojo::Remote<cros::mojom::CrosDocumentScanner> document_scanner_remote_;
};

// static
int CameraAppDeviceImpl::GetPortraitSegResultCode(
    const cros::mojom::CameraMetadataPtr* metadata) {
  auto portrait_mode_segmentation_result = GetMetadataEntryAsSpan<uint8_t>(
      *metadata, static_cast<cros::mojom::CameraMetadataTag>(
                     kPortraitModeSegmentationResultVendorKey));
  CHECK(!portrait_mode_segmentation_result.empty());
  return static_cast<int>(portrait_mode_segmentation_result[0]);
}

CameraAppDeviceImpl::CameraAppDeviceImpl(
    const std::string& device_id,
    scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner)
    : device_id_(device_id),
      allow_new_ipc_weak_ptrs_(true),
      capture_intent_(cros::mojom::CaptureIntent::kDefault),
      camera_device_context_(nullptr),
      document_scanner_(ui_task_runner) {}

CameraAppDeviceImpl::~CameraAppDeviceImpl() {
  // If the instance is bound, then this instance should only be destroyed when
  // the mojo connection is dropped, which also happens on the mojo thread.
  DCHECK(!mojo_task_runner_ || mojo_task_runner_->BelongsToCurrentThread());

  // All the weak pointers of |weak_ptr_factory_| should be invalidated on
  // camera device IPC thread before destroying CameraAppDeviceImpl.
  DCHECK(!weak_ptr_factory_.HasWeakPtrs());
}

void CameraAppDeviceImpl::BindReceiver(
    mojo::PendingReceiver<cros::mojom::CameraAppDevice> receiver) {
  mojo_task_runner_ = base::SingleThreadTaskRunner::GetCurrentDefault();
  receivers_.Add(this, std::move(receiver));
  receivers_.set_disconnect_handler(
      base::BindRepeating(&CameraAppDeviceImpl::OnMojoConnectionError,
                          weak_ptr_factory_for_mojo_.GetWeakPtr()));
}

base::WeakPtr<CameraAppDeviceImpl> CameraAppDeviceImpl::GetWeakPtr() {
  return allow_new_ipc_weak_ptrs_ ? weak_ptr_factory_.GetWeakPtr() : nullptr;
}

void CameraAppDeviceImpl::ResetOnDeviceIpcThread(base::OnceClosure callback,
                                                 bool should_disable_new_ptrs) {
  if (should_disable_new_ptrs) {
    allow_new_ipc_weak_ptrs_ = false;
  }
  weak_ptr_factory_.InvalidateWeakPtrs();
  std::move(callback).Run();
}

std::optional<gfx::Range> CameraAppDeviceImpl::GetFpsRange() {
  base::AutoLock lock(fps_ranges_lock_);

  return specified_fps_range_;
}

gfx::Size CameraAppDeviceImpl::GetStillCaptureResolution() {
  base::AutoLock lock(still_capture_resolution_lock_);

  return still_capture_resolution_;
}

cros::mojom::CaptureIntent CameraAppDeviceImpl::GetCaptureIntent() {
  base::AutoLock lock(capture_intent_lock_);
  return capture_intent_;
}

void CameraAppDeviceImpl::OnResultMetadataAvailable(
    const cros::mojom::CameraMetadataPtr& metadata,
    cros::mojom::StreamType streamType) {
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr(), metadata.Clone(),
                     streamType));
}

void CameraAppDeviceImpl::OnShutterDone() {
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr()));
}

void CameraAppDeviceImpl::OnCameraInfoUpdated(
    cros::mojom::CameraInfoPtr camera_info) {
  base::AutoLock lock(camera_info_lock_);
  camera_info_ = std::move(camera_info);

  if (!mojo_task_runner_) {
    return;
  }
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::NotifyCameraInfoUpdatedOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr()));
}

void CameraAppDeviceImpl::SetCameraDeviceContext(
    CameraDeviceContext* camera_device_context) {
  base::AutoLock lock(camera_device_context_lock_);
  camera_device_context_ = camera_device_context;
}

void CameraAppDeviceImpl::MaybeDetectDocumentCorners(
    std::unique_ptr<gpu::GpuMemoryBufferImpl> gmb,
    VideoRotation rotation) {
  {
    base::AutoLock lock(document_corners_observers_lock_);
    if (document_corners_observers_.empty()) {
      return;
    }
  }
  mojo_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread,
                     weak_ptr_factory_for_mojo_.GetWeakPtr(), std::move(gmb),
                     rotation));
}

bool CameraAppDeviceImpl::IsMultipleStreamsEnabled() {
  base::AutoLock lock(multi_stream_lock_);
  return multi_stream_enabled_;
}

void CameraAppDeviceImpl::TakePortraitModePhoto(
    mojo::PendingRemote<cros::mojom::StillCaptureResultObserver> observer,
    TakePortraitModePhotoCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(portrait_mode_callbacks_lock_);
  portrait_mode_observers_.reset();
  portrait_mode_observers_.Bind(std::move(observer));
  take_portrait_photo_callbacks_.reset();

  // Create two callbacks that will notify the client when the result is
  // returned. The `normal_photo_callback` is for the normal photo, and
  // `portrait_photo_callback` is for the portrait photo.
  PortraitModeCallbacks take_portrait_photo_callbacks;
  take_portrait_photo_callbacks.normal_photo_callback =
      base::BindPostTaskToCurrentDefault(
          base::BindOnce(&CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread,
                         weak_ptr_factory_for_mojo_.GetWeakPtr(),
                         cros::mojom::Effect::kNoEffect));
  take_portrait_photo_callbacks.portrait_photo_callback =
      base::BindPostTaskToCurrentDefault(
          base::BindOnce(&CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread,
                         weak_ptr_factory_for_mojo_.GetWeakPtr(),
                         cros::mojom::Effect::kPortraitMode));
  take_portrait_photo_callbacks_ = std::move(take_portrait_photo_callbacks);

  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetFpsRange(const gfx::Range& fps_range,
                                      SetFpsRangeCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  const int entry_length = 2;

  base::AutoLock camera_info_lock(camera_info_lock_);
  if (!camera_info_) {
    LOG(ERROR) << "Camera info is still not available at this moment";
    std::move(callback).Run(false);
    return;
  }
  auto& static_metadata = camera_info_->static_camera_characteristics;
  auto available_fps_range_entries = GetMetadataEntryAsSpan<int32_t>(
      static_metadata, cros::mojom::CameraMetadataTag::
                           ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES);
  DCHECK(available_fps_range_entries.size() % entry_length == 0);

  bool is_valid = false;
  int min_fps = static_cast<int>(fps_range.GetMin());
  int max_fps = static_cast<int>(fps_range.GetMax());
  for (size_t i = 0; i < available_fps_range_entries.size();
       i += entry_length) {
    if (available_fps_range_entries[i] == min_fps &&
        available_fps_range_entries[i + 1] == max_fps) {
      is_valid = true;
      break;
    }
  }

  base::AutoLock lock(fps_ranges_lock_);

  if (is_valid) {
    specified_fps_range_ = fps_range;
  } else {
    specified_fps_range_ = {};
  }
  std::move(callback).Run(is_valid);
}

void CameraAppDeviceImpl::SetStillCaptureResolution(
    const gfx::Size& resolution,
    SetStillCaptureResolutionCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(still_capture_resolution_lock_);
  still_capture_resolution_ = resolution;
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetCaptureIntent(
    cros::mojom::CaptureIntent capture_intent,
    SetCaptureIntentCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  {
    base::AutoLock lock(capture_intent_lock_);
    capture_intent_ = capture_intent;
  }
  // Reset fps range for VCD to determine it if not explicitly set by app.
  {
    base::AutoLock lock(fps_ranges_lock_);
    specified_fps_range_ = {};
  }
  std::move(callback).Run();
}

void CameraAppDeviceImpl::AddResultMetadataObserver(
    mojo::PendingRemote<cros::mojom::ResultMetadataObserver> observer,
    cros::mojom::StreamType stream_type,
    AddResultMetadataObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  stream_to_metadata_observers_map_[stream_type].Add(std::move(observer));
  std::move(callback).Run();
}

void CameraAppDeviceImpl::AddCameraEventObserver(
    mojo::PendingRemote<cros::mojom::CameraEventObserver> observer,
    AddCameraEventObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  camera_event_observers_.Add(std::move(observer));
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetCameraFrameRotationEnabledAtSource(
    bool is_enabled,
    SetCameraFrameRotationEnabledAtSourceCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  bool is_success = false;
  {
    base::AutoLock lock(camera_device_context_lock_);
    if (camera_device_context_) {
      camera_device_context_->SetCameraFrameRotationEnabledAtSource(is_enabled);
      is_success = true;
    }
  }
  std::move(callback).Run(is_success);
}

void CameraAppDeviceImpl::GetCameraFrameRotation(
    GetCameraFrameRotationCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  uint32_t rotation = 0;
  {
    base::AutoLock lock(camera_device_context_lock_);
    if (camera_device_context_ &&
        !camera_device_context_->IsCameraFrameRotationEnabledAtSource()) {
      // The camera rotation value can only be [0, 90, 180, 270].
      rotation = static_cast<uint32_t>(
          camera_device_context_->GetCameraFrameRotation());
    }
  }
  std::move(callback).Run(rotation);
}

void CameraAppDeviceImpl::RegisterDocumentCornersObserver(
    mojo::PendingRemote<cros::mojom::DocumentCornersObserver> observer,
    RegisterDocumentCornersObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(document_corners_observers_lock_);
  document_corners_observers_.Add(std::move(observer));
  std::move(callback).Run();
}

void CameraAppDeviceImpl::SetMultipleStreamsEnabled(
    bool enabled,
    SetMultipleStreamsEnabledCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(multi_stream_lock_);
  multi_stream_enabled_ = enabled;
  std::move(callback).Run();
}

void CameraAppDeviceImpl::RegisterCameraInfoObserver(
    mojo::PendingRemote<cros::mojom::CameraInfoObserver> observer,
    RegisterCameraInfoObserverCallback callback) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  camera_info_observers_.Add(std::move(observer));
  std::move(callback).Run();

  NotifyCameraInfoUpdatedOnMojoThread();
}

void CameraAppDeviceImpl::OnMojoConnectionError() {
  CameraAppDeviceBridgeImpl::GetInstance()->OnDeviceMojoDisconnected(
      device_id_);
}

bool CameraAppDeviceImpl::IsCloseToPreviousDetectionRequest() {
  return document_detection_timer_ &&
         document_detection_timer_->Elapsed().InMilliseconds() < 300;
}

void CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread(
    std::unique_ptr<gpu::GpuMemoryBufferImpl> image,
    VideoRotation rotation) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  if (IsCloseToPreviousDetectionRequest() ||
      has_ongoing_document_detection_task_) {
    return;
  }

  DCHECK(image);
  if (!image->Map()) {
    LOG(ERROR) << "Failed to map frame buffer";
    return;
  }
  auto frame_size = image->GetSize();
  int width = frame_size.width();
  int height = frame_size.height();

  base::MappedReadOnlyRegion memory = base::ReadOnlySharedMemoryRegion::Create(
      kDetectionWidth * kDetectionHeight * 3 / 2);
  if (!memory.IsValid()) {
    LOG(ERROR) << "Failed to allocate shared memory";
    return;
  }
  auto* y_data = memory.mapping.GetMemoryAs<uint8_t>();
  auto* uv_data = y_data + kDetectionWidth * kDetectionHeight;

  int status = libyuv::NV12Scale(
      static_cast<uint8_t*>(image->memory(0)), image->stride(0),
      static_cast<uint8_t*>(image->memory(1)), image->stride(1), width, height,
      y_data, kDetectionWidth, uv_data, kDetectionWidth, kDetectionWidth,
      kDetectionHeight, libyuv::FilterMode::kFilterNone);
  image->Unmap();
  if (status != 0) {
    LOG(ERROR) << "Failed to scale buffer";
    return;
  }

  has_ongoing_document_detection_task_ = true;
  document_detection_timer_ = std::make_unique<base::ElapsedTimer>();

  document_scanner_.AsyncCall(&DocumentScanner::DetectCornersFromNV12Image)
      .WithArgs(std::move(memory.region),
                base::BindPostTaskToCurrentDefault(base::BindOnce(
                    &CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread,
                    weak_ptr_factory_for_mojo_.GetWeakPtr(), rotation)));
}

void CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread(
    VideoRotation rotation,
    bool success,
    const std::vector<gfx::PointF>& corners) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  has_ongoing_document_detection_task_ = false;
  if (!success) {
    LOG(ERROR) << "Failed to detect document corners";
    return;
  }

  // Rotate a point in coordination space {x: [0.0, 1.0], y: [0.0, 1.0]} with
  // anchor point {x: 0.5, y: 0.5}.
  auto rotate_corner = [&](const gfx::PointF& corner) -> gfx::PointF {
    float x = std::clamp(corner.x(), 0.0f, 1.0f);
    float y = std::clamp(corner.y(), 0.0f, 1.0f);

    switch (rotation) {
      case VIDEO_ROTATION_0:
        return {x, y};
      case VIDEO_ROTATION_90:
        return {1.0f - y, x};
      case VIDEO_ROTATION_180:
        return {1.0f - x, 1.0f - y};
      case VIDEO_ROTATION_270:
        return {y, 1.0f - x};
      default:
        NOTREACHED_IN_MIGRATION();
    }
  };

  std::vector<gfx::PointF> rotated_corners;
  for (auto& corner : corners) {
    rotated_corners.push_back(rotate_corner(corner));
  }

  base::AutoLock lock(document_corners_observers_lock_);
  for (auto& observer : document_corners_observers_) {
    observer->OnDocumentCornersUpdated(rotated_corners);
  }
}

void CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread(
    cros::mojom::Effect effect,
    const int32_t status,
    media::mojom::BlobPtr blob) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  portrait_mode_observers_->OnStillCaptureDone(effect, status, std::move(blob));
}

void CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread() {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  for (auto& observer : camera_event_observers_) {
    observer->OnShutterDone();
  }
}

void CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread(
    cros::mojom::CameraMetadataPtr metadata,
    cros::mojom::StreamType streamType) {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  auto& metadata_observers = stream_to_metadata_observers_map_[streamType];
  for (auto& observer : metadata_observers) {
    observer->OnMetadataAvailable(metadata.Clone());
  }
}

void CameraAppDeviceImpl::NotifyCameraInfoUpdatedOnMojoThread() {
  DCHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(camera_info_lock_);
  if (!camera_info_) {
    return;
  }
  for (auto& observer : camera_info_observers_) {
    observer->OnCameraInfoUpdated(camera_info_.Clone());
  }
}

std::optional<PortraitModeCallbacks>
CameraAppDeviceImpl::ConsumePortraitModeCallbacks() {
  base::AutoLock lock(portrait_mode_callbacks_lock_);
  std::optional<PortraitModeCallbacks> callbacks;
  if (take_portrait_photo_callbacks_.has_value()) {
    callbacks = std::move(take_portrait_photo_callbacks_);
    take_portrait_photo_callbacks_.reset();
  }
  return callbacks;
}

void CameraAppDeviceImpl::SetCropRegion(const gfx::Rect& crop_region,
                                        SetCropRegionCallback callback) {
  CHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(crop_region_lock_);
  crop_region_ = {
      crop_region.x(),
      crop_region.y(),
      crop_region.width(),
      crop_region.height(),
  };

  std::move(callback).Run();
}

void CameraAppDeviceImpl::ResetCropRegion(ResetCropRegionCallback callback) {
  CHECK(mojo_task_runner_->BelongsToCurrentThread());

  base::AutoLock lock(crop_region_lock_);
  crop_region_.reset();

  std::move(callback).Run();
}

std::optional<std::vector<int32_t>> CameraAppDeviceImpl::GetCropRegion() {
  base::AutoLock lock(crop_region_lock_);
  return crop_region_;
}

}  // namespace media