blob: 8c3f8f874a0cf1fdf469e7f1c8f6622929526177 [file] [log] [blame]
// Copyright 2018 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/mirroring/service/video_capture_client.h"
#include "base/functional/bind.h"
#include "base/memory/read_only_shared_memory_region.h"
#include "base/notimplemented.h"
#include "base/task/bind_post_task.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "media/base/video_frame.h"
#include "media/base/video_frame_pool.h"
#include "media/capture/mojom/video_capture_buffer.mojom.h"
#include "media/capture/mojom/video_capture_types.mojom.h"
namespace mirroring {
namespace {
// Required by mojom::VideoCaptureHost interface. Can be any nonzero value.
const base::UnguessableToken& DeviceId() {
// TODO(crbug.com/40252974): Investigate whether there's a better way
// to accomplish this (without using UnguessableToken::Deserialize).
static const base::UnguessableToken device_id(
base::UnguessableToken::Deserialize(1, 1).value());
return device_id;
}
// Required by mojom::VideoCaptureHost interface. Can be any nonzero value.
const base::UnguessableToken& SessionId() {
// TODO(crbug.com/40252974): Investigate whether there's a better way
// to accomplish this (without using UnguessableToken::Deserialize).
static const base::UnguessableToken session_id(
base::UnguessableToken::Deserialize(1, 1).value());
return session_id;
}
} // namespace
VideoCaptureClient::VideoCaptureClient(
const media::VideoCaptureParams& params,
mojo::PendingRemote<media::mojom::VideoCaptureHost> host)
: params_(params), video_capture_host_(std::move(host)) {
DCHECK(video_capture_host_);
}
VideoCaptureClient::~VideoCaptureClient() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
Stop();
}
void VideoCaptureClient::Start(FrameDeliverCallback deliver_callback,
base::OnceClosure error_callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
DCHECK(!deliver_callback.is_null());
frame_deliver_callback_ = std::move(deliver_callback);
error_callback_ = std::move(error_callback);
video_capture_host_->Start(DeviceId(), SessionId(), params_,
receiver_.BindNewPipeAndPassRemote());
}
void VideoCaptureClient::Stop() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
video_capture_host_->Stop(DeviceId());
}
void VideoCaptureClient::Pause() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
if (frame_deliver_callback_.is_null()) {
return;
}
frame_deliver_callback_.Reset();
video_capture_host_->Pause(DeviceId());
}
void VideoCaptureClient::Resume(FrameDeliverCallback deliver_callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK(!deliver_callback.is_null());
if (!frame_deliver_callback_.is_null()) {
return;
}
frame_deliver_callback_ = std::move(deliver_callback);
video_capture_host_->Resume(DeviceId(), SessionId(), params_);
}
void VideoCaptureClient::RequestRefreshFrame() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (frame_deliver_callback_.is_null()) {
return;
}
video_capture_host_->RequestRefreshFrame(DeviceId());
}
void VideoCaptureClient::SwitchVideoCaptureHost(
mojo::PendingRemote<media::mojom::VideoCaptureHost> host) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
switching_video_capture_host_ = true;
video_capture_host_.reset();
video_capture_host_.Bind(std::move(host));
DCHECK(video_capture_host_);
}
void VideoCaptureClient::OnStateChanged(
media::mojom::VideoCaptureResultPtr result) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (result->which() == media::mojom::VideoCaptureResult::Tag::kState) {
media::mojom::VideoCaptureState state = result->get_state();
DVLOG(2) << __func__ << " state: " << state;
switch (state) {
case media::mojom::VideoCaptureState::STARTED:
RequestRefreshFrame();
break;
case media::mojom::VideoCaptureState::PAUSED:
case media::mojom::VideoCaptureState::RESUMED:
break;
case media::mojom::VideoCaptureState::STOPPED:
case media::mojom::VideoCaptureState::ENDED:
client_buffers_.clear();
weak_factory_.InvalidateWeakPtrs();
receiver_.reset();
if (switching_video_capture_host_) {
switching_video_capture_host_ = false;
first_frame_ref_time_ = base::TimeTicks();
accumulated_time_ = last_timestamp_;
Start(std::move(frame_deliver_callback_), std::move(error_callback_));
} else {
error_callback_.Reset();
frame_deliver_callback_.Reset();
}
break;
}
} else {
DVLOG(2) << __func__ << " Failed with an error.";
if (!error_callback_.is_null()) {
std::move(error_callback_).Run();
}
}
}
void VideoCaptureClient::OnNewBuffer(
int32_t buffer_id,
media::mojom::VideoBufferHandlePtr buffer_handle) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
if (!buffer_handle->is_read_only_shmem_region() &&
!buffer_handle->is_unsafe_shmem_region()) {
#if BUILDFLAG(IS_MAC)
if (!buffer_handle->is_gpu_memory_buffer_handle()) {
NOTIMPLEMENTED();
return;
}
#else
NOTIMPLEMENTED();
return;
#endif
}
const auto insert_result =
client_buffers_.insert({buffer_id, std::move(buffer_handle)});
DCHECK(insert_result.second);
}
void VideoCaptureClient::OnBufferReady(media::mojom::ReadyBufferPtr buffer) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer->buffer_id;
bool consume_buffer = !frame_deliver_callback_.is_null();
if (buffer->info->pixel_format != media::PIXEL_FORMAT_NV12 &&
buffer->info->pixel_format != media::PIXEL_FORMAT_I420 &&
buffer->info->pixel_format != media::PIXEL_FORMAT_Y16) {
consume_buffer = false;
LOG(DFATAL) << "Wrong pixel format, got pixel format:"
<< VideoPixelFormatToString(buffer->info->pixel_format);
}
if (!consume_buffer) {
video_capture_host_->ReleaseBuffer(DeviceId(), buffer->buffer_id,
media::VideoCaptureFeedback());
return;
}
base::TimeTicks reference_time = *buffer->info->metadata.reference_time;
if (first_frame_ref_time_.is_null()) {
first_frame_ref_time_ = reference_time;
}
// If the timestamp is not prepared, we use reference time to make a rough
// estimate. e.g. ThreadSafeCaptureOracle::DidCaptureFrame().
// TODO(crbug.com/40472286): Fix upstream capturers to always set timestamp
// and reference time.
if (buffer->info->timestamp.is_zero()) {
buffer->info->timestamp = reference_time - first_frame_ref_time_;
}
// Used by chrome/browser/media/cast_mirroring_performance_browsertest.cc
TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
TRACE_EVENT_SCOPE_THREAD, "timestamp",
(reference_time - base::TimeTicks()).InMicroseconds(),
"time_delta", buffer->info->timestamp.InMicroseconds());
const auto& buffer_iter = client_buffers_.find(buffer->buffer_id);
if (buffer_iter == client_buffers_.end()) {
LOG(DFATAL) << "Ignoring OnBufferReady() for unknown buffer.";
return;
}
scoped_refptr<media::VideoFrame> frame;
BufferFinishedCallback buffer_finished_callback;
if (buffer_iter->second->is_gpu_memory_buffer_handle()) {
#if BUILDFLAG(IS_MAC)
frame = media::VideoFrame::WrapUnacceleratedIOSurface(
buffer_iter->second->get_gpu_memory_buffer_handle().Clone(),
buffer->info->visible_rect, buffer->info->visible_rect.size(),
buffer->info->timestamp);
buffer_finished_callback =
base::BindPostTaskToCurrentDefault(base::BindOnce(
&VideoCaptureClient::OnClientBufferFinished,
weak_factory_.GetWeakPtr(), buffer->buffer_id, MappingKeepAlive()));
#else
NOTREACHED();
#endif
} else if (buffer_iter->second->is_unsafe_shmem_region()) {
base::WritableSharedMemoryMapping mapping =
buffer_iter->second->get_unsafe_shmem_region().Map();
const size_t frame_allocation_size = media::VideoFrame::AllocationSize(
buffer->info->pixel_format, buffer->info->coded_size);
if (mapping.IsValid() && mapping.size() >= frame_allocation_size) {
frame = media::VideoFrame::WrapExternalData(
buffer->info->pixel_format, buffer->info->coded_size,
buffer->info->visible_rect, buffer->info->visible_rect.size(),
mapping, buffer->info->timestamp);
}
buffer_finished_callback =
base::BindPostTaskToCurrentDefault(base::BindOnce(
&VideoCaptureClient::OnClientBufferFinished,
weak_factory_.GetWeakPtr(), buffer->buffer_id, std::move(mapping)));
} else {
// Duplicate base::ReadOnlySharedMemoryRegion here because there is no
// guarantee on lifetime between |client_buffers_| and |frame|.
base::ReadOnlySharedMemoryRegion shm_region =
buffer_iter->second->get_read_only_shmem_region().Duplicate();
base::ReadOnlySharedMemoryMapping mapping = shm_region.Map();
const size_t frame_allocation_size = media::VideoFrame::AllocationSize(
buffer->info->pixel_format, buffer->info->coded_size);
if (mapping.IsValid() && mapping.size() >= frame_allocation_size) {
frame = media::VideoFrame::WrapExternalData(
buffer->info->pixel_format, buffer->info->coded_size,
buffer->info->visible_rect, buffer->info->visible_rect.size(),
mapping, buffer->info->timestamp);
if (frame) {
frame->BackWithOwnedSharedMemory(std::move(shm_region),
std::move(mapping));
}
}
buffer_finished_callback = base::BindPostTaskToCurrentDefault(
base::BindOnce(&VideoCaptureClient::OnClientBufferFinished,
weak_factory_.GetWeakPtr(), buffer->buffer_id,
base::ReadOnlySharedMemoryMapping()));
}
if (!frame) {
LOG(DFATAL) << "Unable to wrap shared memory mapping.";
video_capture_host_->ReleaseBuffer(DeviceId(), buffer->buffer_id,
media::VideoCaptureFeedback());
OnStateChanged(media::mojom::VideoCaptureResult::NewErrorCode(
media::VideoCaptureError::kDeviceClientTooManyFramesDroppedY16));
return;
}
frame->AddDestructionObserver(
base::BindOnce(&VideoCaptureClient::DidFinishConsumingFrame,
std::move(buffer_finished_callback)));
// Convert NV12 frames to I420, because NV12 is not supported by Cast
// Streaming.
// https://crbug.com/1206325
if (frame->format() == media::PIXEL_FORMAT_NV12) {
if (!nv12_to_i420_pool_) {
nv12_to_i420_pool_ = std::make_unique<media::VideoFramePool>();
}
scoped_refptr<media::VideoFrame> new_frame =
nv12_to_i420_pool_->CreateFrame(
media::PIXEL_FORMAT_I420, frame->coded_size(),
frame->visible_rect(), frame->natural_size(), frame->timestamp());
media::EncoderStatus status =
frame_converter_.ConvertAndScale(*frame, *new_frame);
if (!status.is_ok()) {
LOG(DFATAL) << "Unable to convert frame to I420.";
OnStateChanged(media::mojom::VideoCaptureResult::NewErrorCode(
media::VideoCaptureError::kDeviceClientTooManyFramesDroppedY16));
return;
}
frame = new_frame;
}
frame->set_metadata(buffer->info->metadata);
frame->set_color_space(buffer->info->color_space);
frame->set_timestamp(frame->timestamp() + accumulated_time_);
last_timestamp_ = frame->timestamp();
frame_deliver_callback_.Run(frame);
}
void VideoCaptureClient::OnBufferDestroyed(int32_t buffer_id) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
const auto& buffer_iter = client_buffers_.find(buffer_id);
if (buffer_iter != client_buffers_.end()) {
client_buffers_.erase(buffer_iter);
}
}
void VideoCaptureClient::OnFrameDropped(
media::VideoCaptureFrameDropReason reason) {}
void VideoCaptureClient::OnNewCaptureVersion(
const media::CaptureVersion& capture_version) {}
void VideoCaptureClient::OnClientBufferFinished(int buffer_id,
MappingKeepAlive mapping) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
// Buffer was already destroyed.
if (client_buffers_.find(buffer_id) == client_buffers_.end()) {
return;
}
video_capture_host_->ReleaseBuffer(DeviceId(), buffer_id, feedback_);
feedback_ = media::VideoCaptureFeedback();
}
// static
void VideoCaptureClient::DidFinishConsumingFrame(
BufferFinishedCallback callback) {
// Note: This function may be called on any thread by the VideoFrame
// destructor.
DCHECK(!callback.is_null());
std::move(callback).Run();
}
void VideoCaptureClient::ProcessFeedback(
const media::VideoCaptureFeedback& feedback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
feedback_ = feedback;
}
} // namespace mirroring