| // Copyright 2015 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "remoting/protocol/webrtc_video_renderer_adapter.h" |
| |
| #include <memory> |
| #include <string> |
| #include <utility> |
| |
| #include "base/bind.h" |
| #include "base/callback.h" |
| #include "base/location.h" |
| #include "base/memory/ptr_util.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/task_runner_util.h" |
| #include "base/threading/thread_task_runner_handle.h" |
| #include "base/threading/worker_pool.h" |
| #include "remoting/protocol/client_video_stats_dispatcher.h" |
| #include "remoting/protocol/frame_consumer.h" |
| #include "remoting/protocol/frame_stats.h" |
| #include "remoting/protocol/video_renderer.h" |
| #include "remoting/protocol/webrtc_transport.h" |
| #include "third_party/libyuv/include/libyuv/convert_from.h" |
| #include "third_party/webrtc/media/base/videoframe.h" |
| #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" |
| |
| namespace remoting { |
| namespace protocol { |
| |
| namespace { |
| |
| // Maximum number of ClientFrameStats instances to keep. |
| const int kMaxQueuedStats = 200; |
| |
| std::unique_ptr<webrtc::DesktopFrame> ConvertYuvToRgb( |
| scoped_refptr<webrtc::VideoFrameBuffer> yuv_frame, |
| std::unique_ptr<webrtc::DesktopFrame> rgb_frame, |
| FrameConsumer::PixelFormat pixel_format) { |
| DCHECK(rgb_frame->size().equals( |
| webrtc::DesktopSize(yuv_frame->width(), yuv_frame->height()))); |
| auto yuv_to_rgb_function = (pixel_format == FrameConsumer::FORMAT_BGRA) |
| ? &libyuv::I420ToARGB |
| : &libyuv::I420ToABGR; |
| yuv_to_rgb_function(yuv_frame->DataY(), yuv_frame->StrideY(), |
| yuv_frame->DataU(), yuv_frame->StrideU(), |
| yuv_frame->DataV(), yuv_frame->StrideV(), |
| rgb_frame->data(), rgb_frame->stride(), |
| yuv_frame->width(), yuv_frame->height()); |
| |
| rgb_frame->mutable_updated_region()->AddRect( |
| webrtc::DesktopRect::MakeSize(rgb_frame->size())); |
| return rgb_frame; |
| } |
| |
| } // namespace |
| |
| WebrtcVideoRendererAdapter::WebrtcVideoRendererAdapter( |
| const std::string& label, |
| VideoRenderer* video_renderer) |
| : label_(label), |
| video_renderer_(video_renderer), |
| task_runner_(base::ThreadTaskRunnerHandle::Get()), |
| weak_factory_(this) {} |
| |
| WebrtcVideoRendererAdapter::~WebrtcVideoRendererAdapter() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| webrtc::VideoTrackVector video_tracks = media_stream_->GetVideoTracks(); |
| DCHECK(!video_tracks.empty()); |
| video_tracks[0]->RemoveSink(this); |
| } |
| |
| void WebrtcVideoRendererAdapter::SetMediaStream( |
| scoped_refptr<webrtc::MediaStreamInterface> media_stream) { |
| DCHECK_EQ(media_stream->label(), label()); |
| |
| media_stream_ = std::move(media_stream); |
| |
| webrtc::VideoTrackVector video_tracks = media_stream_->GetVideoTracks(); |
| if (video_tracks.empty()) { |
| LOG(ERROR) << "Received media stream with no video tracks."; |
| return; |
| } |
| |
| if (video_tracks.size() > 1U) { |
| LOG(WARNING) << "Received media stream with multiple video tracks."; |
| } |
| |
| video_tracks[0]->AddOrUpdateSink(this, rtc::VideoSinkWants()); |
| } |
| |
| void WebrtcVideoRendererAdapter::SetVideoStatsChannel( |
| std::unique_ptr<MessagePipe> message_pipe) { |
| // Expect that the host also creates video_stats data channel. |
| video_stats_dispatcher_.reset(new ClientVideoStatsDispatcher(label_, this)); |
| video_stats_dispatcher_->Init(std::move(message_pipe), this); |
| } |
| |
| void WebrtcVideoRendererAdapter::OnFrame(const cricket::VideoFrame& frame) { |
| if (static_cast<uint64_t>(frame.timestamp_us()) >= rtc::TimeMicros()) { |
| // The host sets playout delay to 0, so all incoming frames are expected to |
| // be rendered as so as they are received. |
| LOG(WARNING) << "Received frame with playout delay greater than 0."; |
| } |
| |
| task_runner_->PostTask( |
| FROM_HERE, |
| base::Bind(&WebrtcVideoRendererAdapter::HandleFrameOnMainThread, |
| weak_factory_.GetWeakPtr(), frame.transport_frame_id(), |
| base::TimeTicks::Now(), |
| scoped_refptr<webrtc::VideoFrameBuffer>( |
| frame.video_frame_buffer().get()))); |
| } |
| |
| void WebrtcVideoRendererAdapter::OnVideoFrameStats( |
| uint32_t frame_id, |
| const HostFrameStats& host_stats) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| // Drop all ClientFrameStats for frames before |frame_id|. Stats messages are |
| // expected to be received in the same order as the corresponding video |
| // frames, so we are not going to receive HostFrameStats for the frames before |
| // |frame_id|. This may happen only if for some reason the host doesn't |
| // generate stats message for all video frames. |
| while (!client_stats_queue_.empty() && |
| client_stats_queue_.front().first != frame_id) { |
| client_stats_queue_.pop_front(); |
| } |
| |
| // If there are no ClientFrameStats in the queue then queue HostFrameStats |
| // to be processed in FrameRendered(). |
| if (client_stats_queue_.empty()) { |
| if (host_stats_queue_.size() > kMaxQueuedStats) { |
| LOG(ERROR) << "video_stats channel is out of sync with the video stream. " |
| "Performance stats will not be reported."; |
| video_stats_dispatcher_.reset(); |
| return; |
| } |
| host_stats_queue_.push_back(std::make_pair(frame_id, host_stats)); |
| return; |
| } |
| |
| // The correspond frame has been received and now we have both HostFrameStats |
| // and ClientFrameStats. Report the stats to FrameStatsConsumer. |
| DCHECK_EQ(client_stats_queue_.front().first, frame_id); |
| FrameStats frame_stats; |
| frame_stats.client_stats = client_stats_queue_.front().second; |
| client_stats_queue_.pop_front(); |
| frame_stats.host_stats = host_stats; |
| FrameStatsConsumer* frame_stats_consumer = |
| video_renderer_->GetFrameStatsConsumer(); |
| if (frame_stats_consumer) |
| frame_stats_consumer->OnVideoFrameStats(frame_stats); |
| } |
| |
| void WebrtcVideoRendererAdapter::OnChannelInitialized( |
| ChannelDispatcherBase* channel_dispatcher) {} |
| |
| void WebrtcVideoRendererAdapter::OnChannelClosed( |
| ChannelDispatcherBase* channel_dispatcher) { |
| LOG(WARNING) << "video_stats channel was closed by the host."; |
| } |
| |
| void WebrtcVideoRendererAdapter::HandleFrameOnMainThread( |
| uint32_t frame_id, |
| base::TimeTicks time_received, |
| scoped_refptr<webrtc::VideoFrameBuffer> frame) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| std::unique_ptr<ClientFrameStats> stats(new ClientFrameStats()); |
| // TODO(sergeyu): |time_received| is not reported correctly here because the |
| // frame is already decoded at this point. |
| stats->time_received = time_received; |
| |
| std::unique_ptr<webrtc::DesktopFrame> rgb_frame = |
| video_renderer_->GetFrameConsumer()->AllocateFrame( |
| webrtc::DesktopSize(frame->width(), frame->height())); |
| |
| base::PostTaskAndReplyWithResult( |
| base::WorkerPool::GetTaskRunner(false).get(), FROM_HERE, |
| base::Bind(&ConvertYuvToRgb, base::Passed(&frame), |
| base::Passed(&rgb_frame), |
| video_renderer_->GetFrameConsumer()->GetPixelFormat()), |
| base::Bind(&WebrtcVideoRendererAdapter::DrawFrame, |
| weak_factory_.GetWeakPtr(), frame_id, base::Passed(&stats))); |
| } |
| |
| void WebrtcVideoRendererAdapter::DrawFrame( |
| uint32_t frame_id, |
| std::unique_ptr<ClientFrameStats> stats, |
| std::unique_ptr<webrtc::DesktopFrame> frame) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| stats->time_decoded = base::TimeTicks::Now(); |
| video_renderer_->GetFrameConsumer()->DrawFrame( |
| std::move(frame), |
| base::Bind(&WebrtcVideoRendererAdapter::FrameRendered, |
| weak_factory_.GetWeakPtr(), frame_id, base::Passed(&stats))); |
| } |
| |
| void WebrtcVideoRendererAdapter::FrameRendered( |
| uint32_t frame_id, |
| std::unique_ptr<ClientFrameStats> client_stats) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| if (!video_stats_dispatcher_ || !video_stats_dispatcher_->is_connected()) |
| return; |
| |
| client_stats->time_rendered = base::TimeTicks::Now(); |
| |
| // Drop all HostFrameStats for frames before |frame_id|. Stats messages are |
| // expected to be received in the same order as the corresponding video |
| // frames. This may happen only if the host generates HostFrameStats without |
| // the corresponding frame. |
| while (!host_stats_queue_.empty() && |
| host_stats_queue_.front().first != frame_id) { |
| LOG(WARNING) << "Host sent VideoStats message for a frame that was never " |
| "received."; |
| host_stats_queue_.pop_front(); |
| } |
| |
| // If HostFrameStats hasn't been received for |frame_id| then queue |
| // ClientFrameStats to be processed in OnVideoFrameStats(). |
| if (host_stats_queue_.empty()) { |
| if (client_stats_queue_.size() > kMaxQueuedStats) { |
| LOG(ERROR) << "video_stats channel is out of sync with the video " |
| "stream. Performance stats will not be reported."; |
| video_stats_dispatcher_.reset(); |
| return; |
| } |
| client_stats_queue_.push_back(std::make_pair(frame_id, *client_stats)); |
| return; |
| } |
| |
| // The correspond HostFrameStats has been received already and now we have |
| // both HostFrameStats and ClientFrameStats. Report the stats to |
| // FrameStatsConsumer. |
| DCHECK_EQ(host_stats_queue_.front().first, frame_id); |
| FrameStats frame_stats; |
| frame_stats.host_stats = host_stats_queue_.front().second; |
| frame_stats.client_stats = *client_stats; |
| host_stats_queue_.pop_front(); |
| FrameStatsConsumer* frame_stats_consumer = |
| video_renderer_->GetFrameStatsConsumer(); |
| if (frame_stats_consumer) |
| frame_stats_consumer->OnVideoFrameStats(frame_stats); |
| } |
| |
| } // namespace protocol |
| } // namespace remoting |