| // Copyright 2014 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "remoting/client/plugin/pepper_video_renderer_3d.h" |
| |
| #include <math.h> |
| |
| #include <algorithm> |
| #include <utility> |
| |
| #include "ppapi/c/pp_codecs.h" |
| #include "ppapi/c/ppb_opengles2.h" |
| #include "ppapi/c/ppb_video_decoder.h" |
| #include "ppapi/cpp/instance.h" |
| #include "ppapi/lib/gl/include/GLES2/gl2.h" |
| #include "ppapi/lib/gl/include/GLES2/gl2ext.h" |
| #include "remoting/proto/video.pb.h" |
| #include "remoting/protocol/frame_stats.h" |
| #include "remoting/protocol/performance_tracker.h" |
| #include "remoting/protocol/session_config.h" |
| #include "third_party/webrtc/modules/desktop_capture/desktop_region.h" |
| |
| namespace remoting { |
| |
| // The implementation here requires that the decoder allocates at least 3 |
| // pictures. PPB_VideoDecoder didn't support this parameter prior to |
| // 1.1, so we have to pass 0 for backwards compatibility with older versions of |
| // the browser. Currently all API implementations allocate more than 3 buffers |
| // by default. |
| const uint32_t kMinimumPictureCount = 3; |
| |
| class PepperVideoRenderer3D::FrameTracker { |
| public: |
| FrameTracker(std::unique_ptr<VideoPacket> packet, |
| protocol::FrameStatsConsumer* stats_consumer, |
| const base::Closure& done) |
| : packet_(std::move(packet)), |
| stats_consumer_(stats_consumer), |
| done_(done) { |
| stats_.host_stats = protocol::HostFrameStats::GetForVideoPacket(*packet_); |
| stats_.client_stats.time_received = base::TimeTicks::Now(); |
| } |
| |
| ~FrameTracker() { |
| if (stats_consumer_) |
| stats_consumer_->OnVideoFrameStats(stats_); |
| if (!done_.is_null()) |
| done_.Run(); |
| } |
| |
| void OnDecoded() { |
| stats_.client_stats.time_decoded = base::TimeTicks::Now(); |
| } |
| void OnRendered() { |
| stats_.client_stats.time_rendered = base::TimeTicks::Now(); |
| } |
| |
| VideoPacket* packet() { return packet_.get(); } |
| |
| private: |
| std::unique_ptr<VideoPacket> packet_; |
| protocol::FrameStatsConsumer* stats_consumer_; |
| protocol::FrameStats stats_; |
| base::Closure done_; |
| }; |
| |
| class PepperVideoRenderer3D::Picture { |
| public: |
| Picture(pp::VideoDecoder* decoder, PP_VideoPicture picture) |
| : decoder_(decoder), picture_(picture) {} |
| ~Picture() { decoder_->RecyclePicture(picture_); } |
| |
| const PP_VideoPicture& picture() { return picture_; } |
| |
| private: |
| pp::VideoDecoder* decoder_; |
| PP_VideoPicture picture_; |
| }; |
| |
| PepperVideoRenderer3D::PepperVideoRenderer3D() : callback_factory_(this) {} |
| |
| PepperVideoRenderer3D::~PepperVideoRenderer3D() { |
| if (shader_program_) |
| gles2_if_->DeleteProgram(graphics_.pp_resource(), shader_program_); |
| } |
| |
| void PepperVideoRenderer3D::SetPepperContext( |
| pp::Instance* instance, |
| EventHandler* event_handler) { |
| DCHECK(event_handler); |
| DCHECK(!event_handler_); |
| |
| event_handler_ = event_handler; |
| pp_instance_ = instance; |
| } |
| |
| void PepperVideoRenderer3D::OnViewChanged(const pp::View& view) { |
| pp::Size size = view.GetRect().size(); |
| float scale = view.GetDeviceScale(); |
| view_size_.set(ceilf(size.width() * scale), ceilf(size.height() * scale)); |
| graphics_.ResizeBuffers(view_size_.width(), view_size_.height()); |
| |
| force_repaint_ = true; |
| PaintIfNeeded(); |
| } |
| |
| void PepperVideoRenderer3D::EnableDebugDirtyRegion(bool enable) { |
| debug_dirty_region_ = enable; |
| } |
| |
| bool PepperVideoRenderer3D::Initialize( |
| const ClientContext& context, |
| protocol::FrameStatsConsumer* stats_consumer) { |
| stats_consumer_ = stats_consumer; |
| |
| const int32_t context_attributes[] = { |
| PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 8, |
| PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8, |
| PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8, |
| PP_GRAPHICS3DATTRIB_RED_SIZE, 8, |
| PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0, |
| PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0, |
| PP_GRAPHICS3DATTRIB_SAMPLES, 0, |
| PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0, |
| PP_GRAPHICS3DATTRIB_WIDTH, 640, |
| PP_GRAPHICS3DATTRIB_HEIGHT, 480, |
| PP_GRAPHICS3DATTRIB_NONE, |
| }; |
| graphics_ = pp::Graphics3D(pp_instance_, context_attributes); |
| |
| if (graphics_.is_null()) { |
| LOG(WARNING) << "Graphics3D interface is not available."; |
| return false; |
| } |
| if (!pp_instance_->BindGraphics(graphics_)) { |
| LOG(WARNING) << "Failed to bind Graphics3D."; |
| return false; |
| } |
| |
| // Fetch the GLES2 interface to use to render frames. |
| gles2_if_ = static_cast<const PPB_OpenGLES2*>( |
| pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE)); |
| CHECK(gles2_if_); |
| |
| video_decoder_ = pp::VideoDecoder(pp_instance_); |
| if (video_decoder_.is_null()) { |
| LOG(WARNING) << "VideoDecoder interface is not available."; |
| return false; |
| } |
| |
| PP_Resource graphics_3d = graphics_.pp_resource(); |
| |
| gles2_if_->ClearColor(graphics_3d, 1, 0, 0, 1); |
| gles2_if_->Clear(graphics_3d, GL_COLOR_BUFFER_BIT); |
| |
| // Assign vertex positions and texture coordinates to buffers for use in |
| // shader program. |
| static const float kVertices[] = { |
| -1, -1, -1, 1, 1, -1, 1, 1, // Position coordinates. |
| 0, 1, 0, 0, 1, 1, 1, 0, // Texture coordinates. |
| }; |
| |
| GLuint buffer; |
| gles2_if_->GenBuffers(graphics_3d, 1, &buffer); |
| gles2_if_->BindBuffer(graphics_3d, GL_ARRAY_BUFFER, buffer); |
| gles2_if_->BufferData(graphics_3d, GL_ARRAY_BUFFER, sizeof(kVertices), |
| kVertices, GL_STATIC_DRAW); |
| |
| CheckGLError(); |
| |
| return true; |
| } |
| |
| void PepperVideoRenderer3D::OnSessionConfig( |
| const protocol::SessionConfig& config) { |
| PP_VideoProfile video_profile = PP_VIDEOPROFILE_VP8_ANY; |
| switch (config.video_config().codec) { |
| case protocol::ChannelConfig::CODEC_VP8: |
| video_profile = PP_VIDEOPROFILE_VP8_ANY; |
| break; |
| case protocol::ChannelConfig::CODEC_VP9: |
| video_profile = PP_VIDEOPROFILE_VP9_ANY; |
| break; |
| default: |
| NOTREACHED(); |
| } |
| |
| int32_t result = video_decoder_.Initialize( |
| graphics_, video_profile, PP_HARDWAREACCELERATION_WITHFALLBACK, |
| kMinimumPictureCount, |
| callback_factory_.NewCallback(&PepperVideoRenderer3D::OnInitialized)); |
| CHECK_EQ(result, PP_OK_COMPLETIONPENDING) |
| << "video_decoder_.Initialize() returned " << result; |
| } |
| |
| protocol::VideoStub* PepperVideoRenderer3D::GetVideoStub() { |
| return this; |
| } |
| |
| protocol::FrameConsumer* PepperVideoRenderer3D::GetFrameConsumer() { |
| // GetFrameConsumer() is used only for WebRTC-based connections which are not |
| // supported by the plugin. |
| NOTREACHED(); |
| return nullptr; |
| } |
| |
| protocol::FrameStatsConsumer* PepperVideoRenderer3D::GetFrameStatsConsumer() { |
| return stats_consumer_; |
| } |
| |
| void PepperVideoRenderer3D::ProcessVideoPacket( |
| std::unique_ptr<VideoPacket> packet, |
| const base::Closure& done) { |
| VideoPacket* packet_ptr = packet.get(); |
| std::unique_ptr<FrameTracker> frame_tracker( |
| new FrameTracker(std::move(packet), stats_consumer_, done)); |
| |
| // Don't need to do anything if the packet is empty. Host sends empty video |
| // packets when the screen is not changing. |
| if (packet_ptr->data().empty()) |
| return; |
| |
| if (!frame_received_) { |
| event_handler_->OnVideoFirstFrameReceived(); |
| frame_received_ = true; |
| } |
| |
| if (packet_ptr->format().has_screen_width() && |
| packet_ptr->format().has_screen_height()) { |
| frame_size_.set(packet_ptr->format().screen_width(), |
| packet_ptr->format().screen_height()); |
| } |
| |
| // Report the dirty region, for debugging, if requested. |
| if (debug_dirty_region_) { |
| webrtc::DesktopRegion dirty_region; |
| for (int i = 0; i < packet_ptr->dirty_rects_size(); ++i) { |
| Rect remoting_rect = packet_ptr->dirty_rects(i); |
| dirty_region.AddRect(webrtc::DesktopRect::MakeXYWH( |
| remoting_rect.x(), remoting_rect.y(), remoting_rect.width(), |
| remoting_rect.height())); |
| } |
| event_handler_->OnVideoFrameDirtyRegion(dirty_region); |
| } |
| |
| pending_frames_.push_back(std::move(frame_tracker)); |
| DecodeNextPacket(); |
| } |
| |
| void PepperVideoRenderer3D::OnInitialized(int32_t result) { |
| // Assume that VP8 and VP9 codecs are always supported by the browser. |
| CHECK_EQ(result, PP_OK) << "VideoDecoder::Initialize() failed: " << result; |
| initialization_finished_ = true; |
| |
| // Start decoding in case a frame was received during decoder initialization. |
| DecodeNextPacket(); |
| } |
| |
| void PepperVideoRenderer3D::DecodeNextPacket() { |
| if (!initialization_finished_ || decode_pending_ || pending_frames_.empty()) |
| return; |
| |
| const VideoPacket* packet = pending_frames_.front()->packet(); |
| |
| int32_t result = video_decoder_.Decode( |
| packet->frame_id(), packet->data().size(), packet->data().data(), |
| callback_factory_.NewCallback(&PepperVideoRenderer3D::OnDecodeDone)); |
| CHECK_EQ(result, PP_OK_COMPLETIONPENDING); |
| decode_pending_ = true; |
| } |
| |
| void PepperVideoRenderer3D::OnDecodeDone(int32_t result) { |
| DCHECK(decode_pending_); |
| decode_pending_ = false; |
| |
| if (result != PP_OK) { |
| LOG(ERROR) << "VideoDecoder::Decode() returned " << result; |
| event_handler_->OnVideoDecodeError(); |
| return; |
| } |
| |
| pending_frames_.front()->OnDecoded(); |
| // Move the frame from |pending_frames_| to |decoded_frames_|. |
| decoded_frames_.splice(decoded_frames_.end(), pending_frames_, |
| pending_frames_.begin()); |
| |
| DecodeNextPacket(); |
| GetNextPicture(); |
| } |
| |
| void PepperVideoRenderer3D::GetNextPicture() { |
| if (get_picture_pending_) |
| return; |
| |
| int32_t result = |
| video_decoder_.GetPicture(callback_factory_.NewCallbackWithOutput( |
| &PepperVideoRenderer3D::OnPictureReady)); |
| CHECK_EQ(result, PP_OK_COMPLETIONPENDING); |
| get_picture_pending_ = true; |
| } |
| |
| void PepperVideoRenderer3D::OnPictureReady(int32_t result, |
| PP_VideoPicture picture) { |
| DCHECK(get_picture_pending_); |
| get_picture_pending_ = false; |
| |
| if (result != PP_OK) { |
| LOG(ERROR) << "VideoDecoder::GetPicture() returned " << result; |
| event_handler_->OnVideoDecodeError(); |
| return; |
| } |
| |
| // Workaround crbug.com/542945 by filling in visible_rect if it isn't set. |
| if (picture.visible_rect.size.width == 0 || |
| picture.visible_rect.size.height == 0) { |
| static bool warning_logged = false; |
| if (!warning_logged) { |
| LOG(WARNING) << "PPB_VideoDecoder doesn't set visible_rect."; |
| warning_logged = true; |
| } |
| |
| picture.visible_rect.size.width = |
| std::min(frame_size_.width(), picture.texture_size.width); |
| picture.visible_rect.size.height = |
| std::min(frame_size_.height(), picture.texture_size.height); |
| } |
| |
| DCHECK_EQ(static_cast<int32_t>(picture.decode_id), |
| decoded_frames_.front()->packet()->frame_id()); |
| |
| // Move the frame from |decoded_frames_| to |next_picture_frames_|. |
| next_picture_frames_.splice(next_picture_frames_.end(), decoded_frames_, |
| decoded_frames_.begin()); |
| |
| next_picture_.reset(new Picture(&video_decoder_, picture)); |
| |
| PaintIfNeeded(); |
| GetNextPicture(); |
| } |
| |
| void PepperVideoRenderer3D::PaintIfNeeded() { |
| bool need_repaint = next_picture_ || (force_repaint_ && current_picture_); |
| if (paint_pending_ || !need_repaint) |
| return; |
| |
| if (next_picture_) { |
| current_picture_ = std::move(next_picture_); |
| current_picture_frames_.splice(current_picture_frames_.end(), |
| next_picture_frames_); |
| } |
| |
| force_repaint_ = false; |
| |
| const PP_VideoPicture& picture = current_picture_->picture(); |
| PP_Resource graphics_3d = graphics_.pp_resource(); |
| |
| EnsureProgramForTexture(picture.texture_target); |
| |
| gles2_if_->UseProgram(graphics_3d, shader_program_); |
| |
| // Calculate v_scale passed to the vertex shader. |
| double scale_x = picture.visible_rect.size.width; |
| double scale_y = picture.visible_rect.size.height; |
| if (picture.texture_target != GL_TEXTURE_RECTANGLE_ARB) { |
| CHECK(picture.texture_size.width > 0 && picture.texture_size.height > 0); |
| scale_x /= picture.texture_size.width; |
| scale_y /= picture.texture_size.height; |
| } |
| gles2_if_->Uniform2f(graphics_3d, shader_texcoord_scale_location_, |
| scale_x, scale_y); |
| |
| // Set viewport position & dimensions. |
| gles2_if_->Viewport(graphics_3d, 0, 0, view_size_.width(), |
| view_size_.height()); |
| |
| // Select the texture unit GL_TEXTURE0. |
| gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0); |
| |
| // Select the texture. |
| gles2_if_->BindTexture(graphics_3d, picture.texture_target, |
| picture.texture_id); |
| |
| // Select linear filter in case the texture needs to be scaled. |
| gles2_if_->TexParameteri(graphics_3d, picture.texture_target, |
| GL_TEXTURE_MIN_FILTER, GL_LINEAR); |
| |
| // When view dimensions are a multiple of the frame size then use |
| // nearest-neighbor scaling to achieve crisper image. Linear filter is used in |
| // all other cases. |
| GLint mag_filter = GL_LINEAR; |
| CHECK(picture.visible_rect.size.width > 0 && |
| picture.visible_rect.size.height > 0); |
| if (view_size_.width() % picture.visible_rect.size.width == 0 && |
| view_size_.height() % picture.visible_rect.size.height == 0) { |
| mag_filter = GL_NEAREST; |
| } |
| gles2_if_->TexParameteri(graphics_3d, picture.texture_target, |
| GL_TEXTURE_MAG_FILTER, mag_filter); |
| |
| // Render texture by drawing a triangle strip with 4 vertices. |
| gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4); |
| |
| CheckGLError(); |
| |
| // Request PPAPI display the queued texture. |
| int32_t result = graphics_.SwapBuffers( |
| callback_factory_.NewCallback(&PepperVideoRenderer3D::OnPaintDone)); |
| CHECK_EQ(result, PP_OK_COMPLETIONPENDING); |
| paint_pending_ = true; |
| } |
| |
| void PepperVideoRenderer3D::OnPaintDone(int32_t result) { |
| CHECK_EQ(result, PP_OK) << "Graphics3D::SwapBuffers() failed"; |
| |
| paint_pending_ = false; |
| |
| for (const auto& tracker : current_picture_frames_) { |
| tracker->OnRendered(); |
| } |
| current_picture_frames_.clear(); |
| |
| PaintIfNeeded(); |
| } |
| |
| void PepperVideoRenderer3D::EnsureProgramForTexture(uint32_t texture_target) { |
| static const char kVertexShader[] = |
| "varying vec2 v_texCoord; \n" |
| "attribute vec4 a_position; \n" |
| "attribute vec2 a_texCoord; \n" |
| "uniform vec2 v_scale; \n" |
| "void main() \n" |
| "{ \n" |
| " v_texCoord = v_scale * a_texCoord; \n" |
| " gl_Position = a_position; \n" |
| "}"; |
| |
| static const char kFragmentShader2D[] = |
| "precision mediump float; \n" |
| "varying vec2 v_texCoord; \n" |
| "uniform sampler2D s_texture; \n" |
| "void main() \n" |
| "{" |
| " gl_FragColor = texture2D(s_texture, v_texCoord); \n" |
| "}"; |
| |
| static const char kFragmentShaderRectangle[] = |
| "#extension GL_ARB_texture_rectangle : require\n" |
| "precision mediump float; \n" |
| "varying vec2 v_texCoord; \n" |
| "uniform sampler2DRect s_texture; \n" |
| "void main() \n" |
| "{" |
| " gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n" |
| "}"; |
| |
| static const char kFragmentShaderExternal[] = |
| "#extension GL_OES_EGL_image_external : require\n" |
| "precision mediump float; \n" |
| "varying vec2 v_texCoord; \n" |
| "uniform samplerExternalOES s_texture; \n" |
| "void main() \n" |
| "{" |
| " gl_FragColor = texture2D(s_texture, v_texCoord); \n" |
| "}"; |
| |
| // Initialize shader program only if texture type has changed. |
| if (current_shader_program_texture_target_ != texture_target) { |
| current_shader_program_texture_target_ = texture_target; |
| |
| if (texture_target == GL_TEXTURE_2D) { |
| CreateProgram(kVertexShader, kFragmentShader2D); |
| } else if (texture_target == GL_TEXTURE_RECTANGLE_ARB) { |
| CreateProgram(kVertexShader, kFragmentShaderRectangle); |
| } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) { |
| CreateProgram(kVertexShader, kFragmentShaderExternal); |
| } else { |
| LOG(FATAL) << "Unknown texture target: " << texture_target; |
| } |
| } |
| } |
| |
| void PepperVideoRenderer3D::CreateProgram(const char* vertex_shader, |
| const char* fragment_shader) { |
| PP_Resource graphics_3d = graphics_.pp_resource(); |
| if (shader_program_) |
| gles2_if_->DeleteProgram(graphics_3d, shader_program_); |
| |
| // Create shader program. |
| shader_program_ = gles2_if_->CreateProgram(graphics_3d); |
| CreateShaderProgram(GL_VERTEX_SHADER, vertex_shader); |
| CreateShaderProgram(GL_FRAGMENT_SHADER, fragment_shader); |
| gles2_if_->LinkProgram(graphics_3d, shader_program_); |
| gles2_if_->UseProgram(graphics_3d, shader_program_); |
| gles2_if_->Uniform1i( |
| graphics_3d, |
| gles2_if_->GetUniformLocation(graphics_3d, shader_program_, "s_texture"), |
| 0); |
| CheckGLError(); |
| |
| shader_texcoord_scale_location_ = gles2_if_->GetUniformLocation( |
| graphics_3d, shader_program_, "v_scale"); |
| |
| GLint pos_location = gles2_if_->GetAttribLocation( |
| graphics_3d, shader_program_, "a_position"); |
| GLint tc_location = gles2_if_->GetAttribLocation( |
| graphics_3d, shader_program_, "a_texCoord"); |
| CheckGLError(); |
| |
| // Construct the vertex array for DrawArrays(), using the buffer created in |
| // Initialize(). |
| gles2_if_->EnableVertexAttribArray(graphics_3d, pos_location); |
| gles2_if_->VertexAttribPointer(graphics_3d, pos_location, 2, GL_FLOAT, |
| GL_FALSE, 0, 0); |
| gles2_if_->EnableVertexAttribArray(graphics_3d, tc_location); |
| gles2_if_->VertexAttribPointer( |
| graphics_3d, tc_location, 2, GL_FLOAT, GL_FALSE, 0, |
| static_cast<float*>(0) + 8); // Skip position coordinates. |
| |
| gles2_if_->UseProgram(graphics_3d, 0); |
| |
| CheckGLError(); |
| } |
| |
| void PepperVideoRenderer3D::CreateShaderProgram(int type, const char* source) { |
| int size = strlen(source); |
| GLuint shader = gles2_if_->CreateShader(graphics_.pp_resource(), type); |
| gles2_if_->ShaderSource(graphics_.pp_resource(), shader, 1, &source, &size); |
| gles2_if_->CompileShader(graphics_.pp_resource(), shader); |
| gles2_if_->AttachShader(graphics_.pp_resource(), shader_program_, shader); |
| gles2_if_->DeleteShader(graphics_.pp_resource(), shader); |
| } |
| |
| void PepperVideoRenderer3D::CheckGLError() { |
| GLenum error = gles2_if_->GetError(graphics_.pp_resource()); |
| CHECK_EQ(error, static_cast<GLenum>(GL_NO_ERROR)) << "GL error: " << error; |
| } |
| |
| } // namespace remoting |