blob: e53d14173fa850a9e9eda2c58cb4b34834354a23 [file] [log] [blame]
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/gpu/test/rendering_helper.h"
#include <string.h>
#include <algorithm>
#include <memory>
#include <numeric>
#include <utility>
#include <vector>
#include "base/bind.h"
#include "base/bind_helpers.h"
#include "base/callback_helpers.h"
#include "base/command_line.h"
#include "base/mac/scoped_nsautorelease_pool.h"
#include "base/run_loop.h"
#include "base/single_thread_task_runner.h"
#include "base/stl_util.h"
#include "base/strings/stringize_macros.h"
#include "base/synchronization/waitable_event.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "media/gpu/test/texture_ref.h"
#include "ui/gfx/buffer_types.h"
#include "ui/gl/gl_context.h"
#include "ui/gl/gl_implementation.h"
#include "ui/gl/gl_surface.h"
#include "ui/gl/gl_surface_egl.h"
#include "ui/gl/init/gl_factory.h"
#if defined(USE_OZONE)
#include "ui/ozone/public/ozone_platform.h"
#endif // defined(USE_OZONE)
namespace media {
bool RenderingHelper::use_gl_ = false;
VideoFrameTexture::VideoFrameTexture(uint32_t texture_target,
uint32_t texture_id,
const base::Closure& no_longer_needed_cb)
: texture_target_(texture_target),
texture_id_(texture_id),
no_longer_needed_cb_(no_longer_needed_cb) {
DCHECK(no_longer_needed_cb_);
}
VideoFrameTexture::~VideoFrameTexture() {
std::move(no_longer_needed_cb_).Run();
}
RenderingHelper::RenderedVideo::RenderedVideo() {}
RenderingHelper::RenderedVideo::RenderedVideo(const RenderedVideo&) = default;
RenderingHelper::RenderedVideo::~RenderedVideo() {}
// static
void RenderingHelper::InitializeOneOff(bool use_gl, base::WaitableEvent* done) {
base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
#if defined(OS_WIN)
cmd_line->AppendSwitchASCII(switches::kUseGL, gl::kGLImplementationANGLEName);
#else
cmd_line->AppendSwitchASCII(switches::kUseGL, gl::kGLImplementationEGLName);
#endif
use_gl_ = use_gl;
#if defined(USE_OZONE)
ui::OzonePlatform::InitParams params;
params.single_process = true;
ui::OzonePlatform::InitializeForGPU(params);
ui::OzonePlatform::GetInstance()->AfterSandboxEntry();
#endif
if (!use_gl_) {
done->Signal();
return;
}
if (!gl::init::InitializeGLOneOff())
LOG(FATAL) << "Could not initialize GL";
done->Signal();
}
RenderingHelper::RenderingHelper() {
Clear();
}
RenderingHelper::~RenderingHelper() {
CHECK_EQ(videos_.size(), 0U) << "Must call UnInitialize before dtor.";
Clear();
}
void RenderingHelper::Initialize(const RenderingHelperParams& params,
base::WaitableEvent* done) {
// Use videos_.size() != 0 as a proxy for the class having already been
// Initialize()'d, and UnInitialize() before continuing.
if (videos_.size()) {
base::WaitableEvent done(base::WaitableEvent::ResetPolicy::AUTOMATIC,
base::WaitableEvent::InitialState::NOT_SIGNALED);
UnInitialize(&done);
done.Wait();
}
render_task_.Reset(
base::Bind(&RenderingHelper::RenderContent, base::Unretained(this)));
frame_duration_ = params.rendering_fps > 0
? base::TimeDelta::FromSeconds(1) / params.rendering_fps
: base::TimeDelta();
render_as_thumbnails_ = params.render_as_thumbnails;
task_runner_ = base::ThreadTaskRunnerHandle::Get();
videos_.resize(params.num_windows);
// Skip all the GL stuff if we don't use it
if (!use_gl_) {
done->Signal();
return;
}
gl_surface_ = gl::init::CreateOffscreenGLSurface(gfx::Size());
gl_context_ = gl::init::CreateGLContext(nullptr, gl_surface_.get(),
gl::GLContextAttribs());
CHECK(gl_context_->MakeCurrent(gl_surface_.get()));
if (render_as_thumbnails_) {
CHECK_EQ(videos_.size(), 1U);
GLint max_texture_size;
glGetIntegerv(GL_MAX_TEXTURE_SIZE, &max_texture_size);
CHECK_GE(max_texture_size, params.thumbnails_page_size.width());
CHECK_GE(max_texture_size, params.thumbnails_page_size.height());
thumbnails_fbo_size_ = params.thumbnails_page_size;
thumbnail_size_ = params.thumbnail_size;
glGenFramebuffersEXT(1, &thumbnails_fbo_id_);
glGenTextures(1, &thumbnails_texture_id_);
glBindTexture(GL_TEXTURE_2D, thumbnails_texture_id_);
glTexImage2D(GL_TEXTURE_2D,
0,
GL_RGB,
thumbnails_fbo_size_.width(), thumbnails_fbo_size_.height(),
0,
GL_RGB,
GL_UNSIGNED_SHORT_5_6_5,
NULL);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glBindTexture(GL_TEXTURE_2D, 0);
glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, thumbnails_texture_id_, 0);
GLenum fb_status = glCheckFramebufferStatusEXT(GL_FRAMEBUFFER);
CHECK(fb_status == GL_FRAMEBUFFER_COMPLETE) << fb_status;
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glBindFramebufferEXT(GL_FRAMEBUFFER,
gl_surface_->GetBackingFramebufferObject());
}
// These vertices and texture coords. map (0,0) in the texture to the
// bottom left of the viewport. Since we get the video frames with the
// the top left at (0,0) we need to flip the texture y coordinate
// in the vertex shader for this to be rendered the right way up.
// In the case of thumbnail rendering we use the same vertex shader
// to render the FBO the screen, where we do not want this flipping.
// Vertices are 2 floats for position and 2 floats for texcoord each.
static const float kVertices[] = {
-1, 1, 0, 1, // Vertex 0
-1, -1, 0, 0, // Vertex 1
1, 1, 1, 1, // Vertex 2
1, -1, 1, 0, // Vertex 3
};
static const GLvoid* kVertexPositionOffset = 0;
static const GLvoid* kVertexTexcoordOffset =
reinterpret_cast<GLvoid*>(sizeof(float) * 2);
static const GLsizei kVertexStride = sizeof(float) * 4;
glGenBuffersARB(1, &vertex_buffer_);
glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer_);
glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
static const char kVertexShader[] =
STRINGIZE(varying vec2 interp_tc; attribute vec4 in_pos;
attribute vec2 in_tc; uniform bool tex_flip; void main() {
if (tex_flip)
interp_tc = vec2(in_tc.x, 1.0 - in_tc.y);
else
interp_tc = in_tc;
gl_Position = in_pos;
});
#if !defined(OS_WIN)
static const char kFragmentShader[] =
"#extension GL_OES_EGL_image_external : enable\n"
"precision mediump float;\n"
"varying vec2 interp_tc;\n"
"uniform sampler2D tex;\n"
"#ifdef GL_OES_EGL_image_external\n"
"uniform samplerExternalOES tex_external;\n"
"#endif\n"
"void main() {\n"
" vec4 color = texture2D(tex, interp_tc);\n"
"#ifdef GL_OES_EGL_image_external\n"
" color += texture2D(tex_external, interp_tc);\n"
"#endif\n"
" gl_FragColor = color;\n"
"}\n";
#else
static const char kFragmentShader[] =
"#ifdef GL_ES\n"
"precision mediump float;\n"
"#endif\n"
"varying vec2 interp_tc;\n"
"uniform sampler2D tex;\n"
"void main() {\n"
" gl_FragColor = texture2D(tex, interp_tc);\n"
"}\n";
#endif
program_ = glCreateProgram();
CreateShader(program_, GL_VERTEX_SHADER, kVertexShader,
base::size(kVertexShader));
CreateShader(program_, GL_FRAGMENT_SHADER, kFragmentShader,
base::size(kFragmentShader));
glLinkProgram(program_);
int result = GL_FALSE;
glGetProgramiv(program_, GL_LINK_STATUS, &result);
if (!result) {
char log[4096];
glGetShaderInfoLog(program_, base::size(log), NULL, log);
LOG(FATAL) << log;
}
glUseProgram(program_);
glDeleteProgram(program_);
glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0);
glUniform1i(glGetUniformLocation(program_, "tex"), 0);
GLint tex_external = glGetUniformLocation(program_, "tex_external");
if (tex_external != -1) {
glUniform1i(tex_external, 1);
}
int pos_location = glGetAttribLocation(program_, "in_pos");
glEnableVertexAttribArray(pos_location);
glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, kVertexStride,
kVertexPositionOffset);
int tc_location = glGetAttribLocation(program_, "in_tc");
glEnableVertexAttribArray(tc_location);
glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, kVertexStride,
kVertexTexcoordOffset);
// Unbind the vertex buffer
glBindBuffer(GL_ARRAY_BUFFER, 0);
done->Signal();
}
void RenderingHelper::UnInitialize(base::WaitableEvent* done) {
// We have never been initialized in the first place...
if (task_runner_.get() == nullptr) {
done->Signal();
return;
}
CHECK(task_runner_->BelongsToCurrentThread());
render_task_.Cancel();
if (!use_gl_) {
Clear();
done->Signal();
return;
}
if (render_as_thumbnails_) {
glDeleteTextures(1, &thumbnails_texture_id_);
glDeleteFramebuffersEXT(1, &thumbnails_fbo_id_);
}
glDeleteBuffersARB(1, &vertex_buffer_);
gl_context_->ReleaseCurrent(gl_surface_.get());
gl_context_ = NULL;
gl_surface_ = NULL;
Clear();
done->Signal();
}
scoped_refptr<media::test::TextureRef> RenderingHelper::CreateTexture(
uint32_t texture_target,
bool pre_allocate,
VideoPixelFormat pixel_format,
const gfx::Size& size) {
CHECK(task_runner_->BelongsToCurrentThread());
uint32_t texture_id = CreateTextureId(texture_target, size);
base::OnceClosure delete_texture_cb =
use_gl_ ? base::BindOnce(DeleteTexture, texture_id) : base::DoNothing();
if (pre_allocate) {
return media::test::TextureRef::CreatePreallocated(
texture_id, std::move(delete_texture_cb), pixel_format, size,
gfx::BufferUsage::SCANOUT_VDA_WRITE);
}
return media::test::TextureRef::Create(texture_id,
std::move(delete_texture_cb));
}
uint32_t RenderingHelper::CreateTextureId(uint32_t texture_target,
const gfx::Size& size) {
CHECK(task_runner_->BelongsToCurrentThread());
if (!use_gl_) {
return 0;
}
uint32_t texture_id;
glGenTextures(1, &texture_id);
glBindTexture(texture_target, texture_id);
if (texture_target == GL_TEXTURE_2D) {
glTexImage2D(GL_TEXTURE_2D,
0,
GL_RGBA,
size.width(), size.height(),
0,
GL_RGBA,
GL_UNSIGNED_BYTE,
NULL);
}
glTexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
// OpenGLES2.0.25 section 3.8.2 requires CLAMP_TO_EDGE for NPOT textures.
glTexParameteri(texture_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(texture_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
return texture_id;
}
void RenderingHelper::ConsumeVideoFrame(
size_t window_id,
scoped_refptr<VideoFrameTexture> video_frame) {
if (render_as_thumbnails_) {
RenderThumbnail(video_frame->texture_target(), video_frame->texture_id());
} else {
QueueVideoFrame(window_id, std::move(video_frame));
}
}
void RenderingHelper::RenderThumbnail(uint32_t texture_target,
uint32_t texture_id) {
CHECK(task_runner_->BelongsToCurrentThread());
CHECK(use_gl_);
const int width = thumbnail_size_.width();
const int height = thumbnail_size_.height();
const int thumbnails_in_row = thumbnails_fbo_size_.width() / width;
const int thumbnails_in_column = thumbnails_fbo_size_.height() / height;
const int row = (frame_count_ / thumbnails_in_row) % thumbnails_in_column;
const int col = frame_count_ % thumbnails_in_row;
gfx::Rect area(col * width, row * height, width, height);
glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0);
glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
GLSetViewPort(area);
RenderTexture(texture_target, texture_id);
glBindFramebufferEXT(GL_FRAMEBUFFER,
gl_surface_->GetBackingFramebufferObject());
// Need to flush the GL commands before we return the tnumbnail texture to
// the decoder.
glFlush();
++frame_count_;
}
void RenderingHelper::QueueVideoFrame(
size_t window_id,
scoped_refptr<VideoFrameTexture> video_frame) {
CHECK(task_runner_->BelongsToCurrentThread());
RenderedVideo* video = &videos_[window_id];
DCHECK(!video->is_flushing);
// If running at zero fps, return immediately. This will give the frame
// back to the client once it drops its reference to video_frame.
if (frame_duration_.is_zero())
return;
video->pending_frames.push(video_frame);
if (video->frames_to_drop > 0 && video->pending_frames.size() > 1) {
--video->frames_to_drop;
video->pending_frames.pop();
}
// Schedules the first RenderContent() if need.
if (scheduled_render_time_.is_null()) {
scheduled_render_time_ = base::TimeTicks::Now();
task_runner_->PostTask(FROM_HERE, render_task_.callback());
}
}
// static
void RenderingHelper::DeleteTexture(uint32_t texture_id) {
glDeleteTextures(1, &texture_id);
CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
}
// static
void RenderingHelper::GLSetViewPort(const gfx::Rect& area) {
glViewport(area.x(), area.y(), area.width(), area.height());
glScissor(area.x(), area.y(), area.width(), area.height());
}
// static
void RenderingHelper::CreateShader(GLuint program,
GLenum type,
const char* source,
int size) {
GLuint shader = glCreateShader(type);
glShaderSource(shader, 1, &source, &size);
glCompileShader(shader);
int result = GL_FALSE;
glGetShaderiv(shader, GL_COMPILE_STATUS, &result);
if (!result) {
char log[4096];
glGetShaderInfoLog(shader, base::size(log), NULL, log);
LOG(FATAL) << log;
}
glAttachShader(program, shader);
glDeleteShader(shader);
CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
}
// static
void RenderingHelper::RenderTexture(uint32_t texture_target,
uint32_t texture_id) {
// The ExternalOES sampler is bound to GL_TEXTURE1 and the Texture2D sampler
// is bound to GL_TEXTURE0.
if (texture_target == GL_TEXTURE_2D) {
glActiveTexture(GL_TEXTURE0 + 0);
} else if (texture_target == GL_TEXTURE_EXTERNAL_OES) {
glActiveTexture(GL_TEXTURE0 + 1);
}
glBindTexture(texture_target, texture_id);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glBindTexture(texture_target, 0);
CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR);
}
gl::GLContext* RenderingHelper::GetGLContext() {
return gl_context_.get();
}
void RenderingHelper::Clear() {
videos_.clear();
task_runner_ = nullptr;
gl_context_ = NULL;
gl_surface_ = NULL;
render_as_thumbnails_ = false;
frame_count_ = 0;
thumbnails_fbo_id_ = 0;
thumbnails_texture_id_ = 0;
}
void RenderingHelper::GetThumbnailsAsRGBA(std::vector<unsigned char>* rgba,
base::WaitableEvent* done) {
CHECK(render_as_thumbnails_ && use_gl_);
const size_t num_pixels = thumbnails_fbo_size_.GetArea();
rgba->resize(num_pixels * 4);
glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_);
glPixelStorei(GL_PACK_ALIGNMENT, 1);
// We can only count on GL_RGBA/GL_UNSIGNED_BYTE support.
glReadPixels(0, 0, thumbnails_fbo_size_.width(),
thumbnails_fbo_size_.height(), GL_RGBA, GL_UNSIGNED_BYTE,
&(*rgba)[0]);
glBindFramebufferEXT(GL_FRAMEBUFFER,
gl_surface_->GetBackingFramebufferObject());
done->Signal();
}
void RenderingHelper::Flush(size_t window_id) {
videos_[window_id].is_flushing = true;
}
void RenderingHelper::RenderContent() {
CHECK(task_runner_->BelongsToCurrentThread());
// Frames that will be returned to the client (via the no_longer_needed_cb)
// after this vector falls out of scope at the end of this method. We need
// to keep references to them until after SwapBuffers() call below.
std::vector<scoped_refptr<VideoFrameTexture>> frames_to_be_returned;
for (RenderedVideo& video : videos_) {
if (video.pending_frames.empty())
continue;
scoped_refptr<VideoFrameTexture> frame = video.pending_frames.front();
// TODO(owenlin): Render to FBO.
// RenderTexture(frame->texture_target(), frame->texture_id());
if (video.pending_frames.size() > 1 || video.is_flushing) {
frames_to_be_returned.push_back(video.pending_frames.front());
video.pending_frames.pop();
} else {
++video.frames_to_drop;
}
}
ScheduleNextRenderContent();
}
void RenderingHelper::DropOneFrameForAllVideos() {
for (RenderedVideo& video : videos_) {
if (video.pending_frames.empty())
continue;
if (video.pending_frames.size() > 1 || video.is_flushing) {
video.pending_frames.pop();
} else {
++video.frames_to_drop;
}
}
}
void RenderingHelper::ScheduleNextRenderContent() {
const auto vsync_interval = base::TimeDelta::FromSeconds(1) / 60;
scheduled_render_time_ += frame_duration_;
base::TimeTicks now = base::TimeTicks::Now();
base::TimeTicks target;
if (vsync_timebase_.is_null()) {
vsync_timebase_ = now;
}
if (vsync_interval.is_zero()) {
target = std::max(now, scheduled_render_time_);
} else {
// Schedules the next RenderContent() at latest VSYNC before the
// |scheduled_render_time_|.
target = std::max(now + vsync_interval, scheduled_render_time_);
int64_t intervals = (target - vsync_timebase_) / vsync_interval;
target = vsync_timebase_ + intervals * vsync_interval;
}
// When the rendering falls behind, drops frames.
while (scheduled_render_time_ < target) {
scheduled_render_time_ += frame_duration_;
DropOneFrameForAllVideos();
}
task_runner_->PostDelayedTask(FROM_HERE, render_task_.callback(),
target - now);
}
} // namespace media