Add viz::VideoCaptureOverlay to support mouse cursor rendering VIZ-side.
VideoCaptureOverlay provides the post-GPU-readback rendering of small
SkBitmaps onto the VideoFrames captured by viz::FrameSinkVideoCapturer.
This will be used to move mouse cursor rendering out of the browser
process, and into VIZ, as was intended in the original design. More
details in crbug/810133.
A soon-upcoming series of changes will provide mojo IDL definition as
well as integration of VideoCaptureOverlay with FrameSinkVideoCapturer.
Bug: 810133
Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel
Change-Id: I16897dbd361af153d10a1223339d9fb5f7e5f05a
Reviewed-on: https://chromium-review.googlesource.com/1132318
Commit-Queue: Yuri Wiitala <miu@chromium.org>
Reviewed-by: Xiangjun Zhang <xjz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#574480}diff --git a/components/viz/service/BUILD.gn b/components/viz/service/BUILD.gn
index 826d73f..166f325 100644
--- a/components/viz/service/BUILD.gn
+++ b/components/viz/service/BUILD.gn
@@ -144,6 +144,8 @@
"frame_sinks/video_capture/in_flight_frame_delivery.h",
"frame_sinks/video_capture/interprocess_frame_pool.cc",
"frame_sinks/video_capture/interprocess_frame_pool.h",
+ "frame_sinks/video_capture/video_capture_overlay.cc",
+ "frame_sinks/video_capture/video_capture_overlay.h",
"frame_sinks/video_detector.cc",
"frame_sinks/video_detector.h",
"gl/gpu_service_impl.cc",
@@ -319,6 +321,7 @@
"frame_sinks/surface_synchronization_unittest.cc",
"frame_sinks/video_capture/frame_sink_video_capturer_impl_unittest.cc",
"frame_sinks/video_capture/interprocess_frame_pool_unittest.cc",
+ "frame_sinks/video_capture/video_capture_overlay_unittest.cc",
"frame_sinks/video_detector_unittest.cc",
"gl/gpu_service_impl_unittest.cc",
"hit_test/hit_test_aggregator_unittest.cc",
diff --git a/components/viz/service/frame_sinks/video_capture/video_capture_overlay.cc b/components/viz/service/frame_sinks/video_capture/video_capture_overlay.cc
new file mode 100644
index 0000000..75be2cb
--- /dev/null
+++ b/components/viz/service/frame_sinks/video_capture/video_capture_overlay.cc
@@ -0,0 +1,574 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/viz/service/frame_sinks/video_capture/video_capture_overlay.h"
+
+#include <algorithm>
+#include <cmath>
+#include <utility>
+
+#include "base/bind.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/trace_event/trace_event.h"
+#include "media/base/limits.h"
+#include "media/base/video_frame.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "third_party/skia/include/core/SkFilterQuality.h"
+#include "third_party/skia/include/core/SkImageInfo.h"
+#include "ui/gfx/geometry/point.h"
+#include "ui/gfx/geometry/rect_conversions.h"
+
+using media::VideoFrame;
+using media::VideoPixelFormat;
+
+namespace viz {
+
+VideoCaptureOverlay::FrameSource::~FrameSource() = default;
+
+VideoCaptureOverlay::VideoCaptureOverlay(FrameSource* frame_source)
+ : frame_source_(frame_source) {
+ DCHECK(frame_source_);
+}
+
+VideoCaptureOverlay::~VideoCaptureOverlay() = default;
+
+void VideoCaptureOverlay::SetImageAndBounds(const SkBitmap& image,
+ const gfx::RectF& bounds) {
+ const gfx::Rect old_rect = ComputeSourceMutationRect();
+
+ image_ = image;
+ bounds_ = bounds;
+
+ // Reset the cached sprite since the source image has been changed.
+ sprite_ = nullptr;
+
+ const gfx::Rect new_rect = ComputeSourceMutationRect();
+ if (!new_rect.IsEmpty() || !old_rect.IsEmpty()) {
+ frame_source_->InvalidateRect(old_rect);
+ frame_source_->InvalidateRect(new_rect);
+ frame_source_->RequestRefreshFrame();
+ }
+}
+
+void VideoCaptureOverlay::SetBounds(const gfx::RectF& bounds) {
+ if (bounds_ != bounds) {
+ const gfx::Rect old_rect = ComputeSourceMutationRect();
+ bounds_ = bounds;
+ const gfx::Rect new_rect = ComputeSourceMutationRect();
+ if (!new_rect.IsEmpty() || !old_rect.IsEmpty()) {
+ frame_source_->InvalidateRect(old_rect);
+ frame_source_->InvalidateRect(new_rect);
+ frame_source_->RequestRefreshFrame();
+ }
+ }
+}
+
+namespace {
+
+// Scales a |relative| rect having coordinates in the range [0.0,1.0) by the
+// given |span|, snapping all coordinates to even numbers.
+gfx::Rect ToAbsoluteBoundsForI420(const gfx::RectF& relative,
+ const gfx::Rect& span) {
+ const float absolute_left = std::fma(relative.x(), span.width(), span.x());
+ const float absolute_top = std::fma(relative.y(), span.height(), span.y());
+ const float absolute_right =
+ std::fma(relative.right(), span.width(), span.x());
+ const float absolute_bottom =
+ std::fma(relative.bottom(), span.height(), span.y());
+
+ // Compute the largest I420-friendly Rect that is fully-enclosed by the
+ // absolute rect. Use saturated_cast<> to restrict all extreme results [and
+ // Inf and NaN] to a safe range of integers.
+ const int snapped_left =
+ base::saturated_cast<int16_t>(std::ceil(absolute_left / 2.0f)) * 2;
+ const int snapped_top =
+ base::saturated_cast<int16_t>(std::ceil(absolute_top / 2.0f)) * 2;
+ const int snapped_right =
+ base::saturated_cast<int16_t>(std::floor(absolute_right / 2.0f)) * 2;
+ const int snapped_bottom =
+ base::saturated_cast<int16_t>(std::floor(absolute_bottom / 2.0f)) * 2;
+ return gfx::Rect(snapped_left, snapped_top,
+ std::max(0, snapped_right - snapped_left),
+ std::max(0, snapped_bottom - snapped_top));
+}
+
+// Shrinks the given |rect| by the minimum amount necessary to align its corners
+// to even-numbered coordinates. |rect| is assumed to have non-negative values
+// for its coordinates.
+gfx::Rect MinimallyShrinkRectForI420(const gfx::Rect& rect) {
+ DCHECK(gfx::Rect(0, 0, media::limits::kMaxDimension,
+ media::limits::kMaxDimension)
+ .Contains(rect));
+ const int left = rect.x() + (rect.x() % 2);
+ const int top = rect.y() + (rect.y() % 2);
+ const int right = rect.right() - (rect.right() % 2);
+ const int bottom = rect.bottom() - (rect.bottom() % 2);
+ return gfx::Rect(left, top, std::max(0, right - left),
+ std::max(0, bottom - top));
+}
+
+} // namespace
+
+VideoCaptureOverlay::OnceRenderer VideoCaptureOverlay::MakeRenderer(
+ const gfx::Rect& region_in_frame,
+ const VideoPixelFormat frame_format,
+ const gfx::ColorSpace& frame_color_space) {
+ // If there's no image set yet, punt.
+ if (image_.drawsNothing()) {
+ return VideoCaptureOverlay::OnceRenderer();
+ }
+
+ // Determine the bounds of the sprite to be blitted onto the video frame. The
+ // calculations here align to the 2x2 pixel-quads, since dealing with
+ // fractions or partial I420 chroma plane alpha-blending would greatly
+ // complexify the blitting algorithm later on. This introduces a little
+ // inaccuracy in the size and position of the overlay in the final result, but
+ // should be an acceptable trade-off for all use cases.
+ const gfx::Rect bounds_in_frame =
+ ToAbsoluteBoundsForI420(bounds_, region_in_frame);
+ // If the sprite's size will be unreasonably large, punt.
+ if (bounds_in_frame.width() > media::limits::kMaxDimension ||
+ bounds_in_frame.height() > media::limits::kMaxDimension) {
+ return VideoCaptureOverlay::OnceRenderer();
+ }
+
+ // Compute the blit rect: the region of the frame to be modified by future
+ // Sprite::Blit() calls. First, |region_in_frame| must be shrunk to have
+ // even-valued coordinates to ensure the final blit rect is I420-friendly.
+ // Then, the shrunk |region_in_frame| is used to clip |bounds_in_frame|.
+ gfx::Rect blit_rect = MinimallyShrinkRectForI420(region_in_frame);
+ blit_rect.Intersect(bounds_in_frame);
+ // If the two rects didn't intersect at all (i.e., everything has been
+ // clipped), punt.
+ if (blit_rect.IsEmpty()) {
+ return VideoCaptureOverlay::OnceRenderer();
+ }
+
+ // If the cached sprite does not match the computed scaled size, pixel format,
+ // and/or color space, create a new instance for this (and future) renderers.
+ if (!sprite_ || sprite_->size() != bounds_in_frame.size() ||
+ sprite_->format() != frame_format ||
+ sprite_->color_space() != frame_color_space) {
+ sprite_ = base::MakeRefCounted<Sprite>(image_, bounds_in_frame.size(),
+ frame_format, frame_color_space);
+ }
+
+ return base::BindOnce(&Sprite::Blit, sprite_, bounds_in_frame.origin(),
+ blit_rect);
+}
+
+// static
+VideoCaptureOverlay::OnceRenderer VideoCaptureOverlay::MakeCombinedRenderer(
+ const std::vector<std::unique_ptr<VideoCaptureOverlay>>& overlays,
+ const gfx::Rect& region_in_frame,
+ const VideoPixelFormat frame_format,
+ const gfx::ColorSpace& frame_color_space) {
+ if (overlays.empty()) {
+ return VideoCaptureOverlay::OnceRenderer();
+ }
+
+ std::vector<OnceRenderer> renderers;
+ for (const std::unique_ptr<VideoCaptureOverlay>& overlay : overlays) {
+ renderers.emplace_back(overlay->MakeRenderer(region_in_frame, frame_format,
+ frame_color_space));
+ if (renderers.back().is_null()) {
+ renderers.pop_back();
+ }
+ }
+
+ if (renderers.empty()) {
+ return VideoCaptureOverlay::OnceRenderer();
+ }
+
+ return base::BindOnce(
+ [](std::vector<OnceRenderer> renderers, VideoFrame* frame) {
+ for (OnceRenderer& renderer : renderers) {
+ std::move(renderer).Run(frame);
+ }
+ },
+ std::move(renderers));
+}
+
+gfx::Rect VideoCaptureOverlay::ComputeSourceMutationRect() const {
+ if (!image_.drawsNothing() && !bounds_.IsEmpty()) {
+ const gfx::Size& source_size = frame_source_->GetSourceSize();
+ gfx::Rect result = gfx::ToEnclosingRect(
+ gfx::ScaleRect(bounds_, source_size.width(), source_size.height()));
+ result.Intersect(gfx::Rect(source_size));
+ return result;
+ }
+ return gfx::Rect();
+}
+
+VideoCaptureOverlay::Sprite::Sprite(const SkBitmap& image,
+ const gfx::Size& size,
+ const VideoPixelFormat format,
+ const gfx::ColorSpace& color_space)
+ : image_(image), size_(size), format_(format), color_space_(color_space) {
+ DCHECK(!image_.isNull());
+}
+
+VideoCaptureOverlay::Sprite::~Sprite() {
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+}
+
+namespace {
+
+// Returns the pointer to the element at the |offset| position, given a pointer
+// to the element for (0,0) in a row-major image plane.
+template <typename Pointer>
+Pointer PositionPointerInPlane(Pointer plane_begin,
+ int stride,
+ const gfx::Point& offset) {
+ return plane_begin + (offset.y() * stride) + offset.x();
+}
+
+// Returns the pointer to the element at the |offset| position, given a pointer
+// to the element for (0,0) in a row-major bitmap with 4 elements per pixel.
+template <typename Pointer>
+Pointer PositionPointerARGB(Pointer pixels_begin,
+ int stride,
+ const gfx::Point& offset) {
+ return pixels_begin + (offset.y() * stride) + (4 * offset.x());
+}
+
+// Transforms the lower 8 bits of |value| from the [0,255] range to the
+// normalized floating-point [0.0,1.0] range.
+float From255(uint8_t value) {
+ return value / 255.0f;
+}
+
+// Transforms the value from the normalized floating-point [0.0,1.0] range to an
+// unsigned int in the [0,255] range, capping any out-of-range values.
+uint32_t ToClamped255(float value) {
+ value = std::fma(value, 255.0f, 0.5f /* rounding */);
+ return base::saturated_cast<uint8_t>(value);
+}
+
+} // namespace
+
+void VideoCaptureOverlay::Sprite::Blit(const gfx::Point& position,
+ const gfx::Rect& blit_rect,
+ VideoFrame* frame) {
+ DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+ DCHECK(frame);
+ DCHECK_EQ(format_, frame->format());
+ DCHECK(frame->visible_rect().Contains(blit_rect));
+
+ TRACE_EVENT2("gpu.capture", "VideoCaptureOverlay::Sprite::Blit", "x",
+ position.x(), "y", position.y());
+
+ if (!transformed_image_) {
+ TransformImageOnce();
+ }
+
+ // Compute the left-most and top-most pixel to source from the transformed
+ // image. This is usually (0,0) unless only part of the sprite is being
+ // blitted (i.e., cropped at the edge(s) of the video frame).
+ gfx::Point src_origin = blit_rect.origin() - position.OffsetFromOrigin();
+ DCHECK(gfx::Rect(size_).Contains(gfx::Rect(src_origin, blit_rect.size())));
+
+ // Blit the sprite (src) onto the video frame (dest). One of two algorithms is
+ // used, depending on the video frame's format, as the blending calculations
+ // and data layout/format are different.
+ switch (frame->format()) {
+ case media::PIXEL_FORMAT_I420: {
+ // Core assumption: All coordinates are aligned to even-numbered
+ // coordinates.
+ DCHECK_EQ(src_origin.x() % 2, 0);
+ DCHECK_EQ(src_origin.y() % 2, 0);
+ DCHECK_EQ(blit_rect.x() % 2, 0);
+ DCHECK_EQ(blit_rect.y() % 2, 0);
+ DCHECK_EQ(blit_rect.width() % 2, 0);
+ DCHECK_EQ(blit_rect.height() % 2, 0);
+
+ // Helper function to execute a "SrcOver" blit from |src| to |dst|, and
+ // store the results back in |dst|.
+ const auto BlitOntoPlane = [](const gfx::Size& blit_size, int src_stride,
+ const float* src, const float* under_weight,
+ int dst_stride, uint8_t* dst) {
+ for (int row = 0; row < blit_size.height(); ++row, src += src_stride,
+ under_weight += src_stride, dst += dst_stride) {
+ for (int col = 0; col < blit_size.width(); ++col) {
+ dst[col] = ToClamped255(
+ std::fma(From255(dst[col]), under_weight[col], src[col]));
+ }
+ }
+ };
+
+ // Blit the Y plane: |src| points to the pre-multiplied luma values, while
+ // |under_weight| points to the "one minus src alpha" values. Both have
+ // the same stride, |src_stride|.
+ int src_stride = size_.width();
+ const float* under_weight = PositionPointerInPlane(
+ transformed_image_.get(), src_stride, src_origin);
+ const int num_pixels = size_.GetArea();
+ const float* src = under_weight + num_pixels;
+ // Likewise, start |dst| at the upper-left-most pixel within the video
+ // frame's Y plane that will be SrcOver'ed.
+ int dst_stride = frame->stride(VideoFrame::kYPlane);
+ uint8_t* dst =
+ PositionPointerInPlane(frame->visible_data(VideoFrame::kYPlane),
+ dst_stride, blit_rect.origin());
+ BlitOntoPlane(blit_rect.size(), src_stride, src, under_weight, dst_stride,
+ dst);
+
+ // Blit the U and V planes similarly to the Y plane, but reduce all
+ // coordinates by 2x2.
+ src_stride = size_.width() / 2;
+ src_origin = gfx::Point(src_origin.x() / 2, src_origin.y() / 2);
+ under_weight = PositionPointerInPlane(
+ transformed_image_.get() + 2 * num_pixels, src_stride, src_origin);
+ const int num_chroma_pixels = size_.GetArea() / 4;
+ src = under_weight + num_chroma_pixels;
+ dst_stride = frame->stride(VideoFrame::kUPlane);
+ const gfx::Rect chroma_blit_rect(blit_rect.x() / 2, blit_rect.y() / 2,
+ blit_rect.width() / 2,
+ blit_rect.height() / 2);
+ dst = PositionPointerInPlane(frame->visible_data(VideoFrame::kUPlane),
+ dst_stride, chroma_blit_rect.origin());
+ BlitOntoPlane(chroma_blit_rect.size(), src_stride, src, under_weight,
+ dst_stride, dst);
+ src += num_chroma_pixels;
+ dst_stride = frame->stride(VideoFrame::kVPlane);
+ dst = PositionPointerInPlane(frame->visible_data(VideoFrame::kVPlane),
+ dst_stride, chroma_blit_rect.origin());
+ BlitOntoPlane(chroma_blit_rect.size(), src_stride, src, under_weight,
+ dst_stride, dst);
+
+ break;
+ }
+
+ case media::PIXEL_FORMAT_ARGB: {
+ // Start |src| at the upper-left-most pixel within |transformed_image_|
+ // that will be blitted.
+ const int src_stride = size_.width() * 4;
+ const float* src =
+ PositionPointerARGB(transformed_image_.get(), src_stride, src_origin);
+
+ // Likewise, start |dst| at the upper-left-most pixel within the video
+ // frame that will be SrcOver'ed.
+ const int dst_stride = frame->stride(VideoFrame::kARGBPlane);
+ DCHECK_EQ(dst_stride % sizeof(uint32_t), 0u);
+ uint8_t* dst =
+ PositionPointerARGB(frame->visible_data(VideoFrame::kARGBPlane),
+ dst_stride, blit_rect.origin());
+ DCHECK_EQ((dst - frame->visible_data(VideoFrame::kARGBPlane)) %
+ sizeof(uint32_t),
+ 0u);
+
+ // Blend each sprite pixel over the corresponding pixel in the video
+ // frame, and store the result back in the video frame. Note that the
+ // video frame format does NOT have color values pre-multiplied by the
+ // alpha.
+ for (int row = 0; row < blit_rect.height();
+ ++row, src += src_stride, dst += dst_stride) {
+ uint32_t* dst_pixel = reinterpret_cast<uint32_t*>(dst);
+ for (int col = 0; col < blit_rect.width(); ++col) {
+ const int src_idx = 4 * col;
+ const float src_alpha = src[src_idx];
+ const float dst_weight =
+ From255(dst_pixel[col] >> 24) * (1.0f - src_alpha);
+ const float out_alpha = src_alpha + dst_weight;
+ float out_red = std::fma(From255(dst_pixel[col] >> 16), dst_weight,
+ src[src_idx + 1]);
+ float out_green = std::fma(From255(dst_pixel[col] >> 8), dst_weight,
+ src[src_idx + 2]);
+ float out_blue = std::fma(From255(dst_pixel[col] >> 0), dst_weight,
+ src[src_idx + 3]);
+ if (out_alpha != 0.0f) {
+ out_red /= out_alpha;
+ out_green /= out_alpha;
+ out_blue /= out_alpha;
+ }
+ dst_pixel[col] =
+ ((ToClamped255(out_alpha) << 24) | (ToClamped255(out_red) << 16) |
+ (ToClamped255(out_green) << 8) | (ToClamped255(out_blue) << 0));
+ }
+ }
+
+ break;
+ }
+
+ default:
+ NOTREACHED();
+ break;
+ }
+}
+
+void VideoCaptureOverlay::Sprite::TransformImageOnce() {
+ TRACE_EVENT2("gpu.capture", "VideoCaptureOverlay::Sprite::TransformImageOnce",
+ "width", size_.width(), "height", size_.height());
+
+ // Scale the source |image_| to match the format and size required. For the
+ // purposes of color space conversion, the alpha must not be pre-multiplied.
+ const SkImageInfo scaled_image_format =
+ SkImageInfo::Make(size_.width(), size_.height(), kN32_SkColorType,
+ kUnpremul_SkAlphaType, image_.refColorSpace());
+ SkBitmap scaled_image;
+ if (image_.info() == scaled_image_format) {
+ scaled_image = image_;
+ } else {
+ if (!scaled_image.tryAllocPixels(scaled_image_format) ||
+ !image_.pixmap().scalePixels(scaled_image.pixmap(),
+ kMedium_SkFilterQuality)) {
+ // If the allocation, format conversion and/or scaling failed, just reset
+ // the |scaled_image|. This will be checked below.
+ scaled_image.reset();
+ }
+ }
+
+ gfx::ColorSpace image_color_space;
+ if (scaled_image.colorSpace()) {
+ image_color_space = gfx::ColorSpace(*scaled_image.colorSpace());
+ DCHECK(image_color_space.IsValid());
+ } else {
+ // Assume a default linear color space, if no color space was provided.
+ DCHECK(!image_.colorSpace()); // Skia should have set it!
+ image_color_space = gfx::ColorSpace(
+ gfx::ColorSpace::PrimaryID::BT709, gfx::ColorSpace::TransferID::LINEAR,
+ gfx::ColorSpace::MatrixID::RGB, gfx::ColorSpace::RangeID::FULL);
+ }
+
+ // The source image is no longer needed. Reset it to dereference pixel memory.
+ image_.reset();
+
+ // Populate |colors| and |alphas| from the |scaled_image|. If the image
+ // scaling operation failed, this sprite should draw nothing, and so fully
+ // transparent pixels will be generated instead.
+ const int num_pixels = size_.GetArea();
+ std::unique_ptr<float[]> alphas(new float[num_pixels]);
+ std::unique_ptr<gfx::ColorTransform::TriStim[]> colors(
+ new gfx::ColorTransform::TriStim[num_pixels]);
+ if (scaled_image.drawsNothing()) {
+ std::fill(alphas.get(), alphas.get() + num_pixels, 0.0f);
+ std::fill(colors.get(), colors.get() + num_pixels,
+ gfx::ColorTransform::TriStim());
+ } else {
+ int pos = 0;
+ for (int y = 0; y < size_.height(); ++y) {
+ const uint32_t* src = scaled_image.getAddr32(0, y);
+ for (int x = 0; x < size_.width(); ++x) {
+ const uint32_t pixel = src[x];
+ alphas[pos] = ((pixel >> SK_A32_SHIFT) & 0xff) / 255.0f;
+ colors[pos].SetPoint(((pixel >> SK_R32_SHIFT) & 0xff) / 255.0f,
+ ((pixel >> SK_G32_SHIFT) & 0xff) / 255.0f,
+ ((pixel >> SK_B32_SHIFT) & 0xff) / 255.0f);
+ ++pos;
+ }
+ }
+ }
+
+ // Transform the colors, if needed. This may perform RGB→YUV conversion.
+ if (image_color_space != color_space_) {
+ const auto color_transform = gfx::ColorTransform::NewColorTransform(
+ image_color_space, color_space_,
+ gfx::ColorTransform::Intent::INTENT_ABSOLUTE);
+ color_transform->Transform(colors.get(), num_pixels);
+ }
+
+ switch (format_) {
+ case media::PIXEL_FORMAT_I420: {
+ // Produce 5 planes of data: The "one minus alpha" plane, the Y plane, the
+ // subsampled "one minus alpha" plane, the U plane, and the V plane.
+ // Pre-multiply the colors by the alpha to prevent extra work in multiple
+ // later Blit() calls.
+ DCHECK_EQ(size_.width() % 2, 0);
+ DCHECK_EQ(size_.height() % 2, 0);
+ const int num_chroma_pixels = size_.GetArea() / 4;
+ transformed_image_.reset(
+ new float[num_pixels * 2 + num_chroma_pixels * 3]);
+
+ // Copy the alpha values, and pre-multiply the luma values by the alpha.
+ float* out_1_minus_alpha = transformed_image_.get();
+ float* out_luma = out_1_minus_alpha + num_pixels;
+ for (int i = 0; i < num_pixels; ++i) {
+ const float alpha = alphas[i];
+ out_1_minus_alpha[i] = 1.0f - alpha;
+ out_luma[i] = colors[i].x() * alpha;
+ }
+
+ // Downscale the alpha, U, and V planes by 2x2, and pre-multiply the
+ // chroma values by the alpha.
+ float* out_uv_1_minus_alpha = out_luma + num_pixels;
+ float* out_u = out_uv_1_minus_alpha + num_chroma_pixels;
+ float* out_v = out_u + num_chroma_pixels;
+ const float* alpha_row0 = alphas.get();
+ const float* const alpha_row_end = alpha_row0 + num_pixels;
+ const gfx::ColorTransform::TriStim* color_row0 = colors.get();
+ while (alpha_row0 < alpha_row_end) {
+ const float* alpha_row1 = alpha_row0 + size_.width();
+ const gfx::ColorTransform::TriStim* color_row1 =
+ color_row0 + size_.width();
+ for (int col = 0; col < size_.width(); col += 2) {
+ // First, the downscaled alpha is the average of the four original
+ // alpha values:
+ //
+ // sum_of_alphas = a[r,c] + a[r,c+1] + a[r+1,c] + a[r+1,c+1];
+ // average_alpha = sum_of_alphas / 4
+ *(out_uv_1_minus_alpha++) =
+ std::fma(alpha_row0[col] + alpha_row0[col + 1] + alpha_row1[col] +
+ alpha_row1[col + 1],
+ -1.0f / 4.0f, 1.0f);
+ // Then, the downscaled chroma values are the weighted average of the
+ // four original chroma values (weighed by alpha):
+ //
+ // weighted_sum_of_chromas =
+ // c[r,c]*a[r,c] + c[r,c+1]*a[r,c+1] +
+ // c[r+1,c]*a[r+1,c] + c[r+1,c+1]*a[r+1,c+1]
+ // sum_of_weights = sum_of_alphas;
+ // average_chroma = weighted_sum_of_chromas / sum_of_weights
+ //
+ // But then, because the chroma is to be pre-multiplied by the alpha,
+ // the calculations simplify, as follows:
+ //
+ // premul_chroma = average_chroma * average_alpha
+ // = (weighted_sum_of_chromas / sum_of_alphas) *
+ // (sum_of_alphas / 4)
+ // = weighted_sum_of_chromas / 4
+ //
+ // This also automatically solves a special case, when sum_of_alphas
+ // is zero: With the simplified calculations, there is no longer a
+ // "divide-by-zero guard" needed; and the result in this case will be
+ // a zero chroma, which is perfectly acceptable behavior.
+ *(out_u++) = ((color_row0[col].y() * alpha_row0[col]) +
+ (color_row0[col + 1].y() * alpha_row0[col + 1]) +
+ (color_row1[col].y() * alpha_row1[col]) +
+ (color_row1[col + 1].y() * alpha_row1[col + 1])) /
+ 4.0f;
+ *(out_v++) = ((color_row0[col].z() * alpha_row0[col]) +
+ (color_row0[col + 1].z() * alpha_row0[col + 1]) +
+ (color_row1[col].z() * alpha_row1[col]) +
+ (color_row1[col + 1].z() * alpha_row1[col + 1])) /
+ 4.0f;
+ }
+ alpha_row0 = alpha_row1 + size_.width();
+ color_row0 = color_row1 + size_.width();
+ }
+
+ break;
+ }
+
+ case media::PIXEL_FORMAT_ARGB: {
+ // Produce ARGB pixels from |colors| and |alphas|. Pre-multiply the colors
+ // by the alpha to prevent extra work in multiple later Blit() calls.
+ transformed_image_.reset(new float[num_pixels * 4]);
+ float* out = transformed_image_.get();
+ for (int i = 0; i < num_pixels; ++i) {
+ const float alpha = alphas[i];
+ *(out++) = alpha;
+ *(out++) = colors[i].x() * alpha;
+ *(out++) = colors[i].y() * alpha;
+ *(out++) = colors[i].z() * alpha;
+ }
+ break;
+ }
+
+ default:
+ NOTREACHED();
+ break;
+ }
+}
+
+} // namespace viz
diff --git a/components/viz/service/frame_sinks/video_capture/video_capture_overlay.h b/components/viz/service/frame_sinks/video_capture/video_capture_overlay.h
new file mode 100644
index 0000000..1d39a15
--- /dev/null
+++ b/components/viz/service/frame_sinks/video_capture/video_capture_overlay.h
@@ -0,0 +1,186 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_
+#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_
+
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/sequence_checker.h"
+#include "components/viz/service/viz_service_export.h"
+#include "media/base/video_types.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "ui/gfx/color_space.h"
+#include "ui/gfx/color_transform.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/rect_f.h"
+#include "ui/gfx/geometry/size.h"
+
+namespace media {
+class VideoFrame;
+}
+
+namespace viz {
+
+// An overlay image to be blitted onto video frames. A mojo client sets the
+// image, position, and size of the overlay in the video frame; and then this
+// VideoCaptureOverlay scales the image and maps its color space to match that
+// of the video frame before the blitting.
+//
+// As an optimization, the client's bitmap image is transformed (scaled, color
+// space converted, and pre-multiplied by alpha), and then this cached Sprite is
+// re-used for blitting to all successive video frames until some change
+// requires a different transformation. MakeRenderer() produces a Renderer
+// callback that holds a reference to an existing Sprite, or will create a new
+// one if necessary. The Renderer callback can then be run at any point in the
+// future, unaffected by later image, size, or color space settings changes.
+//
+// The blit algorithm uses naive linear blending. Thus, the use of non-linear
+// color spaces will cause loses in color accuracy.
+//
+// TODO(crbug.com/810133): Override the mojom::FrameSinkVideoCaptureOverlay
+// interface.
+class VIZ_SERVICE_EXPORT VideoCaptureOverlay {
+ public:
+ // Interface for notifying the frame source when changes to the overlay's
+ // state occur.
+ class VIZ_SERVICE_EXPORT FrameSource {
+ public:
+ // Returns the current size of the source, or empty if unknown.
+ virtual gfx::Size GetSourceSize() = 0;
+
+ // Notifies the FrameSource that the given source |rect| needs to be
+ // re-captured soon. One or more calls to this method will be followed-up
+ // with a call to RequestRefreshFrame().
+ virtual void InvalidateRect(const gfx::Rect& rect) = 0;
+
+ // Notifies the FrameSource that another frame should be captured and have
+ // its VideoCaptureOverlay re-rendered soon to reflect an updated overlay
+ // image and/or position.
+ virtual void RequestRefreshFrame() = 0;
+
+ protected:
+ virtual ~FrameSource();
+ };
+
+ // A OnceCallback that, when run, renders the overlay on a VideoFrame.
+ using OnceRenderer = base::OnceCallback<void(media::VideoFrame*)>;
+
+ // |frame_source| must outlive this instance.
+ explicit VideoCaptureOverlay(FrameSource* frame_source);
+
+ ~VideoCaptureOverlay();
+
+ // Sets/Changes the overlay |image| and its position and size, relative to the
+ // source content. |bounds| consists of coordinates where the range [0.0,1.0)
+ // indicates the relative position+size within the bounds of the source
+ // content (e.g., 0.0 refers to the top or left edge; 1.0 to just after the
+ // bottom or right edge). Pass empty |bounds| to temporarily hide the overlay
+ // until a later call to SetBounds().
+ void SetImageAndBounds(const SkBitmap& image, const gfx::RectF& bounds);
+ void SetBounds(const gfx::RectF& bounds);
+
+ // Returns a OnceCallback that, when run, renders this VideoCaptureOverlay on
+ // a VideoFrame. The overlay's position and size are computed based on the
+ // given content |region_in_frame|, and its color space is converted to match
+ // the |frame_color_space|. Returns a null OnceCallback if there is nothing to
+ // render at this time.
+ OnceRenderer MakeRenderer(const gfx::Rect& region_in_frame,
+ const media::VideoPixelFormat frame_format,
+ const gfx::ColorSpace& frame_color_space);
+
+ // Returns a OnceCallback that renders all of the given |overlays| in
+ // order. The remaining arguments are the same as in MakeRenderer(). This is a
+ // convenience that produces a single callback, so that client code need not
+ // deal with collections of callbacks. Returns a null OnceCallback if there is
+ // nothing to render at this time.
+ static OnceRenderer MakeCombinedRenderer(
+ const std::vector<std::unique_ptr<VideoCaptureOverlay>>& overlays,
+ const gfx::Rect& region_in_frame,
+ const media::VideoPixelFormat frame_format,
+ const gfx::ColorSpace& frame_color_space);
+
+ private:
+ // Transforms the overlay SkBitmap image by scaling and converting its color
+ // space, and then blitting it onto a VideoFrame. The transformation is lazy:
+ // Meaning, a reference to the SkBitmap image is held until the first call to
+ // Blit(), where the transformation is then executed and the reference to the
+ // original SkBitmap dropped. The transformed data is then cached for re-use
+ // for later Blit() calls.
+ class Sprite : public base::RefCounted<Sprite> {
+ public:
+ Sprite(const SkBitmap& image,
+ const gfx::Size& size,
+ const media::VideoPixelFormat format,
+ const gfx::ColorSpace& color_space);
+
+ const gfx::Size& size() const { return size_; }
+ media::VideoPixelFormat format() const { return format_; }
+ const gfx::ColorSpace& color_space() const { return color_space_; }
+
+ void Blit(const gfx::Point& position,
+ const gfx::Rect& blit_rect,
+ media::VideoFrame* frame);
+
+ private:
+ friend class base::RefCounted<Sprite>;
+ ~Sprite();
+
+ void TransformImageOnce();
+
+ // As Sprites can be long-lived and hidden from external code within
+ // callbacks, ensure that all Blit() calls are in-sequence.
+ SEQUENCE_CHECKER(sequence_checker_);
+
+ // If not null, this is the original, unscaled overlay image. After
+ // TransformImageOnce() has been called, this is set to null.
+ SkBitmap image_;
+
+ // The size, format, and color space of the cached transformed image.
+ const gfx::Size size_;
+ const media::VideoPixelFormat format_;
+ const gfx::ColorSpace color_space_;
+
+ // The transformed source image data. For blitting to ARGB format video
+ // frames, the source image data will consist of 4 elements per pixel pixel
+ // (A, R, G, B). For blitting to the I420 format, the source image data is
+ // not interlaved: Instead, there are 5 planes of data (one minus alpha, Y,
+ // subsampled one minus alpha, U, V). For both formats, the color components
+ // are premultiplied for more-efficient Blit()'s.
+ std::unique_ptr<float[]> transformed_image_;
+
+ DISALLOW_COPY_AND_ASSIGN(Sprite);
+ };
+
+ // Computes the region of the source that, if changed, would require
+ // re-rendering the overlay.
+ gfx::Rect ComputeSourceMutationRect() const;
+
+ FrameSource* const frame_source_;
+
+ // The currently-set overlay image.
+ SkBitmap image_;
+
+ // If empty, the overlay is currently hidden. Otherwise, this consists of
+ // coordinates where the range [0.0,1.0) indicates the relative position+size
+ // within the bounds of the video frame's content region (e.g., 0.0 refers to
+ // the top or left edge; 1.0 to just after the bottom or right edge).
+ gfx::RectF bounds_;
+
+ // The current Sprite. This is set to null whenever a settings change requires
+ // a new Sprite to be generated from the |image_|.
+ scoped_refptr<Sprite> sprite_;
+
+ DISALLOW_COPY_AND_ASSIGN(VideoCaptureOverlay);
+};
+
+} // namespace viz
+
+#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_VIDEO_CAPTURE_OVERLAY_H_
diff --git a/components/viz/service/frame_sinks/video_capture/video_capture_overlay_unittest.cc b/components/viz/service/frame_sinks/video_capture/video_capture_overlay_unittest.cc
new file mode 100644
index 0000000..2bb5112
--- /dev/null
+++ b/components/viz/service/frame_sinks/video_capture/video_capture_overlay_unittest.cc
@@ -0,0 +1,554 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/viz/service/frame_sinks/video_capture/video_capture_overlay.h"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/optional.h"
+#include "base/path_service.h"
+#include "base/run_loop.h"
+#include "base/stl_util.h"
+#include "cc/test/pixel_comparator.h"
+#include "cc/test/pixel_test_utils.h"
+#include "components/viz/test/paths.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_types.h"
+#include "media/base/video_util.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "third_party/skia/include/core/SkColor.h"
+#include "third_party/skia/include/core/SkImageInfo.h"
+#include "third_party/skia/include/core/SkPixmap.h"
+#include "ui/gfx/color_space.h"
+#include "ui/gfx/color_transform.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/rect_f.h"
+#include "ui/gfx/geometry/size.h"
+
+using media::VideoFrame;
+using media::VideoPixelFormat;
+
+using testing::_;
+using testing::InvokeWithoutArgs;
+using testing::NiceMock;
+using testing::Return;
+using testing::StrictMock;
+
+namespace viz {
+namespace {
+
+class MockFrameSource : public VideoCaptureOverlay::FrameSource {
+ public:
+ MOCK_METHOD0(GetSourceSize, gfx::Size());
+ MOCK_METHOD1(InvalidateRect, void(const gfx::Rect& rect));
+ MOCK_METHOD0(RequestRefreshFrame, void());
+};
+
+class VideoCaptureOverlayTest : public testing::Test {
+ public:
+ VideoCaptureOverlayTest() = default;
+
+ NiceMock<MockFrameSource>* frame_source() { return &frame_source_; }
+
+ std::unique_ptr<VideoCaptureOverlay> CreateOverlay() {
+ return std::make_unique<VideoCaptureOverlay>(frame_source());
+ }
+
+ void RunUntilIdle() { base::RunLoop().RunUntilIdle(); }
+
+ // Makes a SkBitmap filled with a 50% white background color plus four rects
+ // of four different colors/opacities. |cycle| causes the four rects to rotate
+ // positions (counter-clockwise by N steps).
+ static SkBitmap MakeTestBitmap(int cycle) {
+ constexpr gfx::Size kTestImageSize = gfx::Size(24, 16);
+ // Test colors have been chosen to exercise different opacities,
+ // intensities, and color channels; to confirm all aspects of the "SrcOver"
+ // image blending algorithms are working properly.
+ constexpr SkColor kTestImageBackground =
+ SkColorSetARGB(0xff, 0xff, 0xff, 0xff);
+ constexpr SkColor kTestImageColors[4] = {
+ SkColorSetARGB(0xaa, 0xff, 0x00, 0x00),
+ SkColorSetARGB(0xbb, 0x00, 0xee, 0x00),
+ SkColorSetARGB(0xcc, 0x00, 0x00, 0x77),
+ SkColorSetARGB(0xdd, 0x66, 0x66, 0x00),
+ };
+ constexpr SkIRect kTestImageColorRects[4] = {
+ SkIRect::MakeXYWH(4, 2, 4, 4), SkIRect::MakeXYWH(16, 2, 4, 4),
+ SkIRect::MakeXYWH(4, 10, 4, 4), SkIRect::MakeXYWH(16, 10, 4, 4),
+ };
+
+ SkBitmap result;
+ const SkImageInfo info = SkImageInfo::MakeN32Premul(
+ kTestImageSize.width(), kTestImageSize.height(),
+ GetLinearSRGB().ToSkColorSpace());
+ CHECK(result.tryAllocPixels(info, info.minRowBytes()));
+ result.eraseColor(kTestImageBackground);
+ for (size_t i = 0; i < base::size(kTestImageColors); ++i) {
+ const size_t idx = (i + cycle) % base::size(kTestImageColors);
+ result.erase(kTestImageColors[idx], kTestImageColorRects[i]);
+ }
+
+ return result;
+ }
+
+ // Returns the sRGB color space, but with a linear transfer function.
+ static gfx::ColorSpace GetLinearSRGB() {
+ return gfx::ColorSpace(
+ gfx::ColorSpace::PrimaryID::BT709, gfx::ColorSpace::TransferID::LINEAR,
+ gfx::ColorSpace::MatrixID::RGB, gfx::ColorSpace::RangeID::FULL);
+ }
+
+ // Returns the BT709 color space (YUV), but with a linear transfer function.
+ static gfx::ColorSpace GetLinearRec709() {
+ return gfx::ColorSpace(
+ gfx::ColorSpace::PrimaryID::BT709, gfx::ColorSpace::TransferID::LINEAR,
+ gfx::ColorSpace::MatrixID::BT709, gfx::ColorSpace::RangeID::LIMITED);
+ }
+
+ static constexpr auto kARGBFormat = VideoPixelFormat::PIXEL_FORMAT_ARGB;
+ static constexpr auto kI420Format = VideoPixelFormat::PIXEL_FORMAT_I420;
+
+ private:
+ NiceMock<MockFrameSource> frame_source_;
+
+ DISALLOW_COPY_AND_ASSIGN(VideoCaptureOverlayTest);
+};
+
+// Tests that MakeRenderer() does not make a OnceRenderer until the client has
+// set the image.
+TEST_F(VideoCaptureOverlayTest, DoesNotRenderWithoutImage) {
+ constexpr gfx::Size kSize = gfx::Size(100, 75);
+ EXPECT_CALL(*frame_source(), GetSourceSize()).WillRepeatedly(Return(kSize));
+ std::unique_ptr<VideoCaptureOverlay> overlay = CreateOverlay();
+
+ // The overlay does not have an image yet, so the renderer should be null.
+ constexpr gfx::Rect kRegionInFrame = gfx::Rect(kSize);
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // Once an image is set, the renderer should not be null.
+ overlay->SetImageAndBounds(MakeTestBitmap(1), gfx::RectF(0, 0, 1, 1));
+ EXPECT_TRUE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+}
+
+// Tests that MakeRenderer() does not make a OnceRenderer if the bounds are set
+// to something outside the frame's content region.
+TEST_F(VideoCaptureOverlayTest, DoesNotRenderIfCompletelyOutOfBounds) {
+ constexpr gfx::Size kSize = gfx::Size(100, 75);
+ EXPECT_CALL(*frame_source(), GetSourceSize()).WillRepeatedly(Return(kSize));
+ std::unique_ptr<VideoCaptureOverlay> overlay = CreateOverlay();
+
+ // The overlay does not have an image yet, so the renderer should be null.
+ constexpr gfx::Rect kRegionInFrame = gfx::Rect(kSize);
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // Setting an image, but out-of-bounds, should always result in a null
+ // renderer.
+ overlay->SetImageAndBounds(MakeTestBitmap(0), gfx::RectF(-1, -1, 1, 1));
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+ overlay->SetBounds(gfx::RectF(1, 1, 1, 1));
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+ overlay->SetBounds(gfx::RectF(-1, 1, 1, 1));
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+ overlay->SetBounds(gfx::RectF(1, -1, 1, 1));
+ EXPECT_FALSE(
+ overlay->MakeRenderer(kRegionInFrame, kI420Format, GetLinearRec709()));
+}
+
+// Tests that that MakeCombinedRenderer() only makes a OnceRenderer when one or
+// more overlays are set to make visible changes to a video frame.
+TEST_F(VideoCaptureOverlayTest,
+ DoesNotDoCombinedRenderIfNoOverlaysWouldRender) {
+ constexpr gfx::Size kSize = gfx::Size(100, 75);
+ EXPECT_CALL(*frame_source(), GetSourceSize()).WillRepeatedly(Return(kSize));
+ std::vector<std::unique_ptr<VideoCaptureOverlay>> overlays;
+ overlays.emplace_back(CreateOverlay());
+ overlays.emplace_back(CreateOverlay());
+
+ // Neither overlay has an image yet, so the combined renderer should be null.
+ constexpr gfx::Rect kRegionInFrame = gfx::Rect(kSize);
+ EXPECT_FALSE(VideoCaptureOverlay::MakeCombinedRenderer(
+ overlays, kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // If just the first overlay renders, the combined renderer should not be
+ // null.
+ overlays[0]->SetImageAndBounds(MakeTestBitmap(0), gfx::RectF(0, 0, 1, 1));
+ EXPECT_TRUE(VideoCaptureOverlay::MakeCombinedRenderer(
+ overlays, kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // If both overlays render, the combined renderer should not be null.
+ overlays[1]->SetImageAndBounds(MakeTestBitmap(1), gfx::RectF(0, 0, 1, 1));
+ EXPECT_TRUE(VideoCaptureOverlay::MakeCombinedRenderer(
+ overlays, kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // If only the second overlay renders, because the first is hidden, the
+ // combined renderer should not be null.
+ overlays[0]->SetBounds(gfx::RectF());
+ EXPECT_TRUE(VideoCaptureOverlay::MakeCombinedRenderer(
+ overlays, kRegionInFrame, kI420Format, GetLinearRec709()));
+
+ // Both overlays are hidden, so the combined renderer should be null.
+ overlays[1]->SetBounds(gfx::RectF());
+ EXPECT_FALSE(VideoCaptureOverlay::MakeCombinedRenderer(
+ overlays, kRegionInFrame, kI420Format, GetLinearRec709()));
+}
+
+class VideoCaptureOverlayRenderTest
+ : public VideoCaptureOverlayTest,
+ public testing::WithParamInterface<VideoPixelFormat> {
+ public:
+ VideoCaptureOverlayRenderTest()
+ : trace_(__FILE__, __LINE__, VideoPixelFormatToString(pixel_format())) {}
+
+ VideoPixelFormat pixel_format() const { return GetParam(); }
+
+ bool is_argb_test() const {
+ return pixel_format() == media::PIXEL_FORMAT_ARGB;
+ }
+
+ gfx::ColorSpace GetColorSpace() const {
+ // For these tests, we use linear RGB and YUV color spaces. This is because
+ // VideoCaptureOverlay does not account for non-linear color spaces when
+ // blending. See class notes.
+ return is_argb_test() ? GetLinearSRGB() : GetLinearRec709();
+ }
+
+ scoped_refptr<VideoFrame> CreateVideoFrame(const gfx::Size& size) const {
+ auto frame = VideoFrame::CreateFrame(pixel_format(), size, gfx::Rect(size),
+ size, base::TimeDelta());
+
+ // Fill the video frame with black. For ARGB tests, also set alpha channel
+ // to 1.0. This allows the expected results of the ARGB tests to be the same
+ // as those of the YUV tests, and so only one set of golden files needs to
+ // be used.
+ if (is_argb_test()) {
+ uint8_t* dst = frame->visible_data(VideoFrame::kARGBPlane);
+ const int stride = frame->stride(VideoFrame::kARGBPlane);
+ for (int row = 0; row < size.height(); ++row, dst += stride) {
+ uint32_t* const begin = reinterpret_cast<uint32_t*>(dst);
+ std::fill(begin, begin + size.width(), UINT32_C(0xff000000));
+ }
+ } else /* if (!is_argb_test()) */ {
+ media::FillYUV(frame.get(), 0x00, 0x80, 0x80);
+ }
+
+ frame->set_color_space(GetColorSpace());
+ return frame;
+ }
+
+ bool FrameMatchesPNG(const VideoFrame& frame, const char* golden_file) {
+ const gfx::ColorSpace png_color_space = GetLinearSRGB();
+ // Note: Using kUnpremul_SkAlphaType since that is the semantics of
+ // PIXEL_FORMAT_ARGB, and converting to kPremul_SkAlphaType before producing
+ // the PNG would lose precision for no good reason.
+ const SkImageInfo canonical_format = SkImageInfo::Make(
+ frame.visible_rect().width(), frame.visible_rect().height(),
+ kN32_SkColorType, kUnpremul_SkAlphaType,
+ png_color_space.ToSkColorSpace());
+ SkBitmap canonical_bitmap;
+ CHECK(canonical_bitmap.tryAllocPixels(canonical_format, 0));
+
+ // Populate |canonical_bitmap| with data from the frame. For I420, use
+ // gfx::ColorTransform to map back from YUV→RGB.
+ switch (frame.format()) {
+ case media::PIXEL_FORMAT_ARGB: {
+ // Map from the video frame's ARGB format to the canonical
+ // representation.
+ const SkImageInfo frame_format = SkImageInfo::Make(
+ frame.visible_rect().width(), frame.visible_rect().height(),
+ kBGRA_8888_SkColorType, kUnpremul_SkAlphaType,
+ frame.ColorSpace().ToSkColorSpace());
+ canonical_bitmap.writePixels(
+ SkPixmap(frame_format, frame.visible_data(VideoFrame::kARGBPlane),
+ frame.stride(VideoFrame::kARGBPlane)),
+ 0, 0);
+ break;
+ }
+
+ case media::PIXEL_FORMAT_I420: {
+ // Map from I420 planar [0,255] (of which only [16,235] is used) values
+ // to interleaved [0.0,1.0] values.
+ const gfx::Size& size = frame.visible_rect().size();
+ std::unique_ptr<gfx::ColorTransform::TriStim[]> colors(
+ new gfx::ColorTransform::TriStim[size.GetArea()]);
+ int pos = 0;
+ for (int row = 0; row < size.height(); ++row) {
+ const uint8_t* y = frame.visible_data(VideoFrame::kYPlane) +
+ (row * frame.stride(VideoFrame::kYPlane));
+ const uint8_t* u = frame.visible_data(VideoFrame::kUPlane) +
+ ((row / 2) * frame.stride(VideoFrame::kUPlane));
+ const uint8_t* v = frame.visible_data(VideoFrame::kVPlane) +
+ ((row / 2) * frame.stride(VideoFrame::kVPlane));
+ for (int col = 0; col < size.width(); ++col) {
+ colors[pos].SetPoint(y[col] / 255.0f, u[col / 2] / 255.0f,
+ v[col / 2] / 255.0f);
+ ++pos;
+ }
+ }
+
+ // Execute the YUV→RGB conversion.
+ gfx::ColorTransform::NewColorTransform(
+ frame.ColorSpace(), png_color_space,
+ gfx::ColorTransform::Intent::INTENT_ABSOLUTE)
+ ->Transform(colors.get(), size.GetArea());
+
+ // Map back from interleaved [0.0,1.0] values to intervealed ARGB,
+ // setting alpha=100%.
+ const auto ToClamped255 = [](float value) -> uint32_t {
+ value = (value * 255.0f) + 0.5f /* rounding */;
+ return base::saturated_cast<uint8_t>(value);
+ };
+ pos = 0;
+ for (int row = 0; row < size.height(); ++row) {
+ uint32_t* out = canonical_bitmap.getAddr32(0, row);
+ for (int col = 0; col < size.width(); ++col) {
+ out[col] = ((UINT32_C(255) << SK_A32_SHIFT) |
+ (ToClamped255(colors[pos].x()) << SK_R32_SHIFT) |
+ (ToClamped255(colors[pos].y()) << SK_G32_SHIFT) |
+ (ToClamped255(colors[pos].z()) << SK_B32_SHIFT));
+ ++pos;
+ }
+ }
+
+ break;
+ }
+
+ default:
+ NOTREACHED();
+ return false;
+ }
+
+ // Determine the full path to the golden file to compare the results.
+ base::FilePath golden_file_path;
+ base::PathService::Get(Paths::DIR_TEST_DATA, &golden_file_path);
+ golden_file_path =
+ golden_file_path.Append(FILE_PATH_LITERAL("video_capture"))
+ .Append(base::FilePath::FromUTF8Unsafe(golden_file));
+
+ // If the very-specific command-line switch is present, rewrite the golden
+ // file. This is only done when the ARGB test runs, for the reasons outlined
+ // in the comments below (regarding FuzzyPixelComparator).
+ if (is_argb_test() &&
+ base::CommandLine::ForCurrentProcess()->HasSwitch(
+ "video-overlay-capture-test-update-golden-files")) {
+ LOG(INFO) << "Rewriting golden file: " << golden_file_path.AsUTF8Unsafe();
+ cc::WritePNGFile(canonical_bitmap, golden_file_path, false);
+ }
+
+ // FuzzyPixelComparator configuration: Allow 100% of pixels to mismatch, but
+ // no single pixel component should be different by more than 1/255 (64/255
+ // for YUV tests), and the absolute average error must not exceed 1/255
+ // (16/255 for YUV tests). The YUV tests allow for more error due to the
+ // expected errors introduced by both color space (dynamic range) and format
+ // (chroma subsampling) conversion.
+ cc::FuzzyPixelComparator comparator(false, 100.0f, 0.0f,
+ is_argb_test() ? 1.0f : 16.0f,
+ is_argb_test() ? 1 : 64, 0);
+ const bool matches_golden_file =
+ cc::MatchesPNGFile(canonical_bitmap, golden_file_path, comparator);
+ // If MatchesPNGFile() returned false, it will have LOG(ERROR)'ed the
+ // expected versus actual PNG data URLs. So, only do the VLOG(1)'s when
+ // MatchesPNGFile() returned true.
+ if (matches_golden_file && VLOG_IS_ON(1)) {
+ SkBitmap expected;
+ if (cc::ReadPNGFile(golden_file_path, &expected)) {
+ VLOG(1) << "Expected bitmap: " << cc::GetPNGDataUrl(expected);
+ }
+ VLOG(1) << "Actual bitmap: " << cc::GetPNGDataUrl(canonical_bitmap);
+ }
+ return matches_golden_file;
+ }
+
+ // The size of the compositor frame sink's Surface.
+ static constexpr gfx::Size kSourceSize = gfx::Size(96, 40);
+
+ private:
+ testing::ScopedTrace trace_;
+
+ DISALLOW_COPY_AND_ASSIGN(VideoCaptureOverlayRenderTest);
+};
+
+// static
+constexpr gfx::Size VideoCaptureOverlayRenderTest::kSourceSize;
+
+// Basic test: Render an overlay image that covers the entire video frame and is
+// not scaled.
+TEST_P(VideoCaptureOverlayRenderTest, FullCover_NoScaling) {
+ StrictMock<MockFrameSource> frame_source;
+ VideoCaptureOverlay overlay(&frame_source);
+
+ EXPECT_CALL(frame_source, GetSourceSize())
+ .WillRepeatedly(Return(kSourceSize));
+ EXPECT_CALL(frame_source, InvalidateRect(gfx::Rect())).RetiresOnSaturation();
+ EXPECT_CALL(frame_source, InvalidateRect(gfx::Rect(kSourceSize)))
+ .RetiresOnSaturation();
+ EXPECT_CALL(frame_source, RequestRefreshFrame());
+
+ const SkBitmap test_bitmap = MakeTestBitmap(0);
+ overlay.SetImageAndBounds(test_bitmap, gfx::RectF(0, 0, 1, 1));
+ const gfx::Size output_size(test_bitmap.width(), test_bitmap.height());
+ VideoCaptureOverlay::OnceRenderer renderer = overlay.MakeRenderer(
+ gfx::Rect(output_size), pixel_format(), GetColorSpace());
+ ASSERT_TRUE(renderer);
+ auto frame = CreateVideoFrame(output_size);
+ std::move(renderer).Run(frame.get());
+ EXPECT_TRUE(FrameMatchesPNG(*frame, "overlay_full_cover.png"));
+}
+
+// Basic test: Render an overlay image that covers the entire video frame and is
+// scaled.
+TEST_P(VideoCaptureOverlayRenderTest, FullCover_WithScaling) {
+ StrictMock<MockFrameSource> frame_source;
+ VideoCaptureOverlay overlay(&frame_source);
+
+ EXPECT_CALL(frame_source, GetSourceSize())
+ .WillRepeatedly(Return(kSourceSize));
+ EXPECT_CALL(frame_source, InvalidateRect(gfx::Rect())).RetiresOnSaturation();
+ EXPECT_CALL(frame_source, InvalidateRect(gfx::Rect(kSourceSize)))
+ .RetiresOnSaturation();
+ EXPECT_CALL(frame_source, RequestRefreshFrame());
+
+ const SkBitmap test_bitmap = MakeTestBitmap(0);
+ overlay.SetImageAndBounds(test_bitmap, gfx::RectF(0, 0, 1, 1));
+ const gfx::Size output_size(test_bitmap.width() * 4,
+ test_bitmap.height() * 4);
+ VideoCaptureOverlay::OnceRenderer renderer = overlay.MakeRenderer(
+ gfx::Rect(output_size), pixel_format(), GetColorSpace());
+ ASSERT_TRUE(renderer);
+ auto frame = CreateVideoFrame(output_size);
+ std::move(renderer).Run(frame.get());
+ EXPECT_TRUE(FrameMatchesPNG(*frame, "overlay_full_cover_scaled.png"));
+}
+
+// Tests that changing the position of the overlay results in it being rendered
+// at different locations in the video frame.
+TEST_P(VideoCaptureOverlayRenderTest, MovesAround) {
+ NiceMock<MockFrameSource> frame_source;
+ EXPECT_CALL(frame_source, GetSourceSize())
+ .WillRepeatedly(Return(kSourceSize));
+ VideoCaptureOverlay overlay(&frame_source);
+
+ const SkBitmap test_bitmap = MakeTestBitmap(0);
+ const gfx::Size frame_size(test_bitmap.width() * 4, test_bitmap.height() * 4);
+
+ const gfx::RectF relative_image_bounds[6] = {
+ gfx::RectF(0.0f, 0.0f, 0.5f, 0.5f),
+ gfx::RectF(1.0f / frame_size.width(), 0.0f, 0.5f, 0.5f),
+ gfx::RectF(2.0f / frame_size.width(), 0.0f, 0.5f, 0.5f),
+ gfx::RectF(2.0f / frame_size.width(), 1.0f / frame_size.height(), 0.5f,
+ 0.5f),
+ gfx::RectF(2.0f / frame_size.width(), 2.0f / frame_size.height(), 0.5f,
+ 0.5f),
+ gfx::RectF(0.5f, 0.5f, 0.5f, 0.5f),
+ };
+
+ VideoCaptureOverlay::OnceRenderer renderers[6];
+ for (int i = 0; i < 6; ++i) {
+ if (i == 0) {
+ overlay.SetImageAndBounds(test_bitmap, relative_image_bounds[i]);
+ } else {
+ overlay.SetBounds(relative_image_bounds[i]);
+ }
+ renderers[i] = overlay.MakeRenderer(gfx::Rect(frame_size), pixel_format(),
+ GetColorSpace());
+ }
+
+ constexpr std::array<const char*, 6> kGoldenFiles = {
+ "overlay_moves_0_0.png", "overlay_moves_1_0.png", "overlay_moves_2_0.png",
+ "overlay_moves_2_1.png", "overlay_moves_2_2.png", "overlay_moves_lr.png",
+ };
+
+ for (int i = 0; i < 6; ++i) {
+ SCOPED_TRACE(testing::Message() << "relative_image_bounds="
+ << relative_image_bounds[i].ToString()
+ << ", frame_size=" << frame_size.ToString()
+ << ", golden_file=" << kGoldenFiles[i]);
+ auto frame = CreateVideoFrame(frame_size);
+ std::move(renderers[i]).Run(frame.get());
+ EXPECT_TRUE(FrameMatchesPNG(*frame, kGoldenFiles[i]));
+ }
+}
+
+// Tests that the overlay will be partially rendered (clipped) when any part of
+// it extends outside the video frame's content region.
+//
+// For this test, the content region is a rectangle, centered within the frame
+// (e.g., the content is being letterboxed), and the test attempts to locate the
+// overlay such that part of it should be clipped. The test succeeds if the
+// overlay is clipped to the content region in the center. For example:
+//
+// +-------------------------------+
+// | |
+// | ...... |
+// | ..****//////////// | **** the drawn part of the overlay
+// | ..****CONTENT///// |
+// | /////REGION///// | .... the clipped part of the overlay
+// | //////////////// | (i.e., not drawn)
+// | |
+// | |
+// +-------------------------------+
+TEST_P(VideoCaptureOverlayRenderTest, ClipsToContentBounds) {
+ NiceMock<MockFrameSource> frame_source;
+ EXPECT_CALL(frame_source, GetSourceSize())
+ .WillRepeatedly(Return(kSourceSize));
+ VideoCaptureOverlay overlay(&frame_source);
+
+ const SkBitmap test_bitmap = MakeTestBitmap(0);
+ const gfx::Size frame_size(test_bitmap.width() * 4, test_bitmap.height() * 4);
+ const gfx::Rect region_in_frame(test_bitmap.width(), test_bitmap.height(),
+ test_bitmap.width() * 2,
+ test_bitmap.height() * 2);
+
+ const gfx::RectF relative_image_bounds[4] = {
+ gfx::RectF(-0.25f, -0.25f, 0.5f, 0.5f),
+ gfx::RectF(0.75f, -0.25f, 0.5f, 0.5f),
+ gfx::RectF(0.75f, 0.75f, 0.5f, 0.5f),
+ gfx::RectF(-0.25f, 0.75f, 0.5f, 0.5f),
+ };
+
+ VideoCaptureOverlay::OnceRenderer renderers[4];
+ for (int i = 0; i < 4; ++i) {
+ if (i == 0) {
+ overlay.SetImageAndBounds(test_bitmap, relative_image_bounds[i]);
+ } else {
+ overlay.SetBounds(relative_image_bounds[i]);
+ }
+ renderers[i] =
+ overlay.MakeRenderer(region_in_frame, pixel_format(), GetColorSpace());
+ }
+
+ constexpr std::array<const char*, 4> kGoldenFiles = {
+ "overlay_clips_ul.png", "overlay_clips_ur.png", "overlay_clips_lr.png",
+ "overlay_clips_ll.png",
+ };
+
+ for (int i = 0; i < 4; ++i) {
+ auto frame = CreateVideoFrame(frame_size);
+ std::move(renderers[i]).Run(frame.get());
+ EXPECT_TRUE(FrameMatchesPNG(*frame, kGoldenFiles[i]));
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(
+ ,
+ VideoCaptureOverlayRenderTest,
+ testing::Values(VideoCaptureOverlayRenderTest::kARGBFormat,
+ VideoCaptureOverlayRenderTest::kI420Format));
+
+} // namespace
+} // namespace viz
diff --git a/components/viz/test/data/video_capture/overlay_clips_ll.png b/components/viz/test/data/video_capture/overlay_clips_ll.png
new file mode 100644
index 0000000..7faf623c
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_clips_ll.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_clips_lr.png b/components/viz/test/data/video_capture/overlay_clips_lr.png
new file mode 100644
index 0000000..1eaf92c
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_clips_lr.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_clips_ul.png b/components/viz/test/data/video_capture/overlay_clips_ul.png
new file mode 100644
index 0000000..f89dc43
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_clips_ul.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_clips_ur.png b/components/viz/test/data/video_capture/overlay_clips_ur.png
new file mode 100644
index 0000000..416b1c59
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_clips_ur.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_full_cover.png b/components/viz/test/data/video_capture/overlay_full_cover.png
new file mode 100644
index 0000000..b2aacf7c
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_full_cover.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_full_cover_scaled.png b/components/viz/test/data/video_capture/overlay_full_cover_scaled.png
new file mode 100644
index 0000000..2485016
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_full_cover_scaled.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_0_0.png b/components/viz/test/data/video_capture/overlay_moves_0_0.png
new file mode 100644
index 0000000..f2efc15
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_0_0.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_1_0.png b/components/viz/test/data/video_capture/overlay_moves_1_0.png
new file mode 100644
index 0000000..0bf0a55
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_1_0.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_2_0.png b/components/viz/test/data/video_capture/overlay_moves_2_0.png
new file mode 100644
index 0000000..62668ce
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_2_0.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_2_1.png b/components/viz/test/data/video_capture/overlay_moves_2_1.png
new file mode 100644
index 0000000..aa26331
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_2_1.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_2_2.png b/components/viz/test/data/video_capture/overlay_moves_2_2.png
new file mode 100644
index 0000000..04b9a350
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_2_2.png
Binary files differ
diff --git a/components/viz/test/data/video_capture/overlay_moves_lr.png b/components/viz/test/data/video_capture/overlay_moves_lr.png
new file mode 100644
index 0000000..352b6e4a
--- /dev/null
+++ b/components/viz/test/data/video_capture/overlay_moves_lr.png
Binary files differ
