blob: 9b35b42e9e856735345234c789f96c0ad9b67bf5 [file] [log] [blame]
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/cast/sender/performance_metrics_overlay.h"
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <string>
#include "base/bind.h"
#include "base/logging.h"
#include "base/numerics/safe_conversions.h"
#include "base/strings/stringprintf.h"
#include "media/base/video_frame.h"
namespace media {
namespace cast {
namespace {
constexpr int kScale = 4; // Physical pixels per one logical pixel.
constexpr int kCharacterWidth = 3; // Logical pixel width of one character.
constexpr int kCharacterHeight = 5; // Logical pixel height of one character.
constexpr int kCharacterSpacing = 1; // Logical pixels between each character.
constexpr int kLineSpacing = 2; // Logical pixels between each line of chars.
constexpr int kPlane = 0; // Y-plane in YUV formats.
// For each pixel in the |rect| (logical coordinates), either decrease the
// intensity or increase it so that the resulting pixel has a perceivably
// different value than it did before. |p_ul| is a pointer to the pixel at
// coordinate (0,0) in a single-channel 8bpp bitmap. |stride| is the number of
// bytes per row in the output bitmap.
void DivergePixels(const gfx::Rect& rect, uint8_t* p_ul, int stride) {
DCHECK(p_ul);
DCHECK_GT(stride, 0);
// These constants and heuristics were chosen based on experimenting with a
// wide variety of content, and converging on a readable result. The amount
// by which the darker pixels are changed is less because each unit of change
// has a larger visual impact on the darker end of the spectrum. Each pixel's
// intensity value is changed as follows:
//
// [16,31] --> [32,63] (always a difference of +16)
// [32,64] --> 16 (a difference between -16 and -48)
// [65,235] --> [17,187] (always a difference of -48)
const int kDivergeDownThreshold = 32;
const int kDivergeDownAmount = 48;
const int kDivergeUpAmount = 32;
const int kMinIntensity = 16;
const int top = rect.y() * kScale;
const int bottom = rect.bottom() * kScale;
const int left = rect.x() * kScale;
const int right = rect.right() * kScale;
for (int y = top; y < bottom; ++y) {
uint8_t* const p_l = p_ul + y * stride;
for (int x = left; x < right; ++x) {
int intensity = p_l[x];
if (intensity >= kDivergeDownThreshold)
intensity = std::max(kMinIntensity, intensity - kDivergeDownAmount);
else
intensity += kDivergeUpAmount;
p_l[x] = static_cast<uint8_t>(intensity);
}
}
}
// Render |line| into |frame| at physical pixel row |top| and aligned to the
// right edge. Only number digits and a smattering of punctuation characters
// will be rendered.
void RenderLineOfText(const std::string& line, int top, VideoFrame* frame) {
// Compute number of physical pixels wide the rendered |line| would be,
// including padding.
const int line_width =
(((kCharacterWidth + kCharacterSpacing) * static_cast<int>(line.size())) +
kCharacterSpacing) * kScale;
// Determine if any characters would render past the left edge of the frame,
// and compute the index of the first character to be rendered.
const int pixels_per_char = (kCharacterWidth + kCharacterSpacing) * kScale;
const size_t first_idx = (line_width < frame->visible_rect().width()) ? 0u :
static_cast<size_t>(
((line_width - frame->visible_rect().width()) / pixels_per_char) + 1);
// Compute the pointer to the pixel at the upper-left corner of the first
// character to be rendered.
const int stride = frame->stride(kPlane);
uint8_t* p_ul =
// Start at the first pixel in the first row...
frame->visible_data(kPlane) + (stride * top)
// ...now move to the right edge of the visible part of the frame...
+ frame->visible_rect().width()
// ...now move left to where line[0] would be rendered...
- line_width
// ...now move right to where line[first_idx] would be rendered.
+ first_idx * pixels_per_char;
// Render each character.
for (size_t i = first_idx; i < line.size(); ++i, p_ul += pixels_per_char) {
switch (line[i]) {
case '0':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 3), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 3), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '1':
DivergePixels(gfx::Rect(1, 0, 1, 5), p_ul, stride);
break;
case '2':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '3':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '4':
DivergePixels(gfx::Rect(0, 0, 1, 2), p_ul, stride);
DivergePixels(gfx::Rect(2, 0, 1, 5), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 2, 1), p_ul, stride);
break;
case '5':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '6':
DivergePixels(gfx::Rect(1, 0, 2, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '7':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 2), p_ul, stride);
DivergePixels(gfx::Rect(1, 3, 1, 2), p_ul, stride);
break;
case '8':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '9':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 2, 1), p_ul, stride);
break;
case 'e':
case 'E':
DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 2, 2, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride);
break;
case '.':
DivergePixels(gfx::Rect(1, 4, 1, 1), p_ul, stride);
break;
case '+':
DivergePixels(gfx::Rect(1, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(1, 3, 1, 1), p_ul, stride);
FALLTHROUGH;
case '-':
DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride);
break;
case 'x':
DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(1, 2, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride);
break;
case ':':
DivergePixels(gfx::Rect(1, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(1, 3, 1, 1), p_ul, stride);
break;
case '!':
DivergePixels(gfx::Rect(1, 0, 1, 3), p_ul, stride);
DivergePixels(gfx::Rect(1, 4, 1, 1), p_ul, stride);
break;
case '%':
DivergePixels(gfx::Rect(0, 0, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(1, 2, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride);
DivergePixels(gfx::Rect(2, 4, 1, 1), p_ul, stride);
break;
case ' ':
default:
break;
}
}
}
} // namespace
scoped_refptr<VideoFrame> MaybeRenderPerformanceMetricsOverlay(
base::TimeDelta target_playout_delay,
bool in_low_latency_mode,
int target_bitrate,
int frames_ago,
double encoder_utilization,
double lossy_utilization,
scoped_refptr<VideoFrame> source) {
if (!VLOG_IS_ON(1))
return source;
if (VideoFrame::PlaneHorizontalBitsPerPixel(source->format(), kPlane) != 8) {
DLOG(WARNING) << "Cannot render overlay: Plane " << kPlane << " not 8bpp.";
return source;
}
// Can't read from unmappable memory (DmaBuf, CVPixelBuffer).
if (!source->IsMappable()) {
DVLOG(2) << "Cannot render overlay: frame uses unmappable memory.";
return source;
}
// Compute the physical pixel top row for the bottom-most line of text.
const int line_height = (kCharacterHeight + kLineSpacing) * kScale;
int top = source->visible_rect().height() - line_height;
if (top < 0)
return source; // No pixels would change: Return source frame.
// Allocate a new frame, identical in configuration to |source| and copy over
// all data and metadata.
const scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
source->format(), source->coded_size(), source->visible_rect(),
source->natural_size(), source->timestamp());
if (!frame)
return source; // Allocation failure: Return source frame.
for (size_t plane = 0, num_planes = VideoFrame::NumPlanes(source->format());
plane < num_planes; ++plane) {
const size_t row_count = VideoFrame::Rows(plane, source->format(),
source->visible_rect().height());
const size_t bytes_per_row = VideoFrame::RowBytes(
plane, source->format(), source->visible_rect().width());
const uint8_t* src = source->visible_data(plane);
const int src_stride = source->stride(plane);
uint8_t* dst = frame->visible_data(plane);
const int dst_stride = frame->stride(plane);
for (size_t row = 0; row < row_count;
++row, src += src_stride, dst += dst_stride) {
memcpy(dst, src, bytes_per_row);
}
}
frame->metadata()->MergeMetadataFrom(source->metadata());
// Important: After all consumers are done with the frame, copy-back the
// changed/new metadata to the source frame, as it contains feedback signals
// that need to propagate back up the video stack. The destruction callback
// for the |frame| holds a ref-counted reference to the source frame to ensure
// the source frame has the right metadata before its destruction observers
// are invoked.
frame->AddDestructionObserver(base::Bind(
[](const VideoFrameMetadata* sent_frame_metadata,
const scoped_refptr<VideoFrame>& source_frame) {
source_frame->metadata()->Clear();
source_frame->metadata()->MergeMetadataFrom(sent_frame_metadata);
},
frame->metadata(), std::move(source)));
// Line 3: Frame duration, resolution, and timestamp.
int frame_duration_ms = 0;
int frame_duration_ms_frac = 0;
base::TimeDelta frame_duration;
if (frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
&frame_duration)) {
const int decimilliseconds = base::saturated_cast<int>(
frame_duration.InMicroseconds() / 100.0 + 0.5);
frame_duration_ms = decimilliseconds / 10;
frame_duration_ms_frac = decimilliseconds % 10;
}
base::TimeDelta rem = frame->timestamp();
const int minutes = rem.InMinutes();
rem -= base::TimeDelta::FromMinutes(minutes);
const int seconds = static_cast<int>(rem.InSeconds());
rem -= base::TimeDelta::FromSeconds(seconds);
const int hundredth_seconds = static_cast<int>(rem.InMilliseconds() / 10);
RenderLineOfText(
base::StringPrintf("%d.%01d %dx%d %d:%02d.%02d", frame_duration_ms,
frame_duration_ms_frac, frame->visible_rect().width(),
frame->visible_rect().height(), minutes, seconds,
hundredth_seconds),
top, frame.get());
// Move up one line's worth of pixels.
top -= line_height;
if (top < 0 || !VLOG_IS_ON(2))
return frame;
// Line 2: Capture duration, target playout delay, low-latency mode, and
// target bitrate.
int capture_duration_ms = 0;
base::TimeTicks capture_begin_time, capture_end_time;
if (frame->metadata()->GetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME,
&capture_begin_time) &&
frame->metadata()->GetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME,
&capture_end_time)) {
capture_duration_ms = base::saturated_cast<int>(
(capture_end_time - capture_begin_time).InMillisecondsF() + 0.5);
}
const int target_playout_delay_ms =
static_cast<int>(target_playout_delay.InMillisecondsF() + 0.5);
const int target_kbits = target_bitrate / 1000;
RenderLineOfText(
base::StringPrintf("%d %4.1d%c %4.1d", capture_duration_ms,
target_playout_delay_ms,
in_low_latency_mode ? '!' : '.', target_kbits),
top, frame.get());
// Move up one line's worth of pixels.
top -= line_height;
if (top < 0 || !VLOG_IS_ON(3))
return frame;
// Line 1: Recent utilization metrics.
const int encoder_pct =
base::saturated_cast<int>(encoder_utilization * 100.0 + 0.5);
const int lossy_pct =
base::saturated_cast<int>(lossy_utilization * 100.0 + 0.5);
RenderLineOfText(base::StringPrintf("%d %3.1d%% %3.1d%%", frames_ago,
encoder_pct, lossy_pct),
top, frame.get());
return frame;
}
} // namespace cast
} // namespace media