| // Copyright 2015 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/cast/sender/performance_metrics_overlay.h" |
| |
| #include <stddef.h> |
| #include <stdint.h> |
| |
| #include <algorithm> |
| #include <string> |
| |
| #include "base/bind.h" |
| #include "base/logging.h" |
| #include "base/numerics/safe_conversions.h" |
| #include "base/strings/stringprintf.h" |
| #include "media/base/video_frame.h" |
| |
| namespace media { |
| namespace cast { |
| |
| namespace { |
| |
| constexpr int kScale = 4; // Physical pixels per one logical pixel. |
| constexpr int kCharacterWidth = 3; // Logical pixel width of one character. |
| constexpr int kCharacterHeight = 5; // Logical pixel height of one character. |
| constexpr int kCharacterSpacing = 1; // Logical pixels between each character. |
| constexpr int kLineSpacing = 2; // Logical pixels between each line of chars. |
| constexpr int kPlane = 0; // Y-plane in YUV formats. |
| |
| // For each pixel in the |rect| (logical coordinates), either decrease the |
| // intensity or increase it so that the resulting pixel has a perceivably |
| // different value than it did before. |p_ul| is a pointer to the pixel at |
| // coordinate (0,0) in a single-channel 8bpp bitmap. |stride| is the number of |
| // bytes per row in the output bitmap. |
| void DivergePixels(const gfx::Rect& rect, uint8_t* p_ul, int stride) { |
| DCHECK(p_ul); |
| DCHECK_GT(stride, 0); |
| |
| // These constants and heuristics were chosen based on experimenting with a |
| // wide variety of content, and converging on a readable result. The amount |
| // by which the darker pixels are changed is less because each unit of change |
| // has a larger visual impact on the darker end of the spectrum. Each pixel's |
| // intensity value is changed as follows: |
| // |
| // [16,31] --> [32,63] (always a difference of +16) |
| // [32,64] --> 16 (a difference between -16 and -48) |
| // [65,235] --> [17,187] (always a difference of -48) |
| const int kDivergeDownThreshold = 32; |
| const int kDivergeDownAmount = 48; |
| const int kDivergeUpAmount = 32; |
| const int kMinIntensity = 16; |
| |
| const int top = rect.y() * kScale; |
| const int bottom = rect.bottom() * kScale; |
| const int left = rect.x() * kScale; |
| const int right = rect.right() * kScale; |
| for (int y = top; y < bottom; ++y) { |
| uint8_t* const p_l = p_ul + y * stride; |
| for (int x = left; x < right; ++x) { |
| int intensity = p_l[x]; |
| if (intensity >= kDivergeDownThreshold) |
| intensity = std::max(kMinIntensity, intensity - kDivergeDownAmount); |
| else |
| intensity += kDivergeUpAmount; |
| p_l[x] = static_cast<uint8_t>(intensity); |
| } |
| } |
| } |
| |
| // Render |line| into |frame| at physical pixel row |top| and aligned to the |
| // right edge. Only number digits and a smattering of punctuation characters |
| // will be rendered. |
| void RenderLineOfText(const std::string& line, int top, VideoFrame* frame) { |
| // Compute number of physical pixels wide the rendered |line| would be, |
| // including padding. |
| const int line_width = |
| (((kCharacterWidth + kCharacterSpacing) * static_cast<int>(line.size())) + |
| kCharacterSpacing) * kScale; |
| |
| // Determine if any characters would render past the left edge of the frame, |
| // and compute the index of the first character to be rendered. |
| const int pixels_per_char = (kCharacterWidth + kCharacterSpacing) * kScale; |
| const size_t first_idx = (line_width < frame->visible_rect().width()) ? 0u : |
| static_cast<size_t>( |
| ((line_width - frame->visible_rect().width()) / pixels_per_char) + 1); |
| |
| // Compute the pointer to the pixel at the upper-left corner of the first |
| // character to be rendered. |
| const int stride = frame->stride(kPlane); |
| uint8_t* p_ul = |
| // Start at the first pixel in the first row... |
| frame->visible_data(kPlane) + (stride * top) |
| // ...now move to the right edge of the visible part of the frame... |
| + frame->visible_rect().width() |
| // ...now move left to where line[0] would be rendered... |
| - line_width |
| // ...now move right to where line[first_idx] would be rendered. |
| + first_idx * pixels_per_char; |
| |
| // Render each character. |
| for (size_t i = first_idx; i < line.size(); ++i, p_ul += pixels_per_char) { |
| switch (line[i]) { |
| case '0': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 3), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 3), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '1': |
| DivergePixels(gfx::Rect(1, 0, 1, 5), p_ul, stride); |
| break; |
| case '2': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '3': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '4': |
| DivergePixels(gfx::Rect(0, 0, 1, 2), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 0, 1, 5), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 2, 1), p_ul, stride); |
| break; |
| case '5': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '6': |
| DivergePixels(gfx::Rect(1, 0, 2, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '7': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 2), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 3, 1, 2), p_ul, stride); |
| break; |
| case '8': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '9': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 2, 1), p_ul, stride); |
| break; |
| case 'e': |
| case 'E': |
| DivergePixels(gfx::Rect(0, 0, 3, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 2, 2, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 4, 3, 1), p_ul, stride); |
| break; |
| case '.': |
| DivergePixels(gfx::Rect(1, 4, 1, 1), p_ul, stride); |
| break; |
| case '+': |
| DivergePixels(gfx::Rect(1, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 3, 1, 1), p_ul, stride); |
| FALLTHROUGH; |
| case '-': |
| DivergePixels(gfx::Rect(0, 2, 3, 1), p_ul, stride); |
| break; |
| case 'x': |
| DivergePixels(gfx::Rect(0, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 2, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 3, 1, 1), p_ul, stride); |
| break; |
| case ':': |
| DivergePixels(gfx::Rect(1, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 3, 1, 1), p_ul, stride); |
| break; |
| case '!': |
| DivergePixels(gfx::Rect(1, 0, 1, 3), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 4, 1, 1), p_ul, stride); |
| break; |
| case '%': |
| DivergePixels(gfx::Rect(0, 0, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 1, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(1, 2, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(0, 3, 1, 1), p_ul, stride); |
| DivergePixels(gfx::Rect(2, 4, 1, 1), p_ul, stride); |
| break; |
| case ' ': |
| default: |
| break; |
| } |
| } |
| } |
| |
| } // namespace |
| |
| scoped_refptr<VideoFrame> MaybeRenderPerformanceMetricsOverlay( |
| base::TimeDelta target_playout_delay, |
| bool in_low_latency_mode, |
| int target_bitrate, |
| int frames_ago, |
| double encoder_utilization, |
| double lossy_utilization, |
| scoped_refptr<VideoFrame> source) { |
| if (!VLOG_IS_ON(1)) |
| return source; |
| |
| if (VideoFrame::PlaneHorizontalBitsPerPixel(source->format(), kPlane) != 8) { |
| DLOG(WARNING) << "Cannot render overlay: Plane " << kPlane << " not 8bpp."; |
| return source; |
| } |
| |
| // Can't read from unmappable memory (DmaBuf, CVPixelBuffer). |
| if (!source->IsMappable()) { |
| DVLOG(2) << "Cannot render overlay: frame uses unmappable memory."; |
| return source; |
| } |
| |
| // Compute the physical pixel top row for the bottom-most line of text. |
| const int line_height = (kCharacterHeight + kLineSpacing) * kScale; |
| int top = source->visible_rect().height() - line_height; |
| if (top < 0) |
| return source; // No pixels would change: Return source frame. |
| |
| // Allocate a new frame, identical in configuration to |source| and copy over |
| // all data and metadata. |
| const scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame( |
| source->format(), source->coded_size(), source->visible_rect(), |
| source->natural_size(), source->timestamp()); |
| if (!frame) |
| return source; // Allocation failure: Return source frame. |
| for (size_t plane = 0, num_planes = VideoFrame::NumPlanes(source->format()); |
| plane < num_planes; ++plane) { |
| const size_t row_count = VideoFrame::Rows(plane, source->format(), |
| source->visible_rect().height()); |
| const size_t bytes_per_row = VideoFrame::RowBytes( |
| plane, source->format(), source->visible_rect().width()); |
| const uint8_t* src = source->visible_data(plane); |
| const int src_stride = source->stride(plane); |
| uint8_t* dst = frame->visible_data(plane); |
| const int dst_stride = frame->stride(plane); |
| for (size_t row = 0; row < row_count; |
| ++row, src += src_stride, dst += dst_stride) { |
| memcpy(dst, src, bytes_per_row); |
| } |
| } |
| frame->metadata()->MergeMetadataFrom(source->metadata()); |
| // Important: After all consumers are done with the frame, copy-back the |
| // changed/new metadata to the source frame, as it contains feedback signals |
| // that need to propagate back up the video stack. The destruction callback |
| // for the |frame| holds a ref-counted reference to the source frame to ensure |
| // the source frame has the right metadata before its destruction observers |
| // are invoked. |
| frame->AddDestructionObserver(base::Bind( |
| [](const VideoFrameMetadata* sent_frame_metadata, |
| const scoped_refptr<VideoFrame>& source_frame) { |
| source_frame->metadata()->Clear(); |
| source_frame->metadata()->MergeMetadataFrom(sent_frame_metadata); |
| }, |
| frame->metadata(), std::move(source))); |
| |
| // Line 3: Frame duration, resolution, and timestamp. |
| int frame_duration_ms = 0; |
| int frame_duration_ms_frac = 0; |
| base::TimeDelta frame_duration; |
| if (frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION, |
| &frame_duration)) { |
| const int decimilliseconds = base::saturated_cast<int>( |
| frame_duration.InMicroseconds() / 100.0 + 0.5); |
| frame_duration_ms = decimilliseconds / 10; |
| frame_duration_ms_frac = decimilliseconds % 10; |
| } |
| base::TimeDelta rem = frame->timestamp(); |
| const int minutes = rem.InMinutes(); |
| rem -= base::TimeDelta::FromMinutes(minutes); |
| const int seconds = static_cast<int>(rem.InSeconds()); |
| rem -= base::TimeDelta::FromSeconds(seconds); |
| const int hundredth_seconds = static_cast<int>(rem.InMilliseconds() / 10); |
| RenderLineOfText( |
| base::StringPrintf("%d.%01d %dx%d %d:%02d.%02d", frame_duration_ms, |
| frame_duration_ms_frac, frame->visible_rect().width(), |
| frame->visible_rect().height(), minutes, seconds, |
| hundredth_seconds), |
| top, frame.get()); |
| |
| // Move up one line's worth of pixels. |
| top -= line_height; |
| if (top < 0 || !VLOG_IS_ON(2)) |
| return frame; |
| |
| // Line 2: Capture duration, target playout delay, low-latency mode, and |
| // target bitrate. |
| int capture_duration_ms = 0; |
| base::TimeTicks capture_begin_time, capture_end_time; |
| if (frame->metadata()->GetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME, |
| &capture_begin_time) && |
| frame->metadata()->GetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME, |
| &capture_end_time)) { |
| capture_duration_ms = base::saturated_cast<int>( |
| (capture_end_time - capture_begin_time).InMillisecondsF() + 0.5); |
| } |
| const int target_playout_delay_ms = |
| static_cast<int>(target_playout_delay.InMillisecondsF() + 0.5); |
| const int target_kbits = target_bitrate / 1000; |
| RenderLineOfText( |
| base::StringPrintf("%d %4.1d%c %4.1d", capture_duration_ms, |
| target_playout_delay_ms, |
| in_low_latency_mode ? '!' : '.', target_kbits), |
| top, frame.get()); |
| |
| // Move up one line's worth of pixels. |
| top -= line_height; |
| if (top < 0 || !VLOG_IS_ON(3)) |
| return frame; |
| |
| // Line 1: Recent utilization metrics. |
| const int encoder_pct = |
| base::saturated_cast<int>(encoder_utilization * 100.0 + 0.5); |
| const int lossy_pct = |
| base::saturated_cast<int>(lossy_utilization * 100.0 + 0.5); |
| RenderLineOfText(base::StringPrintf("%d %3.1d%% %3.1d%%", frames_ago, |
| encoder_pct, lossy_pct), |
| top, frame.get()); |
| |
| return frame; |
| } |
| |
| } // namespace cast |
| } // namespace media |