Report maximum supported resolution/framerates by current VEA encoder.
WebRTC may in the future take them into account for quality adpatation
and video bitrate allocation adjustments.
The reason we make this to be dynamically updated information through
encoder info, instead of reusing existing GetSupportedProfiles IPC
interface: 1. Only VEA itself knows how many encode sessions co-exists
on the platform. Although currently we reuse the same approach for
fetching encode fps/resolution limits, long term we still want to keep
the capability to update these information conditionally. For example,
when 3 encode sessions co-exists on the platform, maximum supported FPS
per encoder instance should be reduced accordingly; 2. We only need to
fetch limitations for current active codec type. GetSupportedProfiles
IPC returns more which we don't necessarily need all of them.
We do not expand ResolutionBitrateLimit to also differentiate between
max_frame_size and min_frame_size. The reason is that RtcVideoEncoder
will re-initialize encoder on resolution change. There both min and max
resolution will be examined against that reported through
GetSupportedProfiles. Unless WebRTC quality scaler will utilize the
min frame size info, we will not add it as part of EncoderInfo.
Bug: 382015342
Change-Id: I78406d182beeaa7e854762f12dae4d1e8036569d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6069866
Reviewed-by: Sida Zhu <zhusida@bytedance.com>
Reviewed-by: Takashi Toyoshima <toyoshim@chromium.org>
Reviewed-by: Eugene Zemtsov <eugene@chromium.org>
Reviewed-by: Henrik Boström <hbos@chromium.org>
Commit-Queue: Jianlin Qiu <jianlin.qiu@intel.com>
Cr-Commit-Position: refs/heads/main@{#1395240}
diff --git a/media/gpu/android/ndk_video_encode_accelerator.cc b/media/gpu/android/ndk_video_encode_accelerator.cc
index 69ba3d0e..074658d 100644
--- a/media/gpu/android/ndk_video_encode_accelerator.cc
+++ b/media/gpu/android/ndk_video_encode_accelerator.cc
@@ -574,6 +574,26 @@
AMediaCodec_releaseName(media_codec_->codec(), name_ptr);
}
+ for (const auto& info : GetEncoderInfoCache()) {
+ if (info.name == codec_name) {
+ // Skip software HEVC encoders, as it only supports the maximum 512x512
+ // resolution at 30fps, and it is also filtered out by
+ // GetSupportedProfiles().
+ if (info.profile.profile == VideoCodecProfile::HEVCPROFILE_MAIN &&
+ info.profile.is_software_codec) {
+ continue;
+ }
+
+ // TODO(crbug.com/382015342): Set the bitrate limits when we can get them
+ // through MediaCodec API.
+ encoder_info_.resolution_rate_limits.emplace_back(
+ info.profile.max_resolution, /*min_start_bitrate_bps=*/0,
+ /*min_bitrate_bps=*/0, /*max_bitrate_bps=*/0,
+ info.profile.max_framerate_numerator,
+ info.profile.max_framerate_denominator);
+ }
+ }
+
encoder_info_.implementation_name =
"NdkVideoEncodeAccelerator(" + codec_name + ")";
encoder_info_.supports_native_handle = false;
diff --git a/media/gpu/mac/vt_video_encode_accelerator_mac.mm b/media/gpu/mac/vt_video_encode_accelerator_mac.mm
index 59d618c..78a7ebd 100644
--- a/media/gpu/mac/vt_video_encode_accelerator_mac.mm
+++ b/media/gpu/mac/vt_video_encode_accelerator_mac.mm
@@ -329,6 +329,23 @@
info.implementation_name = "VideoToolbox";
info.is_hardware_accelerated = IsHardwareEncoder(compression_session);
+ // TODO(crbug.com/382015342): Query bitrate limits, and report them through
+ // VideoEncoderInfo's |resolution_bitrate_limits|.
+ const VideoCodec codec = VideoCodecProfileToVideoCodec(config.output_profile);
+ gfx::Size resolution = GetMaxResolution(codec);
+ info.resolution_rate_limits.emplace_back(
+ resolution, /*min_start_bitrate_bps=*/0,
+ /*min_bitrate_bps=*/0, /*max_bitrate_bps=*/0, kMaxFrameRateNumerator,
+ kMaxFrameRateDenominator);
+ if (resolution.width() != resolution.height()) {
+ resolution.Transpose();
+ info.resolution_rate_limits.emplace_back(
+ resolution,
+ /*min_start_bitrate_bps=*/0, /*min_bitrate_bps=*/0,
+ /*max_bitrate_bps=*/0, kMaxFrameRateNumerator,
+ kMaxFrameRateDenominator);
+ }
+
std::optional<int> max_frame_delay_property;
base::apple::ScopedCFTypeRef<CFNumberRef> max_frame_delay_count;
if (VTSessionCopyProperty(
diff --git a/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc b/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc
index b409f24..0a9e322f 100644
--- a/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc
+++ b/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc
@@ -684,7 +684,7 @@
uint64_t max_possible_framerate = std::floor(
(max_macroblocks_per_second * kMacroBlockWidth * kMacroBlockHeight) /
- max_framerate_and_resolution.resoluion.Area64());
+ max_framerate_and_resolution.resolution.Area64());
return std::clamp(static_cast<uint32_t>(max_possible_framerate), 1u,
max_framerate_and_resolution.frame_rate);
@@ -709,8 +709,8 @@
RETURN_ON_HR_FAILURE(
MFSetAttributeSize(
media_type.Get(), MF_MT_FRAME_SIZE,
- kDefaultMaxFramerateAndResolution.resoluion.width(),
- kDefaultMaxFramerateAndResolution.resoluion.height()),
+ kDefaultMaxFramerateAndResolution.resolution.width(),
+ kDefaultMaxFramerateAndResolution.resolution.height()),
"Set attribute size failed", framerate_and_resolutions);
// Frame rate,30, is dummy value for pass through.
RETURN_ON_HR_FAILURE(
@@ -772,7 +772,7 @@
FramerateAndResolution framerate_and_resolution = {
CalculateMaxFramerateFromMacroBlocksPerSecond(
max_framerate_and_resolution, max_macroblocks_per_second),
- max_framerate_and_resolution.resoluion};
+ max_framerate_and_resolution.resolution};
// Only if the calculated framerate >= the default framerate, we then
// consider this resolution & framerate combination is supported.
@@ -1141,14 +1141,15 @@
for (auto& max_framerate_and_resolution : max_framerate_and_resolutions) {
DVLOG(3) << __func__ << ": " << codec << " codec, max resolution width: "
- << max_framerate_and_resolution.resoluion.width() << ", height: "
- << max_framerate_and_resolution.resoluion.height()
+ << max_framerate_and_resolution.resolution.width()
+ << ", height: "
+ << max_framerate_and_resolution.resolution.height()
<< ", min resolution width: " << min_resolution.width()
<< ", height: " << min_resolution.height()
<< ", framerate: " << max_framerate_and_resolution.frame_rate;
SupportedProfile profile(VIDEO_CODEC_PROFILE_UNKNOWN,
- max_framerate_and_resolution.resoluion,
+ max_framerate_and_resolution.resolution,
max_framerate_and_resolution.frame_rate *
kDefaultFrameRateDenominator,
kDefaultFrameRateDenominator, bitrate_mode,
@@ -1366,6 +1367,23 @@
return false;
}
+ for (auto& [framerate, resolution] : max_framerate_and_resolutions_) {
+ // TODO(crbug.com/382015342): Add implementation for checking bitrate
+ // limits.
+ encoder_info_.resolution_rate_limits.emplace_back(
+ resolution, /*min_start_bitrate_bps=*/0, /*min_bitrate_bps=*/0,
+ /*max_bitrate_bps=*/0, /*max_framerate_numerator=*/framerate,
+ /*max_framerate_denominator=*/1);
+
+ resolution.Transpose();
+ encoder_info_.resolution_rate_limits.emplace_back(
+ resolution,
+ /*min_start_bitrate_bps=*/0, /*min_bitrate_bps=*/0,
+ /*max_bitrate_bps=*/0, /*max_framerate_numerator=*/framerate,
+ /*max_framerate_denominator=*/1);
+ resolution.Transpose();
+ }
+
encoder_info_.implementation_name = "MediaFoundationVideoEncodeAccelerator";
// Currently, MFVEA does not support odd resolution well. The implementation
// here reports alignment of 2 in the EncoderInfo, together with simulcast
diff --git a/media/gpu/windows/media_foundation_video_encode_accelerator_win.h b/media/gpu/windows/media_foundation_video_encode_accelerator_win.h
index 9594dc6..4712f70 100644
--- a/media/gpu/windows/media_foundation_video_encode_accelerator_win.h
+++ b/media/gpu/windows/media_foundation_video_encode_accelerator_win.h
@@ -41,7 +41,7 @@
struct FramerateAndResolution {
uint32_t frame_rate;
- gfx::Size resoluion;
+ gfx::Size resolution;
};
class VideoRateControlWrapper;
diff --git a/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc b/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc
index a4e2320..3efee09 100644
--- a/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc
+++ b/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc
@@ -54,10 +54,10 @@
for (size_t i = 0; i < ::media::VideoEncoderInfo::kMaxSpatialLayers; ++i)
input.fps_allocation[i] = {5, 5, 10};
// Resolution bitrate limits.
- input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit(
- gfx::Size(123, 456), 123456, 123456, 789012));
- input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit(
- gfx::Size(789, 1234), 1234567, 1234567, 7890123));
+ input.resolution_rate_limits.push_back(::media::ResolutionRateLimit(
+ gfx::Size(123, 456), 123456, 123456, 789012, 30, 1));
+ input.resolution_rate_limits.push_back(::media::ResolutionRateLimit(
+ gfx::Size(789, 1234), 1234567, 1234567, 7890123, 30, 1));
// Other bool values.
input.supports_native_handle = true;
input.has_trusted_rate_controller = true;
diff --git a/media/mojo/mojom/video_encoder_info.mojom b/media/mojo/mojom/video_encoder_info.mojom
index 7696793..57b4dd6 100644
--- a/media/mojo/mojom/video_encoder_info.mojom
+++ b/media/mojo/mojom/video_encoder_info.mojom
@@ -8,11 +8,21 @@
// The following mojom classes are the corresponding classes in webrtc project.
// See third_party/webrtc/api/video_codecs/video_encoder.h for details.
-struct ResolutionBitrateLimit {
+// TODO(crbug.com/382015342): Add missing max_framerate_numerator and
+// max_framerate_denominator as member to the corresponding class in webrtc.
+struct ResolutionRateLimit {
+ // Maximum frame size in width and height.
gfx.mojom.Size frame_size;
+ // Minimum allowed start encoding bitrate in bps.
int32 min_start_bitrate_bps;
+ // Minimum allowed encoding bitrate in bps.
int32 min_bitrate_bps;
+ // Maximum allowed encoding bitrate in bps.
int32 max_bitrate_bps;
+ // Maximum allowed encoding frame rate's numerator.
+ uint32 max_framerate_numerator;
+ // Maximum allowed encoding frame rate's denominator.
+ uint32 max_framerate_denominator;
};
// TODO(crbug.com/40489779): Remove |has_*| values and use nullable types.
@@ -37,5 +47,5 @@
// This array size is equal to media::VideoEncoderInfo::kMaxSpatialLayers.
array<array<uint8>, 5> fps_allocation;
- array<ResolutionBitrateLimit> resolution_bitrate_limits;
+ array<ResolutionRateLimit> resolution_rate_limits;
};
diff --git a/media/mojo/mojom/video_encoder_info_mojom_traits.cc b/media/mojo/mojom/video_encoder_info_mojom_traits.cc
index ceaa7b8..a333400 100644
--- a/media/mojo/mojom/video_encoder_info_mojom_traits.cc
+++ b/media/mojo/mojom/video_encoder_info_mojom_traits.cc
@@ -9,15 +9,17 @@
namespace mojo {
// static
-bool StructTraits<media::mojom::ResolutionBitrateLimitDataView,
- media::ResolutionBitrateLimit>::
- Read(media::mojom::ResolutionBitrateLimitDataView data,
- media::ResolutionBitrateLimit* out) {
+bool StructTraits<media::mojom::ResolutionRateLimitDataView,
+ media::ResolutionRateLimit>::
+ Read(media::mojom::ResolutionRateLimitDataView data,
+ media::ResolutionRateLimit* out) {
if (!data.ReadFrameSize(&out->frame_size))
return false;
out->min_start_bitrate_bps = data.min_start_bitrate_bps();
out->min_bitrate_bps = data.min_bitrate_bps();
out->max_bitrate_bps = data.max_bitrate_bps();
+ out->max_framerate_numerator = data.max_framerate_numerator();
+ out->max_framerate_denominator = data.max_framerate_denominator();
return true;
}
@@ -53,8 +55,9 @@
if (!data.ReadFpsAllocation(&fps_allocation))
return false;
- if (!data.ReadResolutionBitrateLimits(&out->resolution_bitrate_limits))
+ if (!data.ReadResolutionRateLimits(&out->resolution_rate_limits)) {
return false;
+ }
return true;
}
diff --git a/media/mojo/mojom/video_encoder_info_mojom_traits.h b/media/mojo/mojom/video_encoder_info_mojom_traits.h
index 20969146..b559002 100644
--- a/media/mojo/mojom/video_encoder_info_mojom_traits.h
+++ b/media/mojo/mojom/video_encoder_info_mojom_traits.h
@@ -13,27 +13,35 @@
namespace mojo {
template <>
-class StructTraits<media::mojom::ResolutionBitrateLimitDataView,
- media::ResolutionBitrateLimit> {
+class StructTraits<media::mojom::ResolutionRateLimitDataView,
+ media::ResolutionRateLimit> {
public:
static const gfx::Size& frame_size(
- const media::ResolutionBitrateLimit& resolution_bitrate_limit) {
- return resolution_bitrate_limit.frame_size;
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.frame_size;
}
static int min_start_bitrate_bps(
- const media::ResolutionBitrateLimit& resolution_bitrate_limit) {
- return resolution_bitrate_limit.min_start_bitrate_bps;
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.min_start_bitrate_bps;
}
static int min_bitrate_bps(
- const media::ResolutionBitrateLimit& resolution_bitrate_limit) {
- return resolution_bitrate_limit.min_bitrate_bps;
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.min_bitrate_bps;
}
static int max_bitrate_bps(
- const media::ResolutionBitrateLimit& resolution_bitrate_limit) {
- return resolution_bitrate_limit.max_bitrate_bps;
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.max_bitrate_bps;
}
- static bool Read(media::mojom::ResolutionBitrateLimitDataView data,
- media::ResolutionBitrateLimit* out);
+ static uint32_t max_framerate_numerator(
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.max_framerate_numerator;
+ }
+ static uint32_t max_framerate_denominator(
+ const media::ResolutionRateLimit& resolution_rate_limit) {
+ return resolution_rate_limit.max_framerate_denominator;
+ }
+ static bool Read(media::mojom::ResolutionRateLimitDataView data,
+ media::ResolutionRateLimit* out);
};
template <>
@@ -97,9 +105,9 @@
fps_allocation(const media::VideoEncoderInfo& video_encoder_info) {
return video_encoder_info.fps_allocation;
}
- static const std::vector<media::ResolutionBitrateLimit>&
- resolution_bitrate_limits(const media::VideoEncoderInfo& video_encoder_info) {
- return video_encoder_info.resolution_bitrate_limits;
+ static const std::vector<media::ResolutionRateLimit>& resolution_rate_limits(
+ const media::VideoEncoderInfo& video_encoder_info) {
+ return video_encoder_info.resolution_rate_limits;
}
static bool Read(media::mojom::VideoEncoderInfoDataView data,
diff --git a/media/video/video_encoder_info.cc b/media/video/video_encoder_info.cc
index a7039d20..f801285a 100644
--- a/media/video/video_encoder_info.cc
+++ b/media/video/video_encoder_info.cc
@@ -10,25 +10,30 @@
namespace media {
-ResolutionBitrateLimit::ResolutionBitrateLimit() = default;
-ResolutionBitrateLimit::ResolutionBitrateLimit(const ResolutionBitrateLimit&) =
- default;
-ResolutionBitrateLimit::ResolutionBitrateLimit(const gfx::Size& frame_size,
- int min_start_bitrate_bps,
- int min_bitrate_bps,
- int max_bitrate_bps)
+ResolutionRateLimit::ResolutionRateLimit() = default;
+ResolutionRateLimit::ResolutionRateLimit(const ResolutionRateLimit&) = default;
+ResolutionRateLimit::ResolutionRateLimit(const gfx::Size& frame_size,
+ int min_start_bitrate_bps,
+ int min_bitrate_bps,
+ int max_bitrate_bps,
+ uint32_t max_framerate_numerator,
+ uint32_t max_framerate_denominator)
: frame_size(frame_size),
min_start_bitrate_bps(min_start_bitrate_bps),
min_bitrate_bps(min_bitrate_bps),
- max_bitrate_bps(max_bitrate_bps) {}
-ResolutionBitrateLimit::~ResolutionBitrateLimit() = default;
+ max_bitrate_bps(max_bitrate_bps),
+ max_framerate_numerator(max_framerate_numerator),
+ max_framerate_denominator(max_framerate_denominator) {}
+ResolutionRateLimit::~ResolutionRateLimit() = default;
-bool operator==(const ResolutionBitrateLimit& lhs,
- const ResolutionBitrateLimit& rhs) {
+bool operator==(const ResolutionRateLimit& lhs,
+ const ResolutionRateLimit& rhs) {
return std::tie(lhs.frame_size, lhs.min_start_bitrate_bps,
- lhs.min_bitrate_bps, lhs.max_bitrate_bps) ==
+ lhs.min_bitrate_bps, lhs.max_bitrate_bps,
+ lhs.max_framerate_numerator, lhs.max_framerate_denominator) ==
std::tie(rhs.frame_size, rhs.min_start_bitrate_bps,
- rhs.min_bitrate_bps, rhs.max_bitrate_bps);
+ rhs.min_bitrate_bps, rhs.max_bitrate_bps,
+ rhs.max_framerate_numerator, rhs.max_framerate_denominator);
}
VideoEncoderInfo::VideoEncoderInfo() = default;
@@ -41,13 +46,13 @@
lhs.is_hardware_accelerated, lhs.supports_simulcast,
lhs.reports_average_qp, lhs.requested_resolution_alignment,
lhs.apply_alignment_to_all_simulcast_layers,
- lhs.fps_allocation, lhs.resolution_bitrate_limits) ==
+ lhs.fps_allocation, lhs.resolution_rate_limits) ==
std::tie(rhs.implementation_name, rhs.frame_delay, rhs.input_capacity,
rhs.supports_native_handle, rhs.has_trusted_rate_controller,
rhs.is_hardware_accelerated, rhs.supports_simulcast,
rhs.reports_average_qp, rhs.requested_resolution_alignment,
rhs.apply_alignment_to_all_simulcast_layers,
- rhs.fps_allocation, rhs.resolution_bitrate_limits);
+ rhs.fps_allocation, rhs.resolution_rate_limits);
}
bool VideoEncoderInfo::DoesSupportGpuSharedImages(VideoPixelFormat format) {
diff --git a/media/video/video_encoder_info.h b/media/video/video_encoder_info.h
index 8aa747a..16aebaa 100644
--- a/media/video/video_encoder_info.h
+++ b/media/video/video_encoder_info.h
@@ -21,23 +21,27 @@
// These chromium classes are the corresponding classes in webrtc project.
// See third_party/webrtc/api/video_codecs/video_encoder.h for the detail.
-struct MEDIA_EXPORT ResolutionBitrateLimit {
- ResolutionBitrateLimit();
- ResolutionBitrateLimit(const ResolutionBitrateLimit&);
- ResolutionBitrateLimit(const gfx::Size& frame_size,
- int min_start_bitrate_bps,
- int min_bitrate_bps,
- int max_bitrate_bps);
- ~ResolutionBitrateLimit();
+struct MEDIA_EXPORT ResolutionRateLimit {
+ ResolutionRateLimit();
+ ResolutionRateLimit(const ResolutionRateLimit&);
+ ResolutionRateLimit(const gfx::Size& frame_size,
+ int min_start_bitrate_bps,
+ int min_bitrate_bps,
+ int max_bitrate_bps,
+ uint32_t max_framerate_numerator,
+ uint32_t max_framerate_denominator);
+ ~ResolutionRateLimit();
gfx::Size frame_size;
int min_start_bitrate_bps = 0;
int min_bitrate_bps = 0;
int max_bitrate_bps = 0;
+ uint32_t max_framerate_numerator = 0;
+ uint32_t max_framerate_denominator = 0;
};
-MEDIA_EXPORT bool operator==(const ResolutionBitrateLimit& lhs,
- const ResolutionBitrateLimit& rhs);
+MEDIA_EXPORT bool operator==(const ResolutionRateLimit& lhs,
+ const ResolutionRateLimit& rhs);
struct MEDIA_EXPORT VideoEncoderInfo {
static constexpr size_t kMaxSpatialLayers = 5;
@@ -82,7 +86,8 @@
size_t number_of_manual_reference_buffers = 0;
std::array<std::vector<uint8_t>, kMaxSpatialLayers> fps_allocation;
- std::vector<ResolutionBitrateLimit> resolution_bitrate_limits;
+
+ std::vector<ResolutionRateLimit> resolution_rate_limits;
// Set of pixel formats that this encoder can handle on the gpu
// withhout readback.
diff --git a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
index 98af676..8e68432 100644
--- a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
+++ b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
@@ -2834,7 +2834,7 @@
media_enc_info.fps_allocation[i].begin(),
media_enc_info.fps_allocation[i].end());
}
- for (const auto& limit : media_enc_info.resolution_bitrate_limits) {
+ for (const auto& limit : media_enc_info.resolution_rate_limits) {
encoder_info_.resolution_bitrate_limits.emplace_back(
limit.frame_size.GetArea(), limit.min_start_bitrate_bps,
limit.min_bitrate_bps, limit.max_bitrate_bps);
