Implement h.265 parameter sets tracker.
RTP spec for H.265 requires parameter sets to be included in IRAP
pictures. This CL adds the helper for rtc_video_encoder to deliver
encoded bitstream for all VEA outputs.
This CL is based on Jianlin's CL https://crrev.com/c/5307256.
Co-authored-by: Jianlin Qiu <jianlin.qiu@intel.com>
Bug: webrtc:13485
Change-Id: I6cfba23c957fdc90deb66b7e580c8a20cd22aef3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5335904
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Commit-Queue: Jianlin Qiu <jianlin.qiu@intel.com>
Cr-Commit-Position: refs/heads/main@{#1277264}
diff --git a/third_party/blink/public/common/BUILD.gn b/third_party/blink/public/common/BUILD.gn
index f8b52280..7be5480 100644
--- a/third_party/blink/public/common/BUILD.gn
+++ b/third_party/blink/public/common/BUILD.gn
@@ -84,6 +84,7 @@
header = "buildflags.h"
flags = [
"RTC_USE_H264=$rtc_use_h264",
+ "RTC_USE_H265=$rtc_use_h265",
"ENABLE_RUST_CRASH=$enable_rust_crash",
"USE_FONTATIONS_BACKEND=$use_typeface_fontations",
]
diff --git a/third_party/blink/renderer/platform/BUILD.gn b/third_party/blink/renderer/platform/BUILD.gn
index 00f58fa5..d940276 100644
--- a/third_party/blink/renderer/platform/BUILD.gn
+++ b/third_party/blink/renderer/platform/BUILD.gn
@@ -17,6 +17,7 @@
import("//third_party/blink/renderer/config.gni")
import("//third_party/blink/renderer/platform/platform_generated.gni")
import("//third_party/libgav1/options.gni")
+import("//third_party/webrtc/webrtc.gni")
import("//v8/gni/v8.gni")
if (is_ios) {
@@ -1870,6 +1871,13 @@
deps += [ "//third_party/pffft" ]
}
+ if (rtc_use_h265) {
+ sources += [
+ "peerconnection/h265_parameter_sets_tracker.cc",
+ "peerconnection/h265_parameter_sets_tracker.h",
+ ]
+ }
+
if (!is_debug && !optimize_for_size) {
configs -= [ "//build/config/compiler:default_optimization" ]
configs += [ "//build/config/compiler:optimize_max" ]
@@ -2285,6 +2293,10 @@
"widget/input/scroll_predictor_unittest.cc",
]
+ if (rtc_use_h265) {
+ sources += [ "peerconnection/h265_parameter_sets_tracker_unittest.cc" ]
+ }
+
if (is_android) {
sources += [ "fonts/android/font_cache_android_test.cc" ]
} else if (is_win) {
@@ -2588,6 +2600,17 @@
dict = "//testing/libfuzzer/fuzzers/dicts/mathml_operator_dictionary.dict"
}
+if (rtc_use_h265) {
+ # Fuzzer for blink::H265ParameterSetsTracker
+ fuzzer_test("h265_parameter_sets_tracker_fuzzer") {
+ sources = [ "peerconnection/h265_parameter_sets_tracker_fuzzer.cc" ]
+ deps = [
+ ":blink_fuzzer_test_support",
+ ":platform",
+ ]
+ }
+}
+
# Fuzzer for blink::OpenTypeMathSupportTest.
fuzzer_test("open_type_math_support_fuzzer") {
sources = [ "fonts/opentype/open_type_math_support_fuzzer.cc" ]
diff --git a/third_party/blink/renderer/platform/peerconnection/DEPS b/third_party/blink/renderer/platform/peerconnection/DEPS
index 8994d0de..67c69f6 100644
--- a/third_party/blink/renderer/platform/peerconnection/DEPS
+++ b/third_party/blink/renderer/platform/peerconnection/DEPS
@@ -51,5 +51,9 @@
],
"resolution_monitor_unittest.cc" : [
"+media/filters/ivf_parser.h",
+ ],
+ "h265_parameter_sets_tracker_fuzzer.cc" : [
+ "+third_party/blink/renderer/platform/testing/blink_fuzzer_test_support.h",
+ "+third_party/blink/renderer/platform/testing/fuzzed_data_provider.h",
]
}
diff --git a/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.cc b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.cc
new file mode 100644
index 0000000..d73a820
--- /dev/null
+++ b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.cc
@@ -0,0 +1,236 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h"
+
+#include <memory>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "base/check.h"
+#include "base/logging.h"
+#include "third_party/webrtc/common_video/h265/h265_common.h"
+#include "third_party/webrtc/common_video/h265/h265_pps_parser.h"
+#include "third_party/webrtc/common_video/h265/h265_sps_parser.h"
+#include "third_party/webrtc/common_video/h265/h265_vps_parser.h"
+
+namespace blink {
+
+namespace {
+constexpr size_t kMaxParameterSetSizeBytes = 1024;
+}
+
+H265ParameterSetsTracker::H265ParameterSetsTracker() = default;
+H265ParameterSetsTracker::~H265ParameterSetsTracker() = default;
+
+H265ParameterSetsTracker::PpsData::PpsData() = default;
+H265ParameterSetsTracker::PpsData::PpsData(PpsData&& rhs) = default;
+H265ParameterSetsTracker::PpsData& H265ParameterSetsTracker::PpsData::operator=(
+ PpsData&& rhs) = default;
+H265ParameterSetsTracker::PpsData::~PpsData() = default;
+
+H265ParameterSetsTracker::SpsData::SpsData() = default;
+H265ParameterSetsTracker::SpsData::SpsData(SpsData&& rhs) = default;
+H265ParameterSetsTracker::SpsData& H265ParameterSetsTracker::SpsData::operator=(
+ SpsData&& rhs) = default;
+H265ParameterSetsTracker::SpsData::~SpsData() = default;
+
+H265ParameterSetsTracker::VpsData::VpsData() = default;
+H265ParameterSetsTracker::VpsData::VpsData(VpsData&& rhs) = default;
+H265ParameterSetsTracker::VpsData& H265ParameterSetsTracker::VpsData::operator=(
+ VpsData&& rhs) = default;
+H265ParameterSetsTracker::VpsData::~VpsData() = default;
+
+H265ParameterSetsTracker::FixedBitstream
+H265ParameterSetsTracker::MaybeFixBitstream(
+ rtc::ArrayView<const uint8_t> bitstream) {
+ if (!bitstream.size()) {
+ return {PacketAction::kRequestKeyframe};
+ }
+
+ bool has_irap_nalu = false;
+ bool prepend_vps = true, prepend_sps = true, prepend_pps = true;
+
+ // Required size of fixed bitstream.
+ size_t required_size = 0;
+ H265ParameterSetsTracker::FixedBitstream fixed;
+ fixed.action = PacketAction::kPassThrough;
+
+ auto vps_data = vps_data_.end();
+ auto sps_data = sps_data_.end();
+ auto pps_data = pps_data_.end();
+ std::optional<uint32_t> pps_id;
+ uint32_t sps_id = 0, vps_id = 0;
+ uint32_t slice_sps_id = 0, slice_pps_id = 0;
+
+ parser_.ParseBitstream(
+ rtc::ArrayView<const uint8_t>(bitstream.data(), bitstream.size()));
+
+ std::vector<webrtc::H265::NaluIndex> nalu_indices =
+ webrtc::H265::FindNaluIndices(bitstream.data(), bitstream.size());
+ for (const auto& nalu_index : nalu_indices) {
+ if (nalu_index.payload_size < 2) {
+ // H.265 NALU header is at least 2 bytes.
+ return {PacketAction::kRequestKeyframe};
+ }
+ const uint8_t* payload_start =
+ bitstream.data() + nalu_index.payload_start_offset;
+ const uint8_t* nalu_start = bitstream.data() + nalu_index.start_offset;
+ size_t nalu_size = nalu_index.payload_size +
+ nalu_index.payload_start_offset -
+ nalu_index.start_offset;
+ uint8_t nalu_type = webrtc::H265::ParseNaluType(payload_start[0]);
+
+ std::optional<webrtc::H265VpsParser::VpsState> vps;
+ std::optional<webrtc::H265SpsParser::SpsState> sps;
+
+ switch (nalu_type) {
+ case webrtc::H265::NaluType::kVps:
+ // H.265 parameter set parsers expect NALU header already stripped.
+ vps = webrtc::H265VpsParser::ParseVps(payload_start + 2,
+ nalu_index.payload_size - 2);
+ // Always replace VPS with the same ID. Same for other parameter sets.
+ if (vps) {
+ std::unique_ptr<VpsData> current_vps_data =
+ std::make_unique<VpsData>();
+ // Copy with start code included. Same for other parameter sets.
+ if (!current_vps_data.get() || !nalu_size ||
+ nalu_size > kMaxParameterSetSizeBytes) {
+ return {PacketAction::kRequestKeyframe};
+ }
+ current_vps_data->size = nalu_size;
+ uint8_t* vps_payload = new uint8_t[current_vps_data->size];
+ memcpy(vps_payload, nalu_start, current_vps_data->size);
+ current_vps_data->payload.reset(vps_payload);
+ vps_data_.Set(vps->id, std::move(current_vps_data));
+ }
+ prepend_vps = false;
+ break;
+ case webrtc::H265::NaluType::kSps:
+ sps = webrtc::H265SpsParser::ParseSps(payload_start + 2,
+ nalu_index.payload_size - 2);
+ if (sps) {
+ std::unique_ptr<SpsData> current_sps_data =
+ std::make_unique<SpsData>();
+ if (!current_sps_data.get() || !nalu_size ||
+ nalu_size > kMaxParameterSetSizeBytes) {
+ return {PacketAction::kRequestKeyframe};
+ }
+ current_sps_data->size = nalu_size;
+ current_sps_data->vps_id = sps->vps_id;
+ uint8_t* sps_payload = new uint8_t[current_sps_data->size];
+ memcpy(sps_payload, nalu_start, current_sps_data->size);
+ current_sps_data->payload.reset(sps_payload);
+ sps_data_.Set(sps->sps_id, std::move(current_sps_data));
+ }
+ prepend_sps = false;
+ break;
+ case webrtc::H265::NaluType::kPps:
+ if (webrtc::H265PpsParser::ParsePpsIds(payload_start + 2,
+ nalu_index.payload_size - 2,
+ &slice_pps_id, &slice_sps_id)) {
+ auto current_sps_data = sps_data_.find(slice_sps_id);
+ if (current_sps_data == sps_data_.end()) {
+ DLOG(WARNING) << "No SPS associated with current parsed PPS found.";
+ fixed.action = PacketAction::kRequestKeyframe;
+ } else {
+ std::unique_ptr<PpsData> current_pps_data =
+ std::make_unique<PpsData>();
+ if (!current_pps_data.get() || !nalu_size ||
+ nalu_size > kMaxParameterSetSizeBytes) {
+ return {PacketAction::kRequestKeyframe};
+ }
+ current_pps_data->size = nalu_size;
+ current_pps_data->sps_id = slice_sps_id;
+ uint8_t* pps_payload = new uint8_t[current_pps_data->size];
+ memcpy(pps_payload, nalu_start, current_pps_data->size);
+ current_pps_data->payload.reset(pps_payload);
+ pps_data_.Set(slice_pps_id, std::move(current_pps_data));
+ }
+ prepend_pps = false;
+ }
+ break;
+ case webrtc::H265::NaluType::kBlaWLp:
+ case webrtc::H265::NaluType::kBlaWRadl:
+ case webrtc::H265::NaluType::kBlaNLp:
+ case webrtc::H265::NaluType::kIdrWRadl:
+ case webrtc::H265::NaluType::kIdrNLp:
+ case webrtc::H265::NaluType::kCra:
+ has_irap_nalu = true;
+ pps_id = parser_.GetLastSlicePpsId();
+ if (!pps_id) {
+ DLOG(WARNING) << "Failed to parse PPS id from current slice.";
+ fixed.action = PacketAction::kRequestKeyframe;
+ break;
+ }
+ pps_data = pps_data_.find(pps_id.value());
+ if (pps_data == pps_data_.end()) {
+ DLOG(WARNING) << "PPS associated with current slice is not found.";
+ fixed.action = PacketAction::kRequestKeyframe;
+ break;
+ }
+
+ sps_id = (pps_data->value)->sps_id;
+ sps_data = sps_data_.find(sps_id);
+ if (sps_data == sps_data_.end()) {
+ DLOG(WARNING) << "SPS associated with current slice is not found.";
+ fixed.action = PacketAction::kRequestKeyframe;
+ break;
+ }
+
+ vps_id = (sps_data->value)->vps_id;
+ vps_data = vps_data_.find(vps_id);
+ if (vps_data == vps_data_.end()) {
+ DLOG(WARNING) << "VPS associated with current slice is not found.";
+ fixed.action = PacketAction::kRequestKeyframe;
+ break;
+ }
+
+ if (!prepend_vps && !prepend_sps && !prepend_pps) {
+ fixed.action = PacketAction::kPassThrough;
+ } else {
+ required_size += vps_data->value->size + sps_data->value->size +
+ pps_data->value->size;
+
+ required_size += bitstream.size();
+ size_t offset = 0;
+
+ fixed.bitstream = webrtc::EncodedImageBuffer::Create(required_size);
+ memcpy(fixed.bitstream->data(), vps_data->value->payload.get(),
+ vps_data->value->size);
+ offset += vps_data->value->size;
+ memcpy(fixed.bitstream->data() + offset,
+ sps_data->value->payload.get(), sps_data->value->size);
+ offset += sps_data->value->size;
+ memcpy(fixed.bitstream->data() + offset,
+ pps_data->value->payload.get(), pps_data->value->size);
+ offset += pps_data->value->size;
+ memcpy(fixed.bitstream->data() + offset, bitstream.data(),
+ bitstream.size());
+
+ fixed.action = PacketAction::kInsert;
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (fixed.action == PacketAction::kRequestKeyframe) {
+ return {PacketAction::kRequestKeyframe};
+ } else if (fixed.action == PacketAction::kInsert) {
+ return fixed;
+ }
+
+ if (has_irap_nalu) {
+ break;
+ }
+ }
+
+ fixed.action = PacketAction::kPassThrough;
+
+ return fixed;
+}
+
+} // namespace blink
diff --git a/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h
new file mode 100644
index 0000000..5fa8746
--- /dev/null
+++ b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h
@@ -0,0 +1,128 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_PEERCONNECTION_H265_PARAMETER_SETS_TRACKER_H_
+#define THIRD_PARTY_BLINK_RENDERER_PLATFORM_PEERCONNECTION_H265_PARAMETER_SETS_TRACKER_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+#include "third_party/blink/renderer/platform/platform_export.h"
+#include "third_party/blink/renderer/platform/wtf/hash_functions.h"
+#include "third_party/blink/renderer/platform/wtf/hash_map.h"
+#include "third_party/blink/renderer/platform/wtf/hash_traits.h"
+#include "third_party/webrtc/api/array_view.h"
+#include "third_party/webrtc/common_video/h265/h265_bitstream_parser.h"
+#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
+
+namespace blink {
+
+// This is used on H.265 sender side to ensure we are always sending
+// bitstream that has parameter set NALUs enclosed into the H.265 IRAP frames.
+// Unlike H.264, the tracker is not intended to be used by receiver side
+// for attempt to fix received bitstream. H.265 receiver must always issue key
+// frame request if parameter set is not part of IRAP picture.
+// For more details, refer to the "sprop-sps, sprop-pps, sprop-vps, sprop-sei:"
+// section of
+// https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-hevc-webrtc-01
+//
+// Parameter sets supported by this tracker include VPS(video parameter set),
+// SPS(sequence parameter set) and PPS(picture parameter set). They are defined
+// in section 7.3.2.1, 7.3.2.2 and 7.3.2.3 of ITU H.265: High efficiency video
+// coding (https://www.itu.int/rec/T-REC-H.265, version 09/23)
+class PLATFORM_EXPORT H265ParameterSetsTracker {
+ public:
+ enum class PacketAction : uint8_t {
+ kInsert = 0,
+ kRequestKeyframe,
+ kPassThrough,
+ };
+ struct FixedBitstream {
+ PacketAction action;
+ rtc::scoped_refptr<webrtc::EncodedImageBuffer> bitstream;
+ };
+
+ H265ParameterSetsTracker();
+ virtual ~H265ParameterSetsTracker();
+
+ // Keeps track of incoming bitstream and insert VPS/SPS/PPS before the VCL
+ // layer NALUs when needed.
+ // Once VPS/SPS/PPS is detected in the bitstream, it will be recorded, and
+ // if an IRAP picture is passed in without associated VPS/SPS/PPS in the
+ // bitstream, will return the fixed bitstream with action set to kInsert; If
+ // the incoming bitstream already contains necessary parameter sets, or
+ // incoming bitstream does not contain IRAP pictures, the returned
+ // FixedBistream's |bitstream member| is not set, and |action| will be set to
+ // kPassThrough; If the incoming bitstream needs to be fixed but corresponding
+ // parameter set is not found, the returned FixedBitstream will get |action|
+ // set to kRequestkeyframe, and its |bitstream| member will not be set.
+ virtual FixedBitstream MaybeFixBitstream(
+ rtc::ArrayView<const uint8_t> bitstream);
+
+ private:
+ // Stores PPS payload and the active SPS ID.
+ struct PpsData {
+ PpsData();
+ PpsData(PpsData&& rhs);
+ PpsData& operator=(PpsData&& rhs);
+ ~PpsData();
+
+ // The value of sps_seq_parameter_set_id for the active SPS.
+ uint32_t sps_id = 0;
+ // Payload size.
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> payload;
+ };
+
+ // Stores SPS payload and the active VPS ID.
+ struct SpsData {
+ SpsData();
+ SpsData(SpsData&& rhs);
+ SpsData& operator=(SpsData&& rhs);
+ ~SpsData();
+
+ // The value of the vps_video_parameter_set_id of the active VPS.
+ uint32_t vps_id = 0;
+ // Payload size.
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> payload;
+ };
+
+ // Stores VPS payload.
+ struct VpsData {
+ VpsData();
+ VpsData(VpsData&& rhs);
+ VpsData& operator=(VpsData&& rhs);
+ ~VpsData();
+
+ // Payload size.
+ size_t size = 0;
+ std::unique_ptr<uint8_t[]> payload;
+ };
+
+ webrtc::H265BitstreamParser parser_;
+ // Map from vps_video_parameter_set_id to the VPS payload associated with this
+ // ID.
+ WTF::HashMap<uint32_t,
+ std::unique_ptr<PpsData>,
+ IntWithZeroKeyHashTraits<uint32_t>>
+ pps_data_;
+ // Map from sps_video_parameter_set_id to the SPS payload associated with this
+ // ID.
+ WTF::HashMap<uint32_t,
+ std::unique_ptr<SpsData>,
+ IntWithZeroKeyHashTraits<uint32_t>>
+ sps_data_;
+ // Map from pps_pic_parameter_set_id to the PPS payload associated with this
+ // ID.
+ WTF::HashMap<uint32_t,
+ std::unique_ptr<VpsData>,
+ IntWithZeroKeyHashTraits<uint32_t>>
+ vps_data_;
+};
+
+} // namespace blink
+
+#endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_PEERCONNECTION_H265_PARAMETER_SETS_TRACKER_H_
diff --git a/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_fuzzer.cc b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_fuzzer.cc
new file mode 100644
index 0000000..a76cbf6
--- /dev/null
+++ b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_fuzzer.cc
@@ -0,0 +1,19 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdint.h>
+
+#include "third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h"
+#include "third_party/blink/renderer/platform/testing/blink_fuzzer_test_support.h"
+#include "third_party/blink/renderer/platform/testing/fuzzed_data_provider.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+ static blink::BlinkFuzzerTestSupport test_support =
+ blink::BlinkFuzzerTestSupport();
+
+ blink::H265ParameterSetsTracker h265_parameter_sets_tracker;
+ h265_parameter_sets_tracker.MaybeFixBitstream(
+ rtc::ArrayView<const uint8_t>(data, size));
+ return 0;
+}
diff --git a/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_unittest.cc b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_unittest.cc
new file mode 100644
index 0000000..bd2577a
--- /dev/null
+++ b/third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker_unittest.cc
@@ -0,0 +1,382 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h"
+
+#include <string.h>
+#include <vector>
+
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace blink {
+namespace {
+
+// VPS/SPS/PPS/IDR for a 1280x720 camera capture from ffmpeg on linux.
+// Contains emulation bytes but no cropping. This buffer is generated with
+// following command: 1) ffmpeg -i /dev/video0 -r 30 -c:v libx265 -s 1280x720
+// camera.h265
+//
+// The VPS/SPS/PPS are kept intact while idr1/idr2/cra1/cra2/trail1/trail2 are
+// created by changing the NALU type of original IDR/TRAIL_R NALUs, and
+// truncated only for testing of the tracker.
+uint8_t vps[] = {0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff,
+ 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03,
+ 0x00, 0x00, 0x03, 0x00, 0x5d, 0x95, 0x98, 0x09};
+uint8_t sps[] = {0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60,
+ 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00,
+ 0x00, 0x03, 0x00, 0x5d, 0xa0, 0x02, 0x80, 0x80, 0x2d,
+ 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x5a, 0x70,
+ 0x80, 0x00, 0x01, 0xf4, 0x80, 0x00, 0x3a, 0x98, 0x04};
+uint8_t pps[] = {0x00, 0x00, 0x00, 0x01, 0x44, 0x01,
+ 0xc1, 0x72, 0xb4, 0x62, 0x40};
+uint8_t idr1[] = {0x00, 0x00, 0x00, 0x01, 0x28, 0x01, 0xaf,
+ 0x08, 0x46, 0x0c, 0x92, 0xa3, 0xf4, 0x77};
+uint8_t idr2[] = {0x00, 0x00, 0x00, 0x01, 0x28, 0x01, 0xaf,
+ 0x08, 0x46, 0x0c, 0x92, 0xa3, 0xf4, 0x77};
+uint8_t trail1[] = {0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0xa4, 0x04, 0x55,
+ 0xa2, 0x6d, 0xce, 0xc0, 0xc3, 0xed, 0x0b, 0xac, 0xbc,
+ 0x00, 0xc4, 0x44, 0x2e, 0xf7, 0x55, 0xfd, 0x05, 0x86};
+uint8_t trail2[] = {0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x23, 0xfc, 0x20,
+ 0x22, 0xad, 0x13, 0x68, 0xce, 0xc3, 0x5a, 0x00, 0x01,
+ 0x80, 0xe9, 0xc6, 0x38, 0x13, 0xec, 0xef, 0x0f, 0xff};
+uint8_t cra[] = {0x00, 0x00, 0x00, 0x01, 0x2A, 0x01, 0xad, 0x00, 0x58, 0x81,
+ 0x04, 0x11, 0xc2, 0x00, 0x44, 0x3f, 0x34, 0x46, 0x3e, 0xcc,
+ 0x86, 0xd9, 0x3f, 0xf1, 0xe1, 0xda, 0x26, 0xb1, 0xc5, 0x50,
+ 0xf2, 0x8b, 0x8d, 0x0c, 0xe9, 0xe1, 0xd3, 0xe0, 0xa7, 0x3e};
+
+// Below two H264 binaries are copied from h264 bitstream parser unittests,
+// to check the behavior of the tracker on stream from mismatched encoder.
+uint8_t sps_pps_h264[] = {0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x80, 0x20, 0xda,
+ 0x01, 0x40, 0x16, 0xe8, 0x06, 0xd0, 0xa1, 0x35, 0x00,
+ 0x00, 0x00, 0x01, 0x68, 0xce, 0x06, 0xe2};
+uint8_t idr_h264[] = {
+ 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x80, 0x20, 0xda, 0x01, 0x40, 0x16,
+ 0xe8, 0x06, 0xd0, 0xa1, 0x35, 0x00, 0x00, 0x00, 0x01, 0x68, 0xce, 0x06,
+ 0xe2, 0x00, 0x00, 0x00, 0x01, 0x65, 0xb8, 0x40, 0xf0, 0x8c, 0x03, 0xf2,
+ 0x75, 0x67, 0xad, 0x41, 0x64, 0x24, 0x0e, 0xa0, 0xb2, 0x12, 0x1e, 0xf8,
+};
+
+using ::testing::ElementsAreArray;
+
+rtc::ArrayView<const uint8_t> Bitstream(
+ const H265ParameterSetsTracker::FixedBitstream& fixed) {
+ return rtc::ArrayView<const uint8_t>(fixed.bitstream->data(),
+ fixed.bitstream->size());
+}
+
+} // namespace
+
+class H265ParameterSetsTrackerTest : public ::testing::Test {
+ public:
+ H265ParameterSetsTracker tracker_;
+};
+
+TEST_F(H265ParameterSetsTrackerTest, NoNalus) {
+ uint8_t data[] = {1, 2, 3};
+
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest, StreamFromMissMatchingH26xCodec) {
+ std::vector<uint8_t> data;
+ unsigned sps_pps_size = sizeof(sps_pps_h264) / sizeof(sps_pps_h264[0]);
+ unsigned idr_size = sizeof(idr_h264) / sizeof(idr_h264[0]);
+ data.insert(data.end(), sps_pps_h264, sps_pps_h264 + sps_pps_size);
+ data.insert(data.end(), idr_h264, idr_h264 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ // This is not an H.265 stream. We simply pass through it.
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest, AllParameterSetsInCurrentIdrSingleSlice) {
+ std::vector<uint8_t> data;
+ data.clear();
+ unsigned vps_size = sizeof(vps) / sizeof(uint8_t);
+ unsigned sps_size = sizeof(sps) / sizeof(uint8_t);
+ unsigned pps_size = sizeof(pps) / sizeof(uint8_t);
+ unsigned idr_size = sizeof(idr1) / sizeof(uint8_t);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size - 1);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest, AllParameterSetsMissingForIdr) {
+ std::vector<uint8_t> data;
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kRequestKeyframe);
+}
+
+TEST_F(H265ParameterSetsTrackerTest, VpsMissingForIdr) {
+ std::vector<uint8_t> data;
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kRequestKeyframe);
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ ParameterSetsSeenBeforeButRepeatedVpsMissingForCurrentIdr) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ // Second IDR but encoder only repeats SPS/PPS(unlikely to happen).
+ std::vector<uint8_t> frame2;
+ unsigned sps2_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps2_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr2_size = sizeof(idr2) / sizeof(idr2[0]);
+ frame2.insert(frame2.end(), sps, sps + sps2_size);
+ frame2.insert(frame2.end(), pps, pps + pps2_size);
+ frame2.insert(frame2.end(), idr2, idr2 + idr2_size);
+ fixed = tracker_.MaybeFixBitstream(frame2);
+
+ // If any of the parameter set is missing, we append all of VPS/SPS/PPS and it
+ // is fine to repeat any of the parameter set twice for current IDR.
+ EXPECT_THAT(fixed.action, H265ParameterSetsTracker::PacketAction::kInsert);
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), vps, vps + vps_size);
+ expected.insert(expected.end(), sps, sps + sps_size);
+ expected.insert(expected.end(), pps, pps + pps_size);
+ expected.insert(expected.end(), sps, sps + sps_size);
+ expected.insert(expected.end(), pps, pps + pps_size);
+ expected.insert(expected.end(), idr2, idr2 + idr2_size);
+ EXPECT_THAT(Bitstream(fixed), ElementsAreArray(expected));
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ AllParameterSetsInCurrentIdrMulitpleSlices) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr1_size = sizeof(idr1) / sizeof(idr1[0]);
+ unsigned idr2_size = sizeof(idr2) / sizeof(idr2[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr1_size);
+ data.insert(data.end(), idr2, idr2 + idr2_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ SingleDeltaSliceWithoutParameterSetsBefore) {
+ std::vector<uint8_t> data;
+ unsigned trail_size = sizeof(trail1) / sizeof(trail1[0]);
+ data.insert(data.end(), trail1, trail1 + trail_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ MultipleDeltaSlicseWithoutParameterSetsBefore) {
+ std::vector<uint8_t> data;
+ unsigned trail1_size = sizeof(trail1) / sizeof(trail1[0]);
+ unsigned trail2_size = sizeof(trail2) / sizeof(trail2[0]);
+ data.insert(data.end(), trail1, trail1 + trail1_size);
+ data.insert(data.end(), trail2, trail2 + trail2_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ ParameterSetsInPreviousIdrNotInCurrentIdr) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ std::vector<uint8_t> frame2;
+ unsigned idr2_size = sizeof(idr2) / sizeof(idr2[0]);
+ frame2.insert(frame2.end(), idr2, idr2 + idr2_size);
+ fixed = tracker_.MaybeFixBitstream(frame2);
+
+ EXPECT_THAT(fixed.action, H265ParameterSetsTracker::PacketAction::kInsert);
+
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), vps, vps + vps_size);
+ expected.insert(expected.end(), sps, sps + sps_size);
+ expected.insert(expected.end(), pps, pps + pps_size);
+ expected.insert(expected.end(), idr2, idr2 + idr2_size);
+ EXPECT_THAT(Bitstream(fixed), ElementsAreArray(expected));
+}
+
+TEST_F(H265ParameterSetsTrackerTest,
+ ParameterSetsInPreviousIdrNotInCurrentCra) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ std::vector<uint8_t> frame2;
+ unsigned cra_size = sizeof(cra) / sizeof(cra[0]);
+ frame2.insert(frame2.end(), cra, cra + cra_size);
+ fixed = tracker_.MaybeFixBitstream(frame2);
+
+ EXPECT_THAT(fixed.action, H265ParameterSetsTracker::PacketAction::kInsert);
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), vps, vps + vps_size);
+ expected.insert(expected.end(), sps, sps + sps_size);
+ expected.insert(expected.end(), pps, pps + pps_size);
+ expected.insert(expected.end(), cra, cra + cra_size);
+ EXPECT_THAT(Bitstream(fixed), ElementsAreArray(expected));
+}
+
+TEST_F(H265ParameterSetsTrackerTest, ParameterSetsInBothPreviousAndCurrentIdr) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ std::vector<uint8_t> frame2;
+ unsigned idr2_size = sizeof(idr2) / sizeof(idr2[0]);
+ frame2.insert(frame2.end(), vps, vps + vps_size);
+ frame2.insert(frame2.end(), sps, sps + sps_size);
+ frame2.insert(frame2.end(), pps, pps + pps_size);
+ frame2.insert(frame2.end(), idr2, idr2 + idr2_size);
+ fixed = tracker_.MaybeFixBitstream(frame2);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+TEST_F(H265ParameterSetsTrackerTest, TwoGopsWithIdrTrailAndCra) {
+ std::vector<uint8_t> data;
+ unsigned vps_size = sizeof(vps) / sizeof(vps[0]);
+ unsigned sps_size = sizeof(sps) / sizeof(sps[0]);
+ unsigned pps_size = sizeof(pps) / sizeof(pps[0]);
+ unsigned idr_size = sizeof(idr1) / sizeof(idr1[0]);
+ data.insert(data.end(), vps, vps + vps_size);
+ data.insert(data.end(), sps, sps + sps_size);
+ data.insert(data.end(), pps, pps + pps_size);
+ data.insert(data.end(), idr1, idr1 + idr_size);
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ tracker_.MaybeFixBitstream(data);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ // Second frame, a TRAIL_R picture.
+ std::vector<uint8_t> frame2;
+ unsigned trail_size = sizeof(trail1) / sizeof(trail1[0]);
+ frame2.insert(frame2.end(), trail1, trail1 + trail_size);
+ fixed = tracker_.MaybeFixBitstream(frame2);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ // Third frame, a TRAIL_R picture.
+ std::vector<uint8_t> frame3;
+ unsigned trail2_size = sizeof(trail2) / sizeof(trail2[0]);
+ frame3.insert(frame3.end(), trail2, trail2 + trail2_size);
+ fixed = tracker_.MaybeFixBitstream(frame3);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+
+ // Fourth frame, a CRA picture.
+ std::vector<uint8_t> frame4;
+ unsigned cra_size = sizeof(cra) / sizeof(cra[0]);
+ frame4.insert(frame4.end(), cra, cra + cra_size);
+ fixed = tracker_.MaybeFixBitstream(frame4);
+
+ EXPECT_THAT(fixed.action, H265ParameterSetsTracker::PacketAction::kInsert);
+
+ std::vector<uint8_t> expected;
+ expected.insert(expected.end(), vps, vps + vps_size);
+ expected.insert(expected.end(), sps, sps + sps_size);
+ expected.insert(expected.end(), pps, pps + pps_size);
+ expected.insert(expected.end(), cra, cra + cra_size);
+ EXPECT_THAT(Bitstream(fixed), ElementsAreArray(expected));
+
+ // Last frame, a TRAIL_R picture with 2 slices.
+ std::vector<uint8_t> frame5;
+ unsigned trail3_size = sizeof(trail1) / sizeof(trail1[0]);
+ unsigned trail4_size = sizeof(trail2) / sizeof(trail2[0]);
+ frame5.insert(frame5.end(), trail1, trail1 + trail3_size);
+ frame5.insert(frame5.end(), trail2, trail2 + trail4_size);
+ fixed = tracker_.MaybeFixBitstream(frame5);
+
+ EXPECT_THAT(fixed.action,
+ H265ParameterSetsTracker::PacketAction::kPassThrough);
+}
+
+} // namespace blink
diff --git a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
index 243b689a..6b2ca6fd 100644
--- a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
+++ b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.cc
@@ -517,6 +517,10 @@
return webrtc::kVideoCodecVP9;
case media::VideoCodec::kAV1:
return webrtc::kVideoCodecAV1;
+#if BUILDFLAG(RTC_USE_H265)
+ case media::VideoCodec::kHEVC:
+ return webrtc::kVideoCodecH265;
+#endif
default:
NOTREACHED() << "Invalid profile " << GetProfileName(profile);
return webrtc::kVideoCodecGeneric;
@@ -651,6 +655,11 @@
void NotifyErrorStatus(const media::EncoderStatus& status) override;
void NotifyEncoderInfoChange(const media::VideoEncoderInfo& info) override;
+#if BUILDFLAG(RTC_USE_H265)
+ void SetH265ParameterSetsTrackerForTesting(
+ std::unique_ptr<H265ParameterSetsTracker> tracker);
+#endif
+
private:
enum {
kInputBufferExtraCount = 1, // The number of input buffers allocated, more
@@ -798,6 +807,12 @@
// CreateAndInitializeVEA() and updated in RequestEncodingParametersChange().
ActiveSpatialLayers active_spatial_layers_;
+#if BUILDFLAG(RTC_USE_H265)
+ // Parameter sets(VPS/SPS/PPS) tracker used for H.265, to ensure parameter
+ // sets are always included in IRAP pictures.
+ std::unique_ptr<H265ParameterSetsTracker> ps_tracker_;
+#endif // BUILDFLAG(RTC_USE_H265)
+
// We cannot immediately return error conditions to the WebRTC user of this
// class, as there is no error callback in the webrtc::VideoEncoder interface.
// Instead, we cache an error status here and return it the next time an
@@ -896,6 +911,12 @@
active_spatial_layers_.begin_index = 0;
active_spatial_layers_.end_index = vea_config.spatial_layers.size();
+#if BUILDFLAG(RTC_USE_H265)
+ if (!ps_tracker_) {
+ ps_tracker_ = std::make_unique<H265ParameterSetsTracker>();
+ }
+#endif // BUILDFLAG(RTC_USE_H265)
+
// RequireBitstreamBuffers or NotifyError will be called and the waiter will
// be signaled.
}
@@ -1222,12 +1243,28 @@
}
webrtc::EncodedImage image;
- image.SetEncodedData(rtc::make_ref_counted<EncodedDataWrapper>(
- std::move(output_mapping), metadata.payload_size_bytes,
- base::BindPostTaskToCurrentDefault(
- base::BindOnce(&RTCVideoEncoder::Impl::BitstreamBufferAvailable,
- weak_this_, bitstream_buffer_id))));
+#if BUILDFLAG(RTC_USE_H265)
+ if (ps_tracker_.get()) {
+ H265ParameterSetsTracker::FixedBitstream fixed =
+ ps_tracker_->MaybeFixBitstream(rtc::MakeArrayView(
+ output_mapping->front(), metadata.payload_size_bytes));
+ if (fixed.action == H265ParameterSetsTracker::PacketAction::kInsert) {
+ image.SetEncodedData(fixed.bitstream);
+ BitstreamBufferAvailable(bitstream_buffer_id);
+ } else {
+#endif // BUILDFLAG(RTC_USE_H265)
+
+ image.SetEncodedData(rtc::make_ref_counted<EncodedDataWrapper>(
+ std::move(output_mapping), metadata.payload_size_bytes,
+ base::BindPostTaskToCurrentDefault(
+ base::BindOnce(&RTCVideoEncoder::Impl::BitstreamBufferAvailable,
+ weak_this_, bitstream_buffer_id))));
+#if BUILDFLAG(RTC_USE_H265)
+ }
+ }
+#endif // BUILDFLAG(RTC_USE_H265)
auto encoded_size = metadata.encoded_size.value_or(input_visible_size_);
+
image._encodedWidth = encoded_size.width();
image._encodedHeight = encoded_size.height();
image.SetRtpTimestamp(rtp_timestamp.value());
@@ -1754,6 +1791,13 @@
encoded_image_callback_ = callback;
}
+#if BUILDFLAG(RTC_USE_H265)
+void RTCVideoEncoder::Impl::SetH265ParameterSetsTrackerForTesting(
+ std::unique_ptr<H265ParameterSetsTracker> tracker) {
+ ps_tracker_ = std::move(tracker);
+}
+#endif
+
RTCVideoEncoder::RTCVideoEncoder(
media::VideoCodecProfile profile,
bool is_constrained_h264,
@@ -2248,4 +2292,15 @@
std::move(error_callback_for_testing_).Run();
}
+#if BUILDFLAG(RTC_USE_H265)
+void RTCVideoEncoder::SetH265ParameterSetsTrackerForTesting(
+ std::unique_ptr<H265ParameterSetsTracker> tracker) {
+ if (!impl_) {
+ DVLOG(1) << "Encoder is not initialized";
+ return;
+ }
+ impl_->SetH265ParameterSetsTrackerForTesting(std::move(tracker));
+}
+#endif
+
} // namespace blink
diff --git a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.h b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.h
index 7df912b..4e78be67 100644
--- a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.h
+++ b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder.h
@@ -19,12 +19,17 @@
#include "media/base/video_decoder_config.h"
#include "media/media_buildflags.h"
#include "media/video/video_encode_accelerator.h"
+#include "third_party/blink/public/common/features.h"
#include "third_party/blink/renderer/platform/platform_export.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/webrtc/api/video/video_bitrate_allocation.h"
#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
#include "ui/gfx/geometry/size.h"
+#if BUILDFLAG(RTC_USE_H265)
+#include "third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h"
+#endif // BUILDFLAG(RTC_USE_H265)
+
namespace base {
class SequencedTaskRunner;
}
@@ -79,6 +84,10 @@
WTF::CrossThreadOnceClosure error_callback_for_testing) {
error_callback_for_testing_ = std::move(error_callback_for_testing);
}
+#if BUILDFLAG(RTC_USE_H265)
+ void SetH265ParameterSetsTrackerForTesting(
+ std::unique_ptr<H265ParameterSetsTracker> tracker);
+#endif
private:
class Impl;
diff --git a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder_test.cc b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder_test.cc
index efcf285..767a0c9 100644
--- a/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder_test.cc
+++ b/third_party/blink/renderer/platform/peerconnection/rtc_video_encoder_test.cc
@@ -38,6 +38,9 @@
#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
#include "third_party/webrtc/rtc_base/ref_counted_object.h"
#include "third_party/webrtc/rtc_base/time_utils.h"
+#if BUILDFLAG(RTC_USE_H265)
+#include "third_party/blink/renderer/platform/peerconnection/h265_parameter_sets_tracker.h"
+#endif
using ::testing::_;
using ::testing::AllOf;
@@ -76,6 +79,8 @@
const uint16_t kSoftwareFallbackInputFrameHeight = 359;
#endif
+constexpr size_t kDefaultEncodedPayloadSize = 100;
+
const webrtc::VideoEncoder::Capabilities kVideoEncoderCapabilities(
/* loss_notification= */ false);
const webrtc::VideoEncoder::Settings
@@ -250,6 +255,26 @@
webrtc_encoder_thread_.FlushForTesting();
}
+#if BUILDFLAG(RTC_USE_H265)
+ void SetH265ParameterSetsTracker(
+ std::unique_ptr<H265ParameterSetsTracker> tracker) {
+ base::WaitableEvent waiter(base::WaitableEvent::ResetPolicy::MANUAL,
+ base::WaitableEvent::InitialState::NOT_SIGNALED);
+ task_runner_->PostTask(
+ FROM_HERE,
+ base::BindOnce(
+ [](RTCVideoEncoder* rtc_video_encoder,
+ std::unique_ptr<H265ParameterSetsTracker> tracker,
+ base::WaitableEvent* waiter) {
+ rtc_video_encoder->SetH265ParameterSetsTrackerForTesting(
+ std::move(tracker));
+ waiter->Signal();
+ },
+ rtc_video_encoder_.get(), std::move(tracker), &waiter));
+ waiter.Wait();
+ }
+#endif
+
private:
RTCVideoEncoderWrapper() : webrtc_encoder_thread_("WebRTC encoder thread") {
webrtc_encoder_thread_.Start();
@@ -365,6 +390,11 @@
case webrtc::kVideoCodecVP9:
media_profile = media::VP9PROFILE_PROFILE0;
break;
+#if BUILDFLAG(RTC_USE_H265)
+ case webrtc::kVideoCodecH265:
+ media_profile = media::HEVCPROFILE_MAIN;
+ break;
+#endif
default:
ADD_FAILURE() << "Unexpected codec type: " << codec_type;
media_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
@@ -489,8 +519,8 @@
void ReturnFrameWithTimeStamp(scoped_refptr<media::VideoFrame> frame,
bool force_keyframe) {
client_->BitstreamBufferReady(
- 0, media::BitstreamBufferMetadata(100, force_keyframe,
- frame->timestamp()));
+ 0, media::BitstreamBufferMetadata(kDefaultEncodedPayloadSize,
+ force_keyframe, frame->timestamp()));
}
void ReturnSVCLayerFrameWithVp9Metadata(
@@ -2530,6 +2560,106 @@
dropframe_verifier.Verify(1, 2);
}
+#if BUILDFLAG(RTC_USE_H265)
+class FakeH265ParameterSetsTracker : public H265ParameterSetsTracker {
+ public:
+ FakeH265ParameterSetsTracker() = delete;
+ explicit FakeH265ParameterSetsTracker(
+ H265ParameterSetsTracker::PacketAction action)
+ : action_(action) {}
+ explicit FakeH265ParameterSetsTracker(rtc::ArrayView<const uint8_t> prefix)
+ : action_(H265ParameterSetsTracker::PacketAction::kInsert),
+ prefix_(prefix) {
+ EXPECT_GT(prefix.size(), 0u);
+ }
+
+ FixedBitstream MaybeFixBitstream(
+ rtc::ArrayView<const uint8_t> bitstream) override {
+ FixedBitstream fixed;
+ fixed.action = action_;
+ if (prefix_.size() > 0) {
+ fixed.bitstream =
+ webrtc::EncodedImageBuffer::Create(bitstream.size() + prefix_.size());
+ memcpy(fixed.bitstream->data(), prefix_.data(), prefix_.size());
+ memcpy(fixed.bitstream->data() + prefix_.size(), bitstream.data(),
+ bitstream.size());
+ }
+ return fixed;
+ }
+
+ private:
+ H265ParameterSetsTracker::PacketAction action_;
+ rtc::ArrayView<const uint8_t> prefix_;
+};
+
+TEST_P(RTCVideoEncoderEncodeTest, EncodeH265WithBitstreamFix) {
+ class FixedBitstreamVerifier : public webrtc::EncodedImageCallback {
+ public:
+ explicit FixedBitstreamVerifier(rtc::ArrayView<const uint8_t> prefix,
+ size_t encoded_image_size)
+ : prefix_(prefix), encoded_image_size_(encoded_image_size) {}
+
+ webrtc::EncodedImageCallback::Result OnEncodedImage(
+ const webrtc::EncodedImage& encoded_image,
+ const webrtc::CodecSpecificInfo* codec_specific_info) override {
+ EXPECT_EQ(encoded_image.size(), encoded_image_size_ + prefix_.size());
+ EXPECT_THAT(
+ rtc::ArrayView<const uint8_t>(encoded_image.data(), prefix_.size()),
+ ::testing::ElementsAreArray(prefix_));
+ waiter_.Signal();
+ return Result(Result::OK);
+ }
+
+ void Wait() { waiter_.Wait(); }
+
+ private:
+ base::WaitableEvent waiter_;
+ rtc::ArrayView<const uint8_t> prefix_;
+ size_t encoded_image_size_;
+ };
+
+ const webrtc::VideoCodecType codec_type = webrtc::kVideoCodecH265;
+ CreateEncoder(codec_type);
+ webrtc::VideoCodec codec = GetDefaultCodec();
+ codec.codecType = codec_type;
+ if (!InitializeOnFirstFrameEnabled()) {
+ ExpectCreateInitAndDestroyVEA();
+ }
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ rtc_encoder_->InitEncode(&codec, kVideoEncoderSettings));
+
+ uint8_t prefix[] = {0x90, 0x91, 0x92, 0x93};
+ rtc::ArrayView<uint8_t> prefix_view =
+ rtc::ArrayView<uint8_t>(prefix, sizeof(prefix));
+ rtc_encoder_->SetH265ParameterSetsTracker(
+ std::make_unique<FakeH265ParameterSetsTracker>(prefix_view));
+ FixedBitstreamVerifier bitstream_verifier(prefix_view,
+ kDefaultEncodedPayloadSize);
+ rtc_encoder_->RegisterEncodeCompleteCallback(&bitstream_verifier);
+
+ if (InitializeOnFirstFrameEnabled()) {
+ ExpectCreateInitAndDestroyVEA();
+ }
+ EXPECT_CALL(*mock_vea_, Encode(_, _))
+ .WillOnce(Invoke(this, &RTCVideoEncoderTest::ReturnFrameWithTimeStamp));
+
+ const rtc::scoped_refptr<webrtc::I420Buffer> buffer =
+ webrtc::I420Buffer::Create(kInputFrameWidth, kInputFrameHeight);
+ FillFrameBuffer(buffer);
+ std::vector<webrtc::VideoFrameType> frame_types;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ rtc_encoder_->Encode(webrtc::VideoFrame::Builder()
+ .set_video_frame_buffer(buffer)
+ .set_timestamp_rtp(0)
+ .set_timestamp_us(0)
+ .set_rotation(webrtc::kVideoRotation_0)
+ .build(),
+ &frame_types));
+ RunUntilIdle();
+}
+#endif
+
const RTCVideoEncoderEncodeTestParam kEncodeTestCases[] = {
{false},
{true},