| // Copyright 2020 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "third_party/blink/renderer/modules/webcodecs/video_encoder.h" |
| |
| #include <string> |
| |
| #include "base/bind.h" |
| #include "base/callback.h" |
| #include "base/callback_helpers.h" |
| #include "base/containers/contains.h" |
| #include "base/cxx17_backports.h" |
| #include "base/feature_list.h" |
| #include "base/logging.h" |
| #include "base/metrics/histogram_macros.h" |
| #include "base/trace_event/common/trace_event_common.h" |
| #include "base/trace_event/trace_event.h" |
| #include "build/build_config.h" |
| #include "build/os_buildflags.h" |
| #include "components/viz/common/gpu/raster_context_provider.h" |
| #include "gpu/GLES2/gl2extchromium.h" |
| #include "gpu/command_buffer/client/raster_interface.h" |
| #include "media/base/async_destroy_video_encoder.h" |
| #include "media/base/bind_to_current_loop.h" |
| #include "media/base/mime_util.h" |
| #include "media/base/offloading_video_encoder.h" |
| #include "media/base/svc_scalability_mode.h" |
| #include "media/base/video_codecs.h" |
| #include "media/base/video_color_space.h" |
| #include "media/base/video_encoder.h" |
| #include "media/base/video_util.h" |
| #include "media/video/gpu_video_accelerator_factories.h" |
| #include "media/video/video_encode_accelerator_adapter.h" |
| #include "media/video/video_encoder_fallback.h" |
| #include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h" |
| #include "third_party/blink/public/platform/platform.h" |
| #include "third_party/blink/public/platform/task_type.h" |
| #include "third_party/blink/renderer/bindings/core/v8/native_value_traits_impl.h" |
| #include "third_party/blink/renderer/bindings/core/v8/script_function.h" |
| #include "third_party/blink/renderer/bindings/core/v8/script_promise.h" |
| #include "third_party/blink/renderer/bindings/core/v8/to_v8_traits.h" |
| #include "third_party/blink/renderer/bindings/core/v8/v8_dom_exception.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_avc_encoder_config.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_chunk_metadata.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_svc_output_metadata.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_color_space_init.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_config.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_init.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_support.h" |
| #include "third_party/blink/renderer/bindings/modules/v8/v8_video_pixel_format.h" |
| #include "third_party/blink/renderer/core/dom/dom_exception.h" |
| #include "third_party/blink/renderer/core/streams/readable_stream.h" |
| #include "third_party/blink/renderer/core/streams/writable_stream.h" |
| #include "third_party/blink/renderer/modules/webcodecs/allow_shared_buffer_source_util.h" |
| #include "third_party/blink/renderer/modules/webcodecs/codec_state_helper.h" |
| #include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h" |
| #include "third_party/blink/renderer/modules/webcodecs/gpu_factories_retriever.h" |
| #include "third_party/blink/renderer/modules/webcodecs/video_color_space.h" |
| #include "third_party/blink/renderer/platform/bindings/enumeration_base.h" |
| #include "third_party/blink/renderer/platform/bindings/exception_state.h" |
| #include "third_party/blink/renderer/platform/bindings/script_state.h" |
| #include "third_party/blink/renderer/platform/graphics/gpu/shared_gpu_context.h" |
| #include "third_party/blink/renderer/platform/graphics/web_graphics_context_3d_video_frame_pool.h" |
| #include "third_party/blink/renderer/platform/instrumentation/use_counter.h" |
| #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h" |
| #include "third_party/blink/renderer/platform/scheduler/public/thread.h" |
| #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h" |
| |
| #if BUILDFLAG(ENABLE_LIBAOM) |
| #include "media/video/av1_video_encoder.h" |
| #endif |
| |
| #if BUILDFLAG(ENABLE_OPENH264) |
| #include "media/video/openh264_video_encoder.h" |
| #endif |
| |
| #if BUILDFLAG(ENABLE_LIBVPX) |
| #include "media/video/vpx_video_encoder.h" |
| #endif |
| |
| namespace WTF { |
| |
| template <> |
| struct CrossThreadCopier<media::EncoderStatus> |
| : public CrossThreadCopierPassThrough<media::EncoderStatus> { |
| STATIC_ONLY(CrossThreadCopier); |
| }; |
| |
| } // namespace WTF |
| |
| namespace blink { |
| |
| namespace { |
| |
| constexpr const char kCategory[] = "media"; |
| constexpr int kMaxActiveEncodes = 5; |
| |
| // Use this function in cases when we can't immediately delete |ptr| because |
| // there might be its methods on the call stack. |
| template <typename T> |
| void DeleteLater(ScriptState* state, std::unique_ptr<T> ptr) { |
| DCHECK(state->ContextIsValid()); |
| auto* context = ExecutionContext::From(state); |
| auto runner = context->GetTaskRunner(TaskType::kInternalDefault); |
| runner->DeleteSoon(FROM_HERE, std::move(ptr)); |
| } |
| |
| bool IsAcceleratedConfigurationSupported( |
| media::VideoCodecProfile profile, |
| const media::VideoEncoder::Options& options, |
| media::GpuVideoAcceleratorFactories* gpu_factories) { |
| if (!gpu_factories || !gpu_factories->IsGpuVideoEncodeAcceleratorEnabled()) |
| return false; |
| |
| auto supported_profiles = |
| gpu_factories->GetVideoEncodeAcceleratorSupportedProfiles().value_or( |
| media::VideoEncodeAccelerator::SupportedProfiles()); |
| |
| bool found_supported_profile = false; |
| for (auto& supported_profile : supported_profiles) { |
| if (supported_profile.profile != profile) |
| continue; |
| |
| if (supported_profile.min_resolution.width() > options.frame_size.width() || |
| supported_profile.min_resolution.height() > |
| options.frame_size.height()) { |
| continue; |
| } |
| |
| if (supported_profile.max_resolution.width() < options.frame_size.width() || |
| supported_profile.max_resolution.height() < |
| options.frame_size.height()) { |
| continue; |
| } |
| |
| double max_supported_framerate = |
| static_cast<double>(supported_profile.max_framerate_numerator) / |
| supported_profile.max_framerate_denominator; |
| if (options.framerate.has_value() && |
| options.framerate.value() > max_supported_framerate) { |
| continue; |
| } |
| |
| if (options.scalability_mode.has_value() && |
| !base::Contains(supported_profile.scalability_modes, |
| options.scalability_mode.value())) { |
| continue; |
| } |
| |
| found_supported_profile = true; |
| break; |
| } |
| return found_supported_profile; |
| } |
| |
| VideoEncoderTraits::ParsedConfig* ParseConfigStatic( |
| const VideoEncoderConfig* config, |
| ExceptionState& exception_state) { |
| constexpr int kMaxSupportedFrameSize = 8000; |
| auto* result = MakeGarbageCollected<VideoEncoderTraits::ParsedConfig>(); |
| |
| result->options.frame_size.set_height(config->height()); |
| if (config->height() == 0 || config->height() > kMaxSupportedFrameSize) { |
| exception_state.ThrowTypeError(String::Format( |
| "Invalid height; expected range from %d to %d, received %d.", 1, |
| kMaxSupportedFrameSize, config->height())); |
| return nullptr; |
| } |
| |
| result->options.frame_size.set_width(config->width()); |
| if (config->width() == 0 || config->width() > kMaxSupportedFrameSize) { |
| exception_state.ThrowTypeError(String::Format( |
| "Invalid width; expected range from %d to %d, received %d.", 1, |
| kMaxSupportedFrameSize, config->width())); |
| return nullptr; |
| } |
| |
| if (config->alpha() == "keep") { |
| exception_state.ThrowDOMException( |
| DOMExceptionCode::kNotSupportedError, |
| "Alpha encoding is not currently supported."); |
| return nullptr; |
| } |
| |
| result->options.latency_mode = |
| (config->latencyMode() == "quality") |
| ? media::VideoEncoder::LatencyMode::Quality |
| : media::VideoEncoder::LatencyMode::Realtime; |
| |
| if (config->hasBitrate()) { |
| uint32_t bps = base::saturated_cast<uint32_t>(config->bitrate()); |
| if (bps == 0) { |
| exception_state.ThrowTypeError("Zero is not a valid bitrate."); |
| return nullptr; |
| } |
| if (config->hasBitrateMode() && config->bitrateMode() == "constant") { |
| result->options.bitrate = media::Bitrate::ConstantBitrate(bps); |
| } else { |
| // VBR in media:Bitrate supports both target and peak bitrate. |
| // Currently webcodecs doesn't expose peak bitrate |
| // (assuming unconstrained VBR), here we just set peak as 10 times |
| // target as a good enough way of expressing unconstrained VBR. |
| result->options.bitrate = media::Bitrate::VariableBitrate(bps, 10 * bps); |
| } |
| } |
| |
| if (config->hasDisplayWidth() && config->hasDisplayHeight()) { |
| result->display_size.emplace(config->displayWidth(), |
| config->displayHeight()); |
| } |
| |
| if (config->hasFramerate()) { |
| constexpr double kMinFramerate = .0001; |
| constexpr double kMaxFramerate = 1'000'000'000; |
| if (std::isnan(config->framerate()) || |
| config->framerate() < kMinFramerate || |
| config->framerate() > kMaxFramerate) { |
| exception_state.ThrowTypeError(String::Format( |
| "Invalid framerate; expected range from %f to %f, received %f.", |
| kMinFramerate, kMaxFramerate, config->framerate())); |
| return nullptr; |
| } |
| result->options.framerate = config->framerate(); |
| } else { |
| result->options.framerate = |
| media::VideoEncodeAccelerator::kDefaultFramerate; |
| } |
| |
| // https://w3c.github.io/webrtc-svc/ |
| if (config->hasScalabilityMode()) { |
| if (config->scalabilityMode() == "L1T2") { |
| result->options.scalability_mode = media::SVCScalabilityMode::kL1T2; |
| } else if (config->scalabilityMode() == "L1T3") { |
| result->options.scalability_mode = media::SVCScalabilityMode::kL1T3; |
| } else { |
| exception_state.ThrowTypeError("Unsupported scalabilityMode."); |
| return nullptr; |
| } |
| } |
| |
| // The IDL defines a default value of "no-preference". |
| DCHECK(config->hasHardwareAcceleration()); |
| |
| result->hw_pref = StringToHardwarePreference( |
| IDLEnumAsString(config->hardwareAcceleration())); |
| |
| bool is_codec_ambiguous = true; |
| result->codec = media::VideoCodec::kUnknown; |
| result->profile = media::VIDEO_CODEC_PROFILE_UNKNOWN; |
| result->level = 0; |
| result->codec_string = config->codec(); |
| |
| // Some codec strings provide color space info, but for WebCodecs this is |
| // ignored. Instead, the VideoFrames given to encode() are the source of truth |
| // for input color space. Note also that the output color space is up to the |
| // underlying codec impl. See https://github.com/w3c/webcodecs/issues/345. |
| media::VideoColorSpace codec_string_color_space; |
| |
| bool parse_succeeded = media::ParseVideoCodecString( |
| "", config->codec().Utf8(), &is_codec_ambiguous, &result->codec, |
| &result->profile, &result->level, &codec_string_color_space); |
| |
| if (!parse_succeeded || is_codec_ambiguous) { |
| exception_state.ThrowTypeError("Unknown codec."); |
| return nullptr; |
| } |
| |
| // We are done with the parsing. |
| if (!config->hasAvc()) |
| return result; |
| |
| // We should only get here with H264 codecs. |
| if (result->codec != media::VideoCodec::kH264) { |
| exception_state.ThrowTypeError( |
| "'avc' field can only be used with AVC codecs"); |
| return nullptr; |
| } |
| |
| std::string avc_format = IDLEnumAsString(config->avc()->format()).Utf8(); |
| if (avc_format == "avc") { |
| result->options.avc.produce_annexb = false; |
| } else if (avc_format == "annexb") { |
| result->options.avc.produce_annexb = true; |
| } else { |
| NOTREACHED(); |
| } |
| |
| return result; |
| } |
| |
| const base::Feature kWebCodecsAv1Encoding{"WebCodecsAv1Encoding", |
| base::FEATURE_ENABLED_BY_DEFAULT}; |
| |
| bool VerifyCodecSupportStatic(VideoEncoderTraits::ParsedConfig* config, |
| ExceptionState* exception_state) { |
| switch (config->codec) { |
| case media::VideoCodec::kAV1: |
| if (!base::FeatureList::IsEnabled(kWebCodecsAv1Encoding)) { |
| if (exception_state) { |
| exception_state->ThrowDOMException( |
| DOMExceptionCode::kNotSupportedError, |
| "AV1 encoding is not supported yet."); |
| } |
| return false; |
| } |
| |
| if (config->profile != |
| media::VideoCodecProfile::AV1PROFILE_PROFILE_MAIN) { |
| if (exception_state) { |
| exception_state->ThrowDOMException( |
| DOMExceptionCode::kNotSupportedError, "Unsupported av1 profile."); |
| } |
| return false; |
| } |
| break; |
| |
| case media::VideoCodec::kVP8: |
| break; |
| |
| case media::VideoCodec::kVP9: |
| if (config->profile == media::VideoCodecProfile::VP9PROFILE_PROFILE1 || |
| config->profile == media::VideoCodecProfile::VP9PROFILE_PROFILE3) { |
| if (exception_state) { |
| exception_state->ThrowDOMException( |
| DOMExceptionCode::kNotSupportedError, "Unsupported vp9 profile."); |
| } |
| return false; |
| } |
| break; |
| |
| case media::VideoCodec::kH264: |
| if (config->options.frame_size.width() % 2 != 0 || |
| config->options.frame_size.height() % 2 != 0) { |
| if (exception_state) { |
| exception_state->ThrowDOMException( |
| DOMExceptionCode::kNotSupportedError, |
| "H264 only supports even sized frames."); |
| } |
| return false; |
| } |
| break; |
| |
| default: |
| if (exception_state) { |
| exception_state->ThrowDOMException(DOMExceptionCode::kNotSupportedError, |
| "Unsupported codec type."); |
| } |
| return false; |
| } |
| |
| return true; |
| } |
| |
| VideoEncoderConfig* CopyConfig(const VideoEncoderConfig& config) { |
| auto* result = VideoEncoderConfig::Create(); |
| result->setCodec(config.codec()); |
| result->setWidth(config.width()); |
| result->setHeight(config.height()); |
| |
| if (config.hasDisplayWidth()) |
| result->setDisplayWidth(config.displayWidth()); |
| |
| if (config.hasDisplayHeight()) |
| result->setDisplayHeight(config.displayHeight()); |
| |
| if (config.hasFramerate()) |
| result->setFramerate(config.framerate()); |
| |
| if (config.hasBitrate()) |
| result->setBitrate(config.bitrate()); |
| |
| if (config.hasScalabilityMode()) |
| result->setScalabilityMode(config.scalabilityMode()); |
| |
| if (config.hasHardwareAcceleration()) |
| result->setHardwareAcceleration(config.hardwareAcceleration()); |
| |
| if (config.hasAlpha()) |
| result->setAlpha(config.alpha()); |
| |
| if (config.hasBitrateMode()) |
| result->setBitrateMode(config.bitrateMode()); |
| |
| if (config.hasLatencyMode()) |
| result->setLatencyMode(config.latencyMode()); |
| |
| if (config.hasAvc() && config.avc()->hasFormat()) { |
| auto* avc = AvcEncoderConfig::Create(); |
| avc->setFormat(config.avc()->format()); |
| result->setAvc(avc); |
| } |
| |
| return result; |
| } |
| |
| const base::Feature kWebCodecsEncoderGpuMemoryBufferReadback { |
| "WebCodecsEncoderGpuMemoryBufferReadback", |
| #if BUILDFLAG(IS_MAC) || BUILDFLAG(IS_WIN) || \ |
| (BUILDFLAG(IS_CHROMEOS) && defined(ARCH_CPU_X86_FAMILY)) |
| base::FEATURE_ENABLED_BY_DEFAULT |
| #else |
| base::FEATURE_DISABLED_BY_DEFAULT |
| #endif |
| }; |
| |
| bool CanUseGpuMemoryBufferReadback(media::VideoPixelFormat format, |
| bool force_opaque) { |
| // GMB readback only works with NV12, so only opaque buffers can be used. |
| return (format == media::PIXEL_FORMAT_XBGR || |
| format == media::PIXEL_FORMAT_XRGB || |
| (force_opaque && (format == media::PIXEL_FORMAT_ABGR || |
| format == media::PIXEL_FORMAT_ARGB))) && |
| base::FeatureList::IsEnabled(kWebCodecsEncoderGpuMemoryBufferReadback); |
| } |
| |
| } // namespace |
| |
| // static |
| const char* VideoEncoderTraits::GetName() { |
| return "VideoEncoder"; |
| } |
| |
| // static |
| VideoEncoder* VideoEncoder::Create(ScriptState* script_state, |
| const VideoEncoderInit* init, |
| ExceptionState& exception_state) { |
| auto* result = |
| MakeGarbageCollected<VideoEncoder>(script_state, init, exception_state); |
| return exception_state.HadException() ? nullptr : result; |
| } |
| |
| VideoEncoder::VideoEncoder(ScriptState* script_state, |
| const VideoEncoderInit* init, |
| ExceptionState& exception_state) |
| : Base(script_state, init, exception_state) { |
| UseCounter::Count(ExecutionContext::From(script_state), |
| WebFeature::kWebCodecs); |
| } |
| |
| VideoEncoder::~VideoEncoder() = default; |
| |
| VideoEncoder::ParsedConfig* VideoEncoder::ParseConfig( |
| const VideoEncoderConfig* config, |
| ExceptionState& exception_state) { |
| return ParseConfigStatic(config, exception_state); |
| } |
| |
| bool VideoEncoder::VerifyCodecSupport(ParsedConfig* config, |
| ExceptionState& exception_state) { |
| return VerifyCodecSupportStatic(config, &exception_state); |
| } |
| |
| void VideoEncoder::UpdateEncoderLog(std::string encoder_name, |
| bool is_hw_accelerated) { |
| // TODO(https://crbug.com/1139089) : Add encoder properties. |
| media::MediaLog* log = logger_->log(); |
| |
| log->SetProperty<media::MediaLogProperty::kVideoEncoderName>(encoder_name); |
| log->SetProperty<media::MediaLogProperty::kIsPlatformVideoEncoder>( |
| is_hw_accelerated); |
| } |
| |
| std::unique_ptr<media::VideoEncoder> |
| VideoEncoder::CreateAcceleratedVideoEncoder( |
| media::VideoCodecProfile profile, |
| const media::VideoEncoder::Options& options, |
| media::GpuVideoAcceleratorFactories* gpu_factories) { |
| if (!IsAcceleratedConfigurationSupported(profile, options, gpu_factories)) |
| return nullptr; |
| |
| return std::make_unique< |
| media::AsyncDestroyVideoEncoder<media::VideoEncodeAcceleratorAdapter>>( |
| std::make_unique<media::VideoEncodeAcceleratorAdapter>(gpu_factories, |
| callback_runner_)); |
| } |
| |
| std::unique_ptr<media::VideoEncoder> CreateAv1VideoEncoder() { |
| #if BUILDFLAG(ENABLE_LIBAOM) |
| return std::make_unique<media::Av1VideoEncoder>(); |
| #else |
| return nullptr; |
| #endif // BUILDFLAG(ENABLE_LIBAOM) |
| } |
| |
| std::unique_ptr<media::VideoEncoder> CreateVpxVideoEncoder() { |
| #if BUILDFLAG(ENABLE_LIBVPX) |
| return std::make_unique<media::VpxVideoEncoder>(); |
| #else |
| return nullptr; |
| #endif // BUILDFLAG(ENABLE_LIBVPX) |
| } |
| |
| std::unique_ptr<media::VideoEncoder> CreateOpenH264VideoEncoder() { |
| #if BUILDFLAG(ENABLE_OPENH264) |
| return std::make_unique<media::OpenH264VideoEncoder>(); |
| #else |
| return nullptr; |
| #endif // BUILDFLAG(ENABLE_OPENH264) |
| } |
| |
| // This method is static and takes |self| in order to make it possible to use it |
| // with a weak |this|. It's needed in to avoid a persistent reference cycle. |
| std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateSoftwareVideoEncoder( |
| VideoEncoder* self, |
| media::VideoCodec codec) { |
| if (!self) |
| return nullptr; |
| std::unique_ptr<media::VideoEncoder> result; |
| switch (codec) { |
| case media::VideoCodec::kAV1: |
| result = CreateAv1VideoEncoder(); |
| self->UpdateEncoderLog("Av1VideoEncoder", false); |
| break; |
| case media::VideoCodec::kVP8: |
| case media::VideoCodec::kVP9: |
| result = CreateVpxVideoEncoder(); |
| self->UpdateEncoderLog("VpxVideoEncoder", false); |
| break; |
| case media::VideoCodec::kH264: |
| result = CreateOpenH264VideoEncoder(); |
| self->UpdateEncoderLog("OpenH264VideoEncoder", false); |
| break; |
| default: |
| break; |
| } |
| if (!result) |
| return nullptr; |
| return std::make_unique<media::OffloadingVideoEncoder>(std::move(result)); |
| } |
| |
| std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateMediaVideoEncoder( |
| const ParsedConfig& config, |
| media::GpuVideoAcceleratorFactories* gpu_factories) { |
| switch (config.hw_pref) { |
| case HardwarePreference::kPreferHardware: { |
| auto result = CreateAcceleratedVideoEncoder( |
| config.profile, config.options, gpu_factories); |
| if (result) |
| UpdateEncoderLog("AcceleratedVideoEncoder", true); |
| return result; |
| } |
| case HardwarePreference::kNoPreference: |
| if (auto result = CreateAcceleratedVideoEncoder( |
| config.profile, config.options, gpu_factories)) { |
| UpdateEncoderLog("AcceleratedVideoEncoder", true); |
| return std::make_unique<media::VideoEncoderFallback>( |
| std::move(result), |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| &VideoEncoder::CreateSoftwareVideoEncoder, |
| WrapCrossThreadWeakPersistent(this), config.codec))); |
| } |
| [[fallthrough]]; |
| case HardwarePreference::kPreferSoftware: |
| return CreateSoftwareVideoEncoder(this, config.codec); |
| |
| default: |
| NOTREACHED(); |
| return nullptr; |
| } |
| } |
| |
| void VideoEncoder::ContinueConfigureWithGpuFactories( |
| Request* request, |
| media::GpuVideoAcceleratorFactories* gpu_factories) { |
| DCHECK(active_config_); |
| DCHECK_EQ(request->type, Request::Type::kConfigure); |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| |
| media_encoder_ = CreateMediaVideoEncoder(*active_config_, gpu_factories); |
| if (!media_encoder_) { |
| HandleError(logger_->MakeException( |
| "Encoder creation error.", |
| media::EncoderStatus( |
| media::EncoderStatus::Codes::kEncoderInitializationError, |
| "Unable to create encoder (most likely unsupported " |
| "codec/acceleration requirement combination)"))); |
| request->EndTracing(); |
| return; |
| } |
| |
| auto output_cb = ConvertToBaseRepeatingCallback(CrossThreadBindRepeating( |
| &VideoEncoder::CallOutputCallback, WrapCrossThreadWeakPersistent(this), |
| // We can't use |active_config_| from |this| because it can change by |
| // the time the callback is executed. |
| WrapCrossThreadPersistent(active_config_.Get()), reset_count_)); |
| |
| auto done_callback = [](VideoEncoder* self, Request* req, |
| media::VideoCodec codec, |
| media::EncoderStatus status) { |
| if (!self || self->reset_count_ != req->reset_count) { |
| req->EndTracing(/*aborted=*/true); |
| return; |
| } |
| DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); |
| DCHECK(self->active_config_); |
| |
| if (!status.is_ok()) { |
| self->HandleError(self->logger_->MakeException( |
| "Encoder initialization error.", std::move(status))); |
| } else { |
| UMA_HISTOGRAM_ENUMERATION("Blink.WebCodecs.VideoEncoder.Codec", codec, |
| media::VideoCodec::kMaxValue); |
| } |
| req->EndTracing(); |
| |
| self->blocking_request_in_progress_ = false; |
| self->ProcessRequests(); |
| }; |
| |
| media_encoder_->Initialize( |
| active_config_->profile, active_config_->options, std::move(output_cb), |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| done_callback, WrapCrossThreadWeakPersistent(this), |
| WrapCrossThreadPersistent(request), active_config_->codec))); |
| } |
| |
| bool VideoEncoder::CanReconfigure(ParsedConfig& original_config, |
| ParsedConfig& new_config) { |
| // Reconfigure is intended for things that don't require changing underlying |
| // codec implementation and can be changed on the fly. |
| return original_config.codec == new_config.codec && |
| original_config.profile == new_config.profile && |
| original_config.level == new_config.level && |
| original_config.hw_pref == new_config.hw_pref; |
| } |
| |
| bool VideoEncoder::HasPendingActivity() const { |
| return (active_encodes_ > 0) || Base::HasPendingActivity(); |
| } |
| |
| bool VideoEncoder::ReadyToProcessNextRequest() { |
| if (active_encodes_ >= kMaxActiveEncodes) |
| return false; |
| |
| return Base::ReadyToProcessNextRequest(); |
| } |
| |
| void VideoEncoder::ProcessEncode(Request* request) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| DCHECK_EQ(state_, V8CodecState::Enum::kConfigured); |
| DCHECK(media_encoder_); |
| DCHECK_EQ(request->type, Request::Type::kEncode); |
| DCHECK_GT(requested_encodes_, 0); |
| |
| bool keyframe = request->encodeOpts->hasKeyFrameNonNull() && |
| request->encodeOpts->keyFrameNonNull(); |
| active_encodes_++; |
| request->StartTracingVideoEncode(keyframe); |
| |
| auto done_callback = [](VideoEncoder* self, Request* req, |
| media::EncoderStatus status) { |
| if (!self || self->reset_count_ != req->reset_count) { |
| req->EndTracing(/*aborted=*/true); |
| return; |
| } |
| DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); |
| |
| self->active_encodes_--; |
| if (!status.is_ok()) { |
| self->HandleError( |
| self->logger_->MakeException("Encoding error.", std::move(status))); |
| } |
| req->EndTracing(); |
| self->ProcessRequests(); |
| }; |
| |
| scoped_refptr<media::VideoFrame> frame = request->input->frame(); |
| |
| // Currently underlying encoders can't handle frame backed by textures, |
| // so let's readback pixel data to CPU memory. |
| // TODO(crbug.com/1229845): We shouldn't be reading back frames here. |
| if (frame->HasTextures() && !frame->HasGpuMemoryBuffer()) { |
| // TODO(crbug.com/1195433): Once support for alpha channel encoding is |
| // implemented, |force_opaque| must be set based on the VideoEncoderConfig. |
| const bool can_use_gmb = |
| !disable_accelerated_frame_pool_ && |
| CanUseGpuMemoryBufferReadback(frame->format(), /*force_opaque=*/true); |
| if (can_use_gmb && !accelerated_frame_pool_) { |
| if (auto wrapper = SharedGpuContext::ContextProviderWrapper()) { |
| accelerated_frame_pool_ = |
| std::make_unique<WebGraphicsContext3DVideoFramePool>(wrapper); |
| } |
| } |
| if (can_use_gmb && accelerated_frame_pool_) { |
| // This will execute shortly after CopyRGBATextureToVideoFrame() |
| // completes. |blocking_request_in_progress_| = true will ensure that |
| // HasPendingActivity() keeps the VideoEncoder alive long enough. |
| auto blit_done_callback = |
| [](VideoEncoder* self, bool keyframe, uint32_t reset_count, |
| base::TimeDelta timestamp, media::VideoFrameMetadata metadata, |
| media::VideoEncoder::EncoderStatusCB done_callback, |
| scoped_refptr<media::VideoFrame> frame) { |
| if (!self || self->reset_count_ != reset_count || !frame) |
| return; |
| |
| // CopyRGBATextureToVideoFrame() operates on mailboxes and not |
| // frames, so we must manually copy over properties relevant to the |
| // encoder. |
| frame->set_timestamp(timestamp); |
| frame->set_metadata(metadata); |
| |
| DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); |
| --self->requested_encodes_; |
| self->blocking_request_in_progress_ = false; |
| self->media_encoder_->Encode(std::move(frame), keyframe, |
| std::move(done_callback)); |
| self->ProcessRequests(); |
| }; |
| |
| auto origin = frame->metadata().texture_origin_is_top_left |
| ? kTopLeft_GrSurfaceOrigin |
| : kBottomLeft_GrSurfaceOrigin; |
| |
| // TODO(crbug.com/1224279): This assumes that all frames are 8-bit sRGB. |
| // Expose the color space and pixel format that is backing |
| // `image->GetMailboxHolder()`, or, alternatively, expose an accelerated |
| // SkImage. |
| auto format = (frame->format() == media::PIXEL_FORMAT_XBGR || |
| frame->format() == media::PIXEL_FORMAT_ABGR) |
| ? viz::ResourceFormat::RGBA_8888 |
| : viz::ResourceFormat::BGRA_8888; |
| |
| // Stall request processing while we wait for the copy to complete. It'd |
| // be nice to not have to do this, but currently the request processing |
| // loop must execute synchronously or flush() will miss frames. |
| blocking_request_in_progress_ = true; |
| // When doing RGBA to YUVA conversion using `accelerated_frame_pool_`, use |
| // sRGB primaries and the 601 YUV matrix. Note that this is subtly |
| // different from the 601 gfx::ColorSpace because the 601 gfx::ColorSpace |
| // has different (non-sRGB) primaries. |
| // https://crbug.com/1258245 |
| constexpr gfx::ColorSpace dst_color_space( |
| gfx::ColorSpace::PrimaryID::BT709, |
| gfx::ColorSpace::TransferID::IEC61966_2_1, |
| gfx::ColorSpace::MatrixID::SMPTE170M, |
| gfx::ColorSpace::RangeID::LIMITED); |
| if (accelerated_frame_pool_->CopyRGBATextureToVideoFrame( |
| format, frame->coded_size(), frame->ColorSpace(), origin, |
| frame->mailbox_holder(0), dst_color_space, |
| WTF::Bind(blit_done_callback, WrapWeakPersistent(this), keyframe, |
| reset_count_, frame->timestamp(), frame->metadata(), |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| done_callback, WrapCrossThreadWeakPersistent(this), |
| WrapCrossThreadPersistent(request)))))) { |
| request->input->close(); |
| return; |
| } |
| |
| // Error occurred, fall through to normal readback path below. |
| blocking_request_in_progress_ = false; |
| disable_accelerated_frame_pool_ = true; |
| accelerated_frame_pool_.reset(); |
| } |
| |
| auto wrapper = SharedGpuContext::ContextProviderWrapper(); |
| scoped_refptr<viz::RasterContextProvider> raster_provider; |
| if (wrapper && wrapper->ContextProvider()) |
| raster_provider = wrapper->ContextProvider()->RasterContextProvider(); |
| if (raster_provider) { |
| auto* ri = raster_provider->RasterInterface(); |
| auto* gr_context = raster_provider->GrContext(); |
| |
| frame = ReadbackTextureBackedFrameToMemorySync(*frame, ri, gr_context, |
| &readback_frame_pool_); |
| } else { |
| frame.reset(); |
| } |
| } |
| |
| if (!frame) { |
| callback_runner_->PostTask( |
| FROM_HERE, ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| done_callback, WrapCrossThreadWeakPersistent(this), |
| WrapCrossThreadPersistent(request), |
| media::EncoderStatus( |
| media::EncoderStatus::Codes::kEncoderFailedEncode, |
| "Can't readback frame textures.")))); |
| return; |
| } |
| |
| // Currently underlying encoders can't handle alpha channel, so let's |
| // wrap a frame with an alpha channel into a frame without it. |
| // For example such frames can come from 2D canvas context with alpha = true. |
| if (frame->storage_type() == media::VideoFrame::STORAGE_OWNED_MEMORY && |
| frame->format() == media::PIXEL_FORMAT_I420A) { |
| frame = media::WrapAsI420VideoFrame(std::move(frame)); |
| } |
| |
| --requested_encodes_; |
| media_encoder_->Encode(frame, keyframe, |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| done_callback, WrapCrossThreadWeakPersistent(this), |
| WrapCrossThreadPersistent(request)))); |
| |
| // We passed a copy of frame() above, so this should be safe to close here. |
| request->input->close(); |
| } |
| |
| void VideoEncoder::ProcessConfigure(Request* request) { |
| DCHECK_NE(state_.AsEnum(), V8CodecState::Enum::kClosed); |
| DCHECK_EQ(request->type, Request::Type::kConfigure); |
| DCHECK(active_config_); |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| |
| request->StartTracing(); |
| |
| blocking_request_in_progress_ = true; |
| |
| if (active_config_->hw_pref == HardwarePreference::kPreferSoftware) { |
| ContinueConfigureWithGpuFactories(request, nullptr); |
| return; |
| } |
| |
| RetrieveGpuFactoriesWithKnownEncoderSupport(CrossThreadBindOnce( |
| &VideoEncoder::ContinueConfigureWithGpuFactories, |
| WrapCrossThreadWeakPersistent(this), WrapCrossThreadPersistent(request))); |
| } |
| |
| void VideoEncoder::ProcessReconfigure(Request* request) { |
| DCHECK_EQ(state_.AsEnum(), V8CodecState::Enum::kConfigured); |
| DCHECK_EQ(request->type, Request::Type::kReconfigure); |
| DCHECK(active_config_); |
| DCHECK(media_encoder_); |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| |
| request->StartTracing(); |
| |
| auto reconf_done_callback = [](VideoEncoder* self, Request* req, |
| media::EncoderStatus status) { |
| if (!self || self->reset_count_ != req->reset_count) { |
| req->EndTracing(/*aborted=*/true); |
| return; |
| } |
| DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); |
| DCHECK(self->active_config_); |
| |
| req->EndTracing(); |
| |
| if (status.is_ok()) { |
| self->blocking_request_in_progress_ = false; |
| self->ProcessRequests(); |
| } else { |
| // Reconfiguration failed. Either encoder doesn't support changing options |
| // or it didn't like this particular change. Let's try to configure it |
| // from scratch. |
| req->type = Request::Type::kConfigure; |
| self->ProcessConfigure(req); |
| } |
| }; |
| |
| auto flush_done_callback = [](VideoEncoder* self, Request* req, |
| decltype(reconf_done_callback) reconf_callback, |
| media::EncoderStatus status) { |
| if (!self || self->reset_count_ != req->reset_count) { |
| req->EndTracing(/*aborted=*/true); |
| return; |
| } |
| DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); |
| if (!status.is_ok()) { |
| self->HandleError(self->logger_->MakeException( |
| "Encoder initialization error.", std::move(status))); |
| self->blocking_request_in_progress_ = false; |
| req->EndTracing(); |
| return; |
| } |
| |
| auto output_cb = |
| ConvertToBaseRepeatingCallback(WTF::CrossThreadBindRepeating( |
| &VideoEncoder::CallOutputCallback, |
| WrapCrossThreadWeakPersistent(self), |
| // We can't use |active_config_| from |this| because it can change |
| // by the time the callback is executed. |
| WrapCrossThreadPersistent(self->active_config_.Get()), |
| self->reset_count_)); |
| |
| self->first_output_after_configure_ = true; |
| self->media_encoder_->ChangeOptions( |
| self->active_config_->options, std::move(output_cb), |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| reconf_callback, WrapCrossThreadWeakPersistent(self), |
| WrapCrossThreadPersistent(req)))); |
| }; |
| |
| blocking_request_in_progress_ = true; |
| media_encoder_->Flush(WTF::Bind( |
| flush_done_callback, WrapCrossThreadWeakPersistent(this), |
| WrapCrossThreadPersistent(request), std::move(reconf_done_callback))); |
| } |
| |
| void VideoEncoder::CallOutputCallback( |
| ParsedConfig* active_config, |
| uint32_t reset_count, |
| media::VideoEncoderOutput output, |
| absl::optional<media::VideoEncoder::CodecDescription> codec_desc) { |
| DCHECK(active_config); |
| if (!script_state_->ContextIsValid() || !output_callback_ || |
| state_.AsEnum() != V8CodecState::Enum::kConfigured || |
| reset_count != reset_count_) { |
| return; |
| } |
| |
| MarkCodecActive(); |
| |
| auto buffer = |
| media::DecoderBuffer::FromArray(std::move(output.data), output.size); |
| buffer->set_timestamp(output.timestamp); |
| buffer->set_is_key_frame(output.key_frame); |
| auto* chunk = MakeGarbageCollected<EncodedVideoChunk>(std::move(buffer)); |
| |
| auto* metadata = EncodedVideoChunkMetadata::Create(); |
| if (active_config->options.scalability_mode.has_value()) { |
| auto* svc_metadata = SvcOutputMetadata::Create(); |
| svc_metadata->setTemporalLayerId(output.temporal_id); |
| metadata->setSvc(svc_metadata); |
| |
| // TODO(https://crbug.com/1275024): Remove these lines after deprecating. |
| if (!base::FeatureList::IsEnabled(kRemoveWebCodecsSpecViolations)) |
| metadata->setTemporalLayerId(output.temporal_id); |
| } |
| |
| // TODO(https://crbug.com/1241448): All encoders should output color space. |
| // For now, fallback to 601 since that is correct most often. |
| gfx::ColorSpace output_color_space = output.color_space.IsValid() |
| ? output.color_space |
| : gfx::ColorSpace::CreateREC601(); |
| |
| if (first_output_after_configure_ || codec_desc.has_value() || |
| output_color_space != last_output_color_space_) { |
| first_output_after_configure_ = false; |
| |
| if (output_color_space != last_output_color_space_) { |
| // TODO(crbug.com/1241448): Make Android obey the contract below. For now |
| // Android VEA only _eventually_ gives a key frame when color space changes. |
| #if !defined(OS_ANDROID) |
| DCHECK(output.key_frame) << "Encoders should generate a keyframe when " |
| << "changing color space"; |
| #endif |
| last_output_color_space_ = output_color_space; |
| } |
| |
| auto* decoder_config = VideoDecoderConfig::Create(); |
| decoder_config->setCodec(active_config->codec_string); |
| decoder_config->setCodedHeight(active_config->options.frame_size.height()); |
| decoder_config->setCodedWidth(active_config->options.frame_size.width()); |
| |
| if (active_config->display_size.has_value()) { |
| decoder_config->setDisplayAspectHeight( |
| active_config->display_size.value().height()); |
| decoder_config->setDisplayAspectWidth( |
| active_config->display_size.value().width()); |
| } |
| |
| VideoColorSpace* color_space = |
| MakeGarbageCollected<VideoColorSpace>(output_color_space); |
| decoder_config->setColorSpace(color_space->toJSON()); |
| |
| if (codec_desc.has_value()) { |
| auto* desc_array_buf = DOMArrayBuffer::Create(codec_desc.value().data(), |
| codec_desc.value().size()); |
| decoder_config->setDescription( |
| MakeGarbageCollected<AllowSharedBufferSource>(desc_array_buf)); |
| } |
| metadata->setDecoderConfig(decoder_config); |
| } |
| |
| TRACE_EVENT_BEGIN1(kCategory, GetTraceNames()->output.c_str(), "timestamp", |
| chunk->timestamp()); |
| |
| ScriptState::Scope scope(script_state_); |
| output_callback_->InvokeAndReportException(nullptr, chunk, metadata); |
| |
| TRACE_EVENT_END0(kCategory, GetTraceNames()->output.c_str()); |
| } |
| |
| void VideoEncoder::ResetInternal() { |
| Base::ResetInternal(); |
| active_encodes_ = 0; |
| } |
| |
| static void isConfigSupportedWithSoftwareOnly( |
| ScriptPromiseResolver* resolver, |
| VideoEncoderSupport* support, |
| VideoEncoderTraits::ParsedConfig* config) { |
| std::unique_ptr<media::VideoEncoder> software_encoder; |
| switch (config->codec) { |
| case media::VideoCodec::kAV1: |
| software_encoder = CreateAv1VideoEncoder(); |
| break; |
| case media::VideoCodec::kVP8: |
| case media::VideoCodec::kVP9: |
| software_encoder = CreateVpxVideoEncoder(); |
| break; |
| case media::VideoCodec::kH264: |
| software_encoder = CreateOpenH264VideoEncoder(); |
| break; |
| default: |
| break; |
| } |
| if (!software_encoder) { |
| support->setSupported(false); |
| resolver->Resolve(support); |
| return; |
| } |
| |
| auto done_callback = [](std::unique_ptr<media::VideoEncoder> sw_encoder, |
| ScriptPromiseResolver* resolver, |
| VideoEncoderSupport* support, |
| media::EncoderStatus status) { |
| support->setSupported(status.is_ok()); |
| resolver->Resolve(support); |
| DeleteLater(resolver->GetScriptState(), std::move(sw_encoder)); |
| }; |
| |
| auto* software_encoder_raw = software_encoder.get(); |
| software_encoder_raw->Initialize( |
| config->profile, config->options, base::DoNothing(), |
| ConvertToBaseOnceCallback( |
| CrossThreadBindOnce(done_callback, std::move(software_encoder), |
| WrapCrossThreadPersistent(resolver), |
| WrapCrossThreadPersistent(support)))); |
| } |
| |
| static void isConfigSupportedWithHardwareOnly( |
| ScriptPromiseResolver* resolver, |
| VideoEncoderSupport* support, |
| VideoEncoderTraits::ParsedConfig* config, |
| media::GpuVideoAcceleratorFactories* gpu_factories) { |
| bool supported = IsAcceleratedConfigurationSupported( |
| config->profile, config->options, gpu_factories); |
| support->setSupported(supported); |
| resolver->Resolve(support); |
| } |
| |
| class FindAnySupported final : public NewScriptFunction::Callable { |
| public: |
| ScriptValue Call(ScriptState* state, ScriptValue value) override { |
| ExceptionContext context( |
| ExceptionContext::Context::kConstructorOperationInvoke, |
| "VideoEncoderSupport"); |
| ExceptionState exception_state(state->GetIsolate(), context); |
| HeapVector<Member<VideoEncoderSupport>> supports = |
| NativeValueTraits<IDLSequence<VideoEncoderSupport>>::NativeValue( |
| state->GetIsolate(), value.V8Value(), exception_state); |
| |
| VideoEncoderSupport* result = nullptr; |
| // We don't really expect exceptions here, but if isConfigSupported() is |
| // given a VideoEncoderConfig with uint64 values above max JS int (2^53 - 1) |
| // creation of |supports| vector will fail. This can happen during fuzzing. |
| if (!exception_state.HadException()) { |
| for (auto& support : supports) { |
| result = support; |
| if (result->supported()) |
| break; |
| } |
| } |
| return ScriptValue::From(state, result); |
| } |
| }; |
| |
| // static |
| ScriptPromise VideoEncoder::isConfigSupported(ScriptState* script_state, |
| const VideoEncoderConfig* config, |
| ExceptionState& exception_state) { |
| auto* parsed_config = ParseConfigStatic(config, exception_state); |
| if (!parsed_config) { |
| DCHECK(exception_state.HadException()); |
| return ScriptPromise(); |
| } |
| auto* config_copy = CopyConfig(*config); |
| |
| // Run very basic coarse synchronous validation |
| if (!VerifyCodecSupportStatic(parsed_config, nullptr)) { |
| auto* support = VideoEncoderSupport::Create(); |
| support->setConfig(config_copy); |
| support->setSupported(false); |
| |
| return ScriptPromise::Cast( |
| script_state, |
| ToV8Traits<VideoEncoderSupport>::ToV8(script_state, support) |
| .ToLocalChecked()); |
| } |
| |
| // Create promises for resolving hardware and software encoding support and |
| // put them into |promises|. Simultaneously run both versions of |
| // isConfigSupported(), each version fulfills its own promise. |
| HeapVector<ScriptPromise> promises; |
| if (parsed_config->hw_pref != HardwarePreference::kPreferSoftware) { |
| // Hardware support not denied, detect support by hardware encoders. |
| auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state); |
| promises.push_back(resolver->Promise()); |
| auto* support = VideoEncoderSupport::Create(); |
| support->setConfig(config_copy); |
| auto gpu_retrieved_callback = CrossThreadBindOnce( |
| isConfigSupportedWithHardwareOnly, WrapCrossThreadPersistent(resolver), |
| WrapCrossThreadPersistent(support), |
| WrapCrossThreadPersistent(parsed_config)); |
| RetrieveGpuFactoriesWithKnownEncoderSupport( |
| std::move(gpu_retrieved_callback)); |
| } |
| |
| if (parsed_config->hw_pref != HardwarePreference::kPreferHardware) { |
| // Hardware support not required, detect support by software encoders. |
| auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state); |
| promises.push_back(resolver->Promise()); |
| auto* support = VideoEncoderSupport::Create(); |
| support->setConfig(config_copy); |
| isConfigSupportedWithSoftwareOnly(resolver, support, parsed_config); |
| } |
| |
| // Wait for all |promises| to resolve and check if any of them have |
| // support=true. |
| auto* find_any_supported = MakeGarbageCollected<NewScriptFunction>( |
| script_state, MakeGarbageCollected<FindAnySupported>()); |
| |
| return ScriptPromise::All(script_state, promises).Then(find_any_supported); |
| } |
| |
| } // namespace blink |
