blob: a55085d76227cf922990213cee55d4680503f49a [file] [log] [blame]
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <atomic>
#include <memory>
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "base/time/time.h"
#include "build/build_config.h"
#include "media/muxers/webm_muxer.h"
#include "media/video/video_encode_accelerator.h"
#include "third_party/blink/public/common/media/video_capture.h"
#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_sink.h"
#include "third_party/blink/public/platform/web_media_stream_track.h"
#include "third_party/blink/renderer/modules/mediarecorder/buildflags.h"
#include "third_party/blink/renderer/modules/modules_export.h"
#include "third_party/blink/renderer/platform/wtf/cross_thread_copier.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/blink/renderer/platform/wtf/hash_map.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
#include "third_party/skia/include/core/SkBitmap.h"
namespace base {
class Thread;
} // namespace base
namespace cc {
class PaintCanvas;
} // namespace cc
namespace media {
class PaintCanvasVideoRenderer;
class VideoFrame;
} // namespace media
namespace video_track_recorder {
#if defined(OS_ANDROID)
const int kVEAEncoderMinResolutionWidth = 176;
const int kVEAEncoderMinResolutionHeight = 144;
const int kVEAEncoderMinResolutionWidth = 640;
const int kVEAEncoderMinResolutionHeight = 480;
} // namespace video_track_recorder
namespace WTF {
template <>
struct CrossThreadCopier<media::WebmMuxer::VideoParameters> {
using Type = media::WebmMuxer::VideoParameters;
static Type Copy(Type pointer) { return pointer; }
} // namespace WTF
namespace blink {
class MediaStreamVideoTrack;
class Thread;
// VideoTrackRecorder is a MediaStreamVideoSink that encodes the video frames
// received from a Stream Video Track. This class is constructed and used on a
// single thread, namely the main Render thread. This mirrors the other
// MediaStreamVideo* classes that are constructed/configured on Main Render
// thread but that pass frames on Render IO thread. It has an internal Encoder
// with its own threading subtleties, see the implementation file.
class MODULES_EXPORT VideoTrackRecorder
: public GarbageCollected<VideoTrackRecorder>,
public WebMediaStreamSink {
USING_PRE_FINALIZER(VideoTrackRecorder, Prefinalize);
// Do not change the order of codecs; add new ones right before LAST.
enum class CodecId {
using OnEncodedVideoCB = base::RepeatingCallback<void(
const media::WebmMuxer::VideoParameters& params,
std::string encoded_data,
std::string encoded_alpha,
base::TimeTicks capture_timestamp,
bool is_key_frame)>;
using OnErrorCB = base::RepeatingClosure;
// Wraps a counter in a class in order to enable use of base::WeakPtr<>.
// See for why this was added.
class Counter {
uint32_t count() const { return count_; }
void IncreaseCount();
void DecreaseCount();
base::WeakPtr<Counter> GetWeakPtr();
uint32_t count_;
base::WeakPtrFactory<Counter> weak_factory_{this};
// Base class to describe a generic Encoder, encapsulating all actual encoder
// (re)configurations, encoding and delivery of received frames. This class is
// ref-counted to allow the MediaStreamVideoTrack to hold a reference to it
// (via the callback that MediaStreamVideoSink passes along) and to jump back
// and forth to an internal encoder thread. Moreover, this class:
// - is created on its parent's thread (usually the main Render thread), that
// is, |main_task_runner_|.
// - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB
// on that thread as well. This task runner is cached on first frame arrival,
// and is supposed to be the render IO thread (but this is not enforced);
// - uses an internal |encoding_task_runner_| for actual encoder interactions,
// namely configuration, encoding (which might take some time) and
// destruction. This task runner can be passed on the creation. If nothing is
// passed, a new encoding thread is created and used.
class Encoder : public WTF::ThreadSafeRefCounted<Encoder> {
const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
int32_t bits_per_second,
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner =
// Start encoding |frame|, returning via |on_encoded_video_callback_|. This
// call will also trigger an encode configuration upon first frame arrival
// or parameter change, and an EncodeOnEncodingTaskRunner() to actually
// encode the frame. If the |frame|'s data is not directly available (e.g.
// it's a texture) then RetrieveFrameOnMainThread() is called, and if even
// that fails, black frames are sent instead.
void StartFrameEncode(scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_timestamp);
void RetrieveFrameOnMainThread(scoped_refptr<media::VideoFrame> video_frame,
base::TimeTicks capture_timestamp);
using OnEncodedVideoInternalCB = WTF::CrossThreadFunction<void(
const media::WebmMuxer::VideoParameters& params,
std::string encoded_data,
std::string encoded_alpha,
base::TimeTicks capture_timestamp,
bool is_key_frame)>;
static void OnFrameEncodeCompleted(
const OnEncodedVideoInternalCB& on_encoded_video_cb,
const media::WebmMuxer::VideoParameters& params,
std::string data,
std::string alpha_data,
base::TimeTicks capture_timestamp,
bool keyframe);
void SetPaused(bool paused);
virtual bool CanEncodeAlphaChannel();
friend class WTF::ThreadSafeRefCounted<Encoder>;
friend class VideoTrackRecorderTest;
// This destructor may run on either |main_task_runner|,
// |encoding_task_runner|, or |origin_task_runner_|. Main ownership lies
// with VideoTrackRecorder. Shared ownership is handed out to
// asynchronous tasks running on |encoding_task_runner| for encoding. Shared
// ownership is also handed out to a MediaStreamVideoTrack which pushes
// frames on |origin_task_runner_|. Each of these may end up being the last
// reference.
virtual ~Encoder();
virtual void EncodeOnEncodingTaskRunner(
scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_timestamp) = 0;
// Called when the frame reference is released after encode.
void FrameReleased(scoped_refptr<media::VideoFrame> frame);
// A helper function to convert the given |frame| to an I420 video frame.
// Used mainly by the software encoders since I420 is the only supported
// pixel format. The function is best-effort. If for any reason the
// conversion fails, the original |frame| will be returned.
static scoped_refptr<media::VideoFrame> ConvertToI420ForSoftwareEncoder(
scoped_refptr<media::VideoFrame> frame);
// Used to shutdown properly on the same thread we were created.
const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
// Task runner where frames to encode and reply callbacks must happen.
scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
// Task runner where encoding interactions happen.
scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner_;
// Optional thread for encoding. Active for the lifetime of VpxEncoder.
std::unique_ptr<Thread> encoding_thread_;
// While |paused_|, frames are not encoded. Used only from
// |encoding_thread_|.
// Use an atomic variable since it can be set on the main thread and read
// on the io thread at the same time.
std::atomic_bool paused_;
// This callback should be exercised on IO thread.
const OnEncodedVideoCB on_encoded_video_callback_;
// Target bitrate for video encoding. If 0, a standard bitrate is used.
const int32_t bits_per_second_;
// Number of frames that we keep the reference alive for encode.
// Operated and released exclusively on |origin_task_runner_|.
std::unique_ptr<Counter> num_frames_in_encode_;
// Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by
// textures). Created on-demand on |main_task_runner_|.
std::unique_ptr<media::PaintCanvasVideoRenderer> video_renderer_;
SkBitmap bitmap_;
std::unique_ptr<cc::PaintCanvas> canvas_;
// Class to encapsulate the enumeration of CodecIds/VideoCodecProfiles
// supported by the VEA underlying platform. Provides methods to query the
// preferred CodecId and to check if a given CodecId is supported.
class MODULES_EXPORT CodecEnumerator {
CodecEnumerator(const media::VideoEncodeAccelerator::SupportedProfiles&
// Returns the first CodecId that has an associated VEA VideoCodecProfile,
// or VP8 if none available.
CodecId GetPreferredCodecId() const;
// Returns VEA's first supported VideoCodedProfile for a given CodecId, or
media::VideoCodecProfile GetFirstSupportedVideoCodecProfile(
CodecId codec) const;
// Returns a list of supported media::VEA::SupportedProfile for a given
// CodecId, or empty vector if CodecId is unsupported.
media::VideoEncodeAccelerator::SupportedProfiles GetSupportedProfiles(
CodecId codec) const;
// VEA-supported profiles grouped by CodecId.
HashMap<CodecId, media::VideoEncodeAccelerator::SupportedProfiles>
CodecId preferred_codec_id_ = CodecId::LAST;
static CodecId GetPreferredCodecId();
// Returns true if the device has a hardware accelerated encoder which can
// encode video of the given |width|x|height| and |framerate| to specific
// |codec|.
// Note: default framerate value means no restriction.
static bool CanUseAcceleratedEncoder(CodecId codec,
size_t width,
size_t height,
double framerate = 0.0);
CodecId codec,
MediaStreamComponent* track,
const OnEncodedVideoCB& on_encoded_video_cb,
int32_t bits_per_second,
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner);
~VideoTrackRecorder() override;
void Pause();
void Resume();
void OnVideoFrameForTesting(scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_time);
void Trace(blink::Visitor*);
friend class VideoTrackRecorderTest;
void InitializeEncoder(CodecId codec,
const OnEncodedVideoCB& on_encoded_video_callback,
int32_t bits_per_second,
bool allow_vea_encoder,
scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_time);
void OnError();
void ConnectToTrack(const VideoCaptureDeliverFrameCB& callback);
void DisconnectFromTrack();
void Prefinalize();
// Used to check that we are destroyed on the same thread we were created.
// We need to hold on to the Blink track to remove ourselves on dtor.
Member<MediaStreamComponent> track_;
// Inner class to encode using whichever codec is configured.
scoped_refptr<Encoder> encoder_;
base::RepeatingCallback<void(bool allow_vea_encoder,
scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_time)>
bool should_pause_encoder_on_initialization_;
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
} // namespace blink