blob: 63244ef87a45e1ce4fc890481df613d0b74adf23 [file] [log] [blame]
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <memory>
#include "base/macros.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "content/public/common/buildflags.h"
#include "content/public/renderer/media_stream_video_sink.h"
#include "media/muxers/webm_muxer.h"
#include "third_party/blink/public/platform/web_media_stream_track.h"
#include "third_party/skia/include/core/SkBitmap.h"
namespace base {
class Thread;
} // namespace base
namespace cc {
class PaintCanvas;
} // namespace cc
namespace media {
class PaintCanvasVideoRenderer;
class VideoFrame;
} // namespace media
namespace video_track_recorder {
#if defined(OS_ANDROID)
const int kVEAEncoderMinResolutionWidth = 176;
const int kVEAEncoderMinResolutionHeight = 144;
const int kVEAEncoderMinResolutionWidth = 640;
const int kVEAEncoderMinResolutionHeight = 480;
} // namespace video_track_recorder
namespace content {
// VideoTrackRecorder is a MediaStreamVideoSink that encodes the video frames
// received from a Stream Video Track. This class is constructed and used on a
// single thread, namely the main Render thread. This mirrors the other
// MediaStreamVideo* classes that are constructed/configured on Main Render
// thread but that pass frames on Render IO thread. It has an internal Encoder
// with its own threading subtleties, see the implementation file.
class CONTENT_EXPORT VideoTrackRecorder : public MediaStreamVideoSink {
// Do not change the order of codecs; add new ones right before LAST.
enum class CodecId {
using OnEncodedVideoCB =
base::Callback<void(const media::WebmMuxer::VideoParameters& params,
std::unique_ptr<std::string> encoded_data,
std::unique_ptr<std::string> encoded_alpha,
base::TimeTicks capture_timestamp,
bool is_key_frame)>;
using OnErrorCB = base::Closure;
// Wraps a counter in a class in order to enable use of base::WeakPtr<>.
// See for why this was added.
class Counter {
uint32_t count() const { return count_; }
void IncreaseCount();
void DecreaseCount();
base::WeakPtr<Counter> GetWeakPtr();
uint32_t count_;
base::WeakPtrFactory<Counter> weak_factory_;
// Base class to describe a generic Encoder, encapsulating all actual encoder
// (re)configurations, encoding and delivery of received frames. This class is
// ref-counted to allow the MediaStreamVideoTrack to hold a reference to it
// (via the callback that MediaStreamVideoSink passes along) and to jump back
// and forth to an internal encoder thread. Moreover, this class:
// - is created on its parent's thread (usually the main Render thread), that
// is, |main_task_runner_|.
// - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB
// on that thread as well. This task runner is cached on first frame arrival,
// and is supposed to be the render IO thread (but this is not enforced);
// - uses an internal |encoding_task_runner_| for actual encoder interactions,
// namely configuration, encoding (which might take some time) and
// destruction. This task runner can be passed on the creation. If nothing is
// passed, a new encoding thread is created and used.
class Encoder : public base::RefCountedThreadSafe<Encoder> {
Encoder(const OnEncodedVideoCB& on_encoded_video_callback,
int32_t bits_per_second,
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner =
// Start encoding |frame|, returning via |on_encoded_video_callback_|. This
// call will also trigger an encode configuration upon first frame arrival
// or parameter change, and an EncodeOnEncodingTaskRunner() to actually
// encode the frame. If the |frame|'s data is not directly available (e.g.
// it's a texture) then RetrieveFrameOnMainThread() is called, and if even
// that fails, black frames are sent instead.
void StartFrameEncode(const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks capture_timestamp);
void RetrieveFrameOnMainThread(
const scoped_refptr<media::VideoFrame>& video_frame,
base::TimeTicks capture_timestamp);
static void OnFrameEncodeCompleted(
const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
const media::WebmMuxer::VideoParameters& params,
std::unique_ptr<std::string> data,
std::unique_ptr<std::string> alpha_data,
base::TimeTicks capture_timestamp,
bool keyframe);
void SetPaused(bool paused);
virtual bool CanEncodeAlphaChannel();
friend class base::RefCountedThreadSafe<Encoder>;
friend class VideoTrackRecorderTest;
// This destructor may run on either |main_task_runner|,
// |encoding_task_runner|, or |origin_task_runner_|. Main ownership lies
// with VideoTrackRecorder. Shared ownership is handed out to
// asynchronous tasks running on |encoding_task_runner| for encoding. Shared
// ownership is also handed out to a MediaStreamVideoTrack which pushes
// frames on |origin_task_runner_|. Each of these may end up being the last
// reference.
virtual ~Encoder();
virtual void EncodeOnEncodingTaskRunner(
scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_timestamp) = 0;
// Called when the frame reference is released after encode.
void FrameReleased(const scoped_refptr<media::VideoFrame>& frame);
// Used to shutdown properly on the same thread we were created.
const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
// Task runner where frames to encode and reply callbacks must happen.
scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
// Task runner where encoding interactions happen.
scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner_;
// Optional thread for encoding. Active for the lifetime of VpxEncoder.
std::unique_ptr<base::Thread> encoding_thread_;
// While |paused_|, frames are not encoded. Used only from
// |encoding_thread_|.
bool paused_;
// This callback should be exercised on IO thread.
const OnEncodedVideoCB on_encoded_video_callback_;
// Target bitrate for video encoding. If 0, a standard bitrate is used.
const int32_t bits_per_second_;
// Number of frames that we keep the reference alive for encode.
// Operated and released exclusively on |origin_task_runner_|.
std::unique_ptr<Counter> num_frames_in_encode_;
// Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by
// textures). Created on-demand on |main_task_runner_|.
std::unique_ptr<media::PaintCanvasVideoRenderer> video_renderer_;
SkBitmap bitmap_;
std::unique_ptr<cc::PaintCanvas> canvas_;
static CodecId GetPreferredCodecId();
static bool CanUseAcceleratedEncoder(CodecId codec,
size_t width,
size_t height);
CodecId codec,
const blink::WebMediaStreamTrack& track,
const OnEncodedVideoCB& on_encoded_video_cb,
int32_t bits_per_second,
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner);
~VideoTrackRecorder() override;
void Pause();
void Resume();
void OnVideoFrameForTesting(const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks capture_time);
friend class VideoTrackRecorderTest;
void InitializeEncoder(CodecId codec,
const OnEncodedVideoCB& on_encoded_video_callback,
int32_t bits_per_second,
bool allow_vea_encoder,
const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks capture_time);
void OnError();
// Used to check that we are destroyed on the same thread we were created.
// We need to hold on to the Blink track to remove ourselves on dtor.
blink::WebMediaStreamTrack track_;
// Inner class to encode using whichever codec is configured.
scoped_refptr<Encoder> encoder_;
base::Callback<void(bool allow_vea_encoder,
const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks capture_time)>
bool should_pause_encoder_on_initialization_;
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
base::WeakPtrFactory<VideoTrackRecorder> weak_ptr_factory_;
} // namespace content