content/renderer/media_recorder/video_track_recorder.h

// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CONTENT_RENDERER_MEDIA_RECORDER_VIDEO_TRACK_RECORDER_H_
#define CONTENT_RENDERER_MEDIA_RECORDER_VIDEO_TRACK_RECORDER_H_

#include <memory>

#include "base/macros.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "content/public/common/buildflags.h"
#include "content/public/renderer/media_stream_video_sink.h"
#include "media/muxers/webm_muxer.h"
#include "third_party/blink/public/platform/web_media_stream_track.h"
#include "third_party/skia/include/core/SkBitmap.h"

namespace base {
class Thread;
}  // namespace base

namespace cc {
class PaintCanvas;
}  // namespace cc

namespace media {
class PaintCanvasVideoRenderer;
class VideoFrame;
}  // namespace media

namespace video_track_recorder {
#if defined(OS_ANDROID)
const int kVEAEncoderMinResolutionWidth = 176;
const int kVEAEncoderMinResolutionHeight = 144;
#else
const int kVEAEncoderMinResolutionWidth = 640;
const int kVEAEncoderMinResolutionHeight = 480;
#endif
}  // namespace video_track_recorder

namespace content {

// VideoTrackRecorder is a MediaStreamVideoSink that encodes the video frames
// received from a Stream Video Track. This class is constructed and used on a
// single thread, namely the main Render thread. This mirrors the other
// MediaStreamVideo* classes that are constructed/configured on Main Render
// thread but that pass frames on Render IO thread. It has an internal Encoder
// with its own threading subtleties, see the implementation file.
class CONTENT_EXPORT VideoTrackRecorder : public MediaStreamVideoSink {
 public:
  // Do not change the order of codecs; add new ones right before LAST.
  enum class CodecId {
    VP8,
    VP9,
#if BUILDFLAG(RTC_USE_H264)
    H264,
#endif
    LAST
  };

  using OnEncodedVideoCB =
      base::Callback<void(const media::WebmMuxer::VideoParameters& params,
                          std::unique_ptr<std::string> encoded_data,
                          std::unique_ptr<std::string> encoded_alpha,
                          base::TimeTicks capture_timestamp,
                          bool is_key_frame)>;
  using OnErrorCB = base::Closure;

  // Wraps a counter in a class in order to enable use of base::WeakPtr<>.
  // See https://crbug.com/859610 for why this was added.
  class Counter {
   public:
    Counter();
    ~Counter();
    uint32_t count() const { return count_; }
    void IncreaseCount();
    void DecreaseCount();
    base::WeakPtr<Counter> GetWeakPtr();

   private:
    uint32_t count_;
    base::WeakPtrFactory<Counter> weak_factory_;
  };

  // Base class to describe a generic Encoder, encapsulating all actual encoder
  // (re)configurations, encoding and delivery of received frames. This class is
  // ref-counted to allow the MediaStreamVideoTrack to hold a reference to it
  // (via the callback that MediaStreamVideoSink passes along) and to jump back
  // and forth to an internal encoder thread. Moreover, this class:
  // - is created on its parent's thread (usually the main Render thread), that
  // is, |main_task_runner_|.
  // - receives VideoFrames on |origin_task_runner_| and runs OnEncodedVideoCB
  // on that thread as well. This task runner is cached on first frame arrival,
  // and is supposed to be the render IO thread (but this is not enforced);
  // - uses an internal |encoding_task_runner_| for actual encoder interactions,
  // namely configuration, encoding (which might take some time) and
  // destruction. This task runner can be passed on the creation. If nothing is
  // passed, a new encoding thread is created and used.
  class Encoder : public base::RefCountedThreadSafe<Encoder> {
   public:
    Encoder(const OnEncodedVideoCB& on_encoded_video_callback,
            int32_t bits_per_second,
            scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
            scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner =
                nullptr);

    // Start encoding |frame|, returning via |on_encoded_video_callback_|. This
    // call will also trigger an encode configuration upon first frame arrival
    // or parameter change, and an EncodeOnEncodingTaskRunner() to actually
    // encode the frame. If the |frame|'s data is not directly available (e.g.
    // it's a texture) then RetrieveFrameOnMainThread() is called, and if even
    // that fails, black frames are sent instead.
    void StartFrameEncode(const scoped_refptr<media::VideoFrame>& frame,
                          base::TimeTicks capture_timestamp);
    void RetrieveFrameOnMainThread(
        const scoped_refptr<media::VideoFrame>& video_frame,
        base::TimeTicks capture_timestamp);

    static void OnFrameEncodeCompleted(
        const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
        const media::WebmMuxer::VideoParameters& params,
        std::unique_ptr<std::string> data,
        std::unique_ptr<std::string> alpha_data,
        base::TimeTicks capture_timestamp,
        bool keyframe);

    void SetPaused(bool paused);
    virtual bool CanEncodeAlphaChannel();

   protected:
    friend class base::RefCountedThreadSafe<Encoder>;
    friend class VideoTrackRecorderTest;

    // This destructor may run on either |main_task_runner|,
    // |encoding_task_runner|, or |origin_task_runner_|. Main ownership lies
    // with VideoTrackRecorder. Shared ownership is handed out to
    // asynchronous tasks running on |encoding_task_runner| for encoding. Shared
    // ownership is also handed out to a MediaStreamVideoTrack which pushes
    // frames on |origin_task_runner_|. Each of these may end up being the last
    // reference.
    virtual ~Encoder();

    virtual void EncodeOnEncodingTaskRunner(
        scoped_refptr<media::VideoFrame> frame,
        base::TimeTicks capture_timestamp) = 0;

    // Called when the frame reference is released after encode.
    void FrameReleased(const scoped_refptr<media::VideoFrame>& frame);

    // Used to shutdown properly on the same thread we were created.
    const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;

    // Task runner where frames to encode and reply callbacks must happen.
    scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;

    // Task runner where encoding interactions happen.
    scoped_refptr<base::SingleThreadTaskRunner> encoding_task_runner_;

    // Optional thread for encoding. Active for the lifetime of VpxEncoder.
    std::unique_ptr<base::Thread> encoding_thread_;

    // While |paused_|, frames are not encoded. Used only from
    // |encoding_thread_|.
    bool paused_;

    // This callback should be exercised on IO thread.
    const OnEncodedVideoCB on_encoded_video_callback_;

    // Target bitrate for video encoding. If 0, a standard bitrate is used.
    const int32_t bits_per_second_;

    // Number of frames that we keep the reference alive for encode.
    // Operated and released exclusively on |origin_task_runner_|.
    std::unique_ptr<Counter> num_frames_in_encode_;

    // Used to retrieve incoming opaque VideoFrames (i.e. VideoFrames backed by
    // textures). Created on-demand on |main_task_runner_|.
    std::unique_ptr<media::PaintCanvasVideoRenderer> video_renderer_;
    SkBitmap bitmap_;
    std::unique_ptr<cc::PaintCanvas> canvas_;

    DISALLOW_COPY_AND_ASSIGN(Encoder);
  };

  static CodecId GetPreferredCodecId();
  static bool CanUseAcceleratedEncoder(CodecId codec,
                                       size_t width,
                                       size_t height);

  VideoTrackRecorder(
      CodecId codec,
      const blink::WebMediaStreamTrack& track,
      const OnEncodedVideoCB& on_encoded_video_cb,
      int32_t bits_per_second,
      scoped_refptr<base::SingleThreadTaskRunner> main_task_runner);
  ~VideoTrackRecorder() override;

  void Pause();
  void Resume();

  void OnVideoFrameForTesting(const scoped_refptr<media::VideoFrame>& frame,
                              base::TimeTicks capture_time);
 private:
  friend class VideoTrackRecorderTest;
  void InitializeEncoder(CodecId codec,
                         const OnEncodedVideoCB& on_encoded_video_callback,
                         int32_t bits_per_second,
                         bool allow_vea_encoder,
                         const scoped_refptr<media::VideoFrame>& frame,
                         base::TimeTicks capture_time);
  void OnError();

  // Used to check that we are destroyed on the same thread we were created.
  THREAD_CHECKER(main_thread_checker_);

  // We need to hold on to the Blink track to remove ourselves on dtor.
  blink::WebMediaStreamTrack track_;

  // Inner class to encode using whichever codec is configured.
  scoped_refptr<Encoder> encoder_;

  base::Callback<void(bool allow_vea_encoder,
                      const scoped_refptr<media::VideoFrame>& frame,
                      base::TimeTicks capture_time)>
      initialize_encoder_callback_;

  bool should_pause_encoder_on_initialization_;

  scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;

  base::WeakPtrFactory<VideoTrackRecorder> weak_ptr_factory_;

  DISALLOW_COPY_AND_ASSIGN(VideoTrackRecorder);
};

}  // namespace content
#endif  // CONTENT_RENDERER_MEDIA_RECORDER_VIDEO_TRACK_RECORDER_H_