blob: afa58f8e11367051c749734c31dffded79a1b6dc [file] [log] [blame]
// Copyright 2013 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include "base/memory/raw_ptr.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/synchronization/condition_variable.h"
#include "base/synchronization/lock.h"
#include "base/task/sequenced_task_runner.h"
#include "base/time/time.h"
#include "base/timer/timer.h"
#include "media/base/decryptor.h"
#include "media/base/demuxer_stream.h"
#include "media/base/frame_rate_estimator.h"
#include "media/base/limits.h"
#include "media/base/media_log.h"
#include "media/base/pipeline_status.h"
#include "media/base/tuneable.h"
#include "media/base/video_decoder.h"
#include "media/base/video_decoder_config.h"
#include "media/base/video_frame.h"
#include "media/base/video_renderer.h"
#include "media/base/video_renderer_sink.h"
#include "media/filters/decoder_stream.h"
#include "media/filters/video_renderer_algorithm.h"
#include "media/renderers/renderer_impl_factory.h"
#include "media/video/gpu_memory_buffer_video_frame_pool.h"
namespace base {
class TickClock;
} // namespace base
namespace media {
// VideoRendererImpl handles reading from a VideoDecoderStream storing the
// results in a queue of decoded frames and executing a callback when a frame is
// ready for rendering.
class MEDIA_EXPORT VideoRendererImpl
: public VideoRenderer,
public VideoRendererSink::RenderCallback {
// |decoders| contains the VideoDecoders to use when initializing.
// Implementors should avoid doing any sort of heavy work in this method and
// instead post a task to a common/worker thread to handle rendering. Slowing
// down the video thread may result in losing synchronization with audio.
// Setting |drop_frames_| to true causes the renderer to drop expired frames.
const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
VideoRendererSink* sink,
const CreateVideoDecodersCB& create_video_decoders_cb,
bool drop_frames,
MediaLog* media_log,
std::unique_ptr<GpuMemoryBufferVideoFramePool> gmb_pool,
MediaPlayerLoggingID media_player_id);
VideoRendererImpl(const VideoRendererImpl&) = delete;
VideoRendererImpl& operator=(const VideoRendererImpl&) = delete;
~VideoRendererImpl() override;
// VideoRenderer implementation.
void Initialize(DemuxerStream* stream,
CdmContext* cdm_context,
RendererClient* client,
const TimeSource::WallClockTimeCB& wall_clock_time_cb,
PipelineStatusCallback init_cb) override;
void Flush(base::OnceClosure callback) override;
void StartPlayingFrom(base::TimeDelta timestamp) override;
void OnTimeProgressing() override;
void OnTimeStopped() override;
void SetLatencyHint(absl::optional<base::TimeDelta> latency_hint) override;
void SetTickClockForTesting(const base::TickClock* tick_clock);
size_t frames_queued_for_testing() const {
return algorithm_->frames_queued();
size_t effective_frames_queued_for_testing() const {
return algorithm_->effective_frames_queued();
int min_buffered_frames_for_testing() const { return min_buffered_frames_; }
int max_buffered_frames_for_testing() const { return max_buffered_frames_; }
// VideoRendererSink::RenderCallback implementation.
scoped_refptr<VideoFrame> Render(base::TimeTicks deadline_min,
base::TimeTicks deadline_max,
RenderingMode rendering_mode) override;
void OnFrameDropped() override;
base::TimeDelta GetPreferredRenderInterval() override;
// Callback for |video_decoder_stream_| initialization.
void OnVideoDecoderStreamInitialized(bool success);
void FinishInitialization(PipelineStatus status);
void FinishFlush();
// Functions to notify certain events to the RendererClient.
void OnPlaybackError(PipelineStatus error);
void OnPlaybackEnded();
void OnStatisticsUpdate(const PipelineStatistics& stats);
void OnBufferingStateChange(BufferingState state);
void OnWaiting(WaitingReason reason);
// Called by the VideoDecoderStream when a config change occurs. Will notify
// RenderClient of the new config.
void OnConfigChange(const VideoDecoderConfig& config);
// Called when the decoder stream and selector have a fallback after failed
// decode.
void OnFallback(PipelineStatus status);
// Callback for |video_decoder_stream_| to deliver decoded video frames and
// report video decoding status.
void FrameReady(VideoDecoderStream::ReadResult result);
// Helper method for enqueueing a frame to |alogorithm_|.
void AddReadyFrame_Locked(scoped_refptr<VideoFrame> frame);
// Helper method that schedules an asynchronous read from the
// |video_decoder_stream_| as long as there isn't a pending read and we have
// capacity.
void AttemptRead_Locked();
// Called when VideoDecoderStream::Reset() completes.
void OnVideoDecoderStreamResetDone();
// Returns true if the renderer has enough data for playback purposes.
// Note that having enough data may be due to reaching end of stream.
bool HaveEnoughData_Locked() const;
void TransitionToHaveEnough_Locked();
void TransitionToHaveNothing();
void TransitionToHaveNothing_Locked();
// Runs |statistics_cb_| with |frames_decoded_| and |frames_dropped_|, resets
// them to 0. If |force_update| is true, sends an update even if no frames
// have been decoded since the last update.
void UpdateStats_Locked(bool force_update = false);
// Notifies |client_| if the current frame rate has changed since it was last
// reported, and tracks what the most recently reported frame rate was.
void ReportFrameRateIfNeeded_Locked();
// Update |min_buffered_frames_| and |max_buffered_frames_| using the latest
// |average_frame_duration|. Should only be called when |latency_hint_| > 0.
void UpdateLatencyHintBufferingCaps_Locked(
base::TimeDelta average_frame_duration);
// Returns true if algorithm_->effective_frames_queued() >= |buffering_cap|,
// or when the number of ineffective frames >= kAbsoluteMaxFrames.
bool HaveReachedBufferingCap(size_t buffering_cap) const;
// Starts or stops |sink_| respectively. Do not call while |lock_| is held.
void StartSink();
void StopSink();
// Fires |ended_cb_| if there are no remaining usable frames and
// |received_end_of_stream_| is true. Sets |rendered_end_of_stream_| if it
// does so.
// When called from the media thread, |time_progressing| should reflect the
// value of |time_progressing_|. When called from Render() on the sink
// callback thread, |time_progressing| must be true since Render() could not
// have been called otherwise.
void MaybeFireEndedCallback_Locked(bool time_progressing);
// Helper method for converting a single media timestamp to wall clock time.
base::TimeTicks ConvertMediaTimestamp(base::TimeDelta media_timestamp);
base::TimeTicks GetCurrentMediaTimeAsWallClockTime();
// Helper method for checking if a frame timestamp plus the frame's expected
// duration is before |start_timestamp_|.
bool IsBeforeStartTime(const VideoFrame& frame);
// Attempts to remove frames which are no longer effective for rendering when
// |buffering_state_| == BUFFERING_HAVE_NOTHING or |was_background_rendering_|
// is true. If the current media time as provided by |wall_clock_time_cb_| is
// null, no frame expiration will be done.
// When background rendering the method will expire all frames before the
// current wall clock time since it's expected that there will be long delays
// between each Render() call in this state.
// When in the underflow state the method will first attempt to remove expired
// frames before the current media time plus duration. If |sink_started_| is
// true, nothing more can be done. However, if false, and there are still no
// effective frames in the queue, the entire frame queue will be released to
// avoid any stalling.
void RemoveFramesForUnderflowOrBackgroundRendering();
// Notifies |client_| in the event of frame size or opacity changes. Must be
// called on |task_runner_|.
void CheckForMetadataChanges(VideoPixelFormat pixel_format,
const gfx::Size& natural_size);
// Both calls AttemptRead_Locked() and CheckForMetadataChanges(). Must be
// called on |task_runner_|.
void AttemptReadAndCheckForMetadataChanges(VideoPixelFormat pixel_format,
const gfx::Size& natural_size);
scoped_refptr<base::SequencedTaskRunner> task_runner_;
// Sink which calls into VideoRendererImpl via Render() for video frames. Do
// not call any methods on the sink while |lock_| is held or the two threads
// might deadlock. Do not call Start() or Stop() on the sink directly, use
// StartSink() and StopSink() to ensure background rendering is started. Only
// access these values on |task_runner_|.
const raw_ptr<VideoRendererSink> sink_;
bool sink_started_;
// Stores the last decoder config that was passed to
// RendererClient::OnVideoConfigChange. Used to prevent signaling config
// to the upper layers when when the new config is the same.
VideoDecoderConfig current_decoder_config_;
// Used for accessing data members.
base::Lock lock_;
raw_ptr<RendererClient> client_;
// Pool of GpuMemoryBuffers and resources used to create hardware frames.
// Ensure this is destructed after |algorithm_| for optimal memory release
// when a frames are still held by the compositor. Must be destructed after
// |video_decoder_stream_| since it holds a callback to the pool.
std::unique_ptr<GpuMemoryBufferVideoFramePool> gpu_memory_buffer_pool_;
// Provides video frames to VideoRendererImpl.
std::unique_ptr<VideoDecoderStream> video_decoder_stream_;
// Passed in during Initialize().
raw_ptr<DemuxerStream> demuxer_stream_;
raw_ptr<MediaLog> media_log_;
MediaPlayerLoggingID player_id_;
// Flag indicating low-delay mode.
bool low_delay_;
// Keeps track of whether we received the end of stream buffer and finished
// rendering.
bool received_end_of_stream_;
bool rendered_end_of_stream_;
// Important detail: being in kPlaying doesn't imply that video is being
// rendered. Rather, it means that the renderer is ready to go. The actual
// rendering of video is controlled by time advancing via |get_time_cb_|.
// Video renderer can be reinitialized completely by calling Initialize again
// when it is in a kFlushed state with video sink stopped.
// kUninitialized
// +------> | Initialize()
// | |
// | V
// | kInitializing
// | | Decoders initialized
// | |
// | V Decoders reset
// ---- kFlushed <------------------ kFlushing
// | StartPlayingFrom() ^
// | |
// | | Flush()
// `---------> kPlaying --------'
enum State { kUninitialized, kInitializing, kFlushing, kFlushed, kPlaying };
State state_;
// TODO(servolk): Consider using DecoderFactory here instead of the
// CreateVideoDecodersCB.
CreateVideoDecodersCB create_video_decoders_cb_;
// Keep track of the outstanding read on the VideoDecoderStream. Flushing can
// only complete once the read has completed.
bool pending_read_;
bool drop_frames_;
BufferingState buffering_state_;
// Playback operation callbacks.
PipelineStatusCallback init_cb_;
base::OnceClosure flush_cb_;
TimeSource::WallClockTimeCB wall_clock_time_cb_;
base::TimeDelta start_timestamp_;
// Keeps track of the number of frames decoded and dropped since the
// last call to |statistics_cb_|. These must be accessed under lock.
PipelineStatistics stats_;
raw_ptr<const base::TickClock> tick_clock_;
// Algorithm for selecting which frame to render; manages frames and all
// timing related information. Ensure this is destructed before
// |gpu_memory_buffer_pool_| for optimal memory release when a frames are
// still held by the compositor.
std::unique_ptr<VideoRendererAlgorithm> algorithm_;
// Indicates that Render() was called with |background_rendering| set to true,
// so we've entered a background rendering mode where dropped frames are not
// counted. Must be accessed under |lock_| once |sink_| is started.
bool was_background_rendering_;
// Indicates whether or not media time is currently progressing or not. Must
// only be accessed from |task_runner_|.
bool time_progressing_;
// Indicates if a frame has been processed by CheckForMetadataChanges().
bool have_renderered_frames_;
// Tracks last frame properties to detect and notify client of any changes.
gfx::Size last_frame_natural_size_;
bool last_frame_opaque_;
// The last value from |video_decoder_stream_->AverageDuration()|.
base::TimeDelta last_decoder_stream_avg_duration_;
// Indicates if we've painted the first valid frame after StartPlayingFrom().
bool painted_first_frame_;
// The initial value for |min_buffered_frames_| and |max_buffered_frames_|.
Tuneable<size_t> initial_buffering_size_ = {
"MediaInitialBufferingSizeForHaveEnough", 3, limits::kMaxVideoFrames, 10};
// The number of frames required to transition from BUFFERING_HAVE_NOTHING to
size_t min_buffered_frames_;
// The maximum number of frames to buffer. Always >= |min_buffered_frames_|.
// May be greater-than when |latency_hint_| set that decreases the minimum
// buffering limit.
size_t max_buffered_frames_;
// Last Render() and last FrameReady() times respectively. Used to avoid
// triggering underflow when background rendering.
base::TimeTicks last_render_time_;
base::TimeTicks last_frame_ready_time_;
// Running average of frame durations.
FrameRateEstimator fps_estimator_;
// Last FPS, if any, reported to the client.
absl::optional<int> last_reported_fps_;
// Value saved from last call to SetLatencyHint(). Used to recompute buffering
// limits as framerate fluctuates.
absl::optional<base::TimeDelta> latency_hint_;
// When latency_hint_ > 0, we make regular adjustments to buffering caps as
// |algorithm_->average_frame_duration()| fluctuates, but we only want to emit
// one MEDIA_LOG.
bool is_latency_hint_media_logged_ = false;
// NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<VideoRendererImpl> weak_factory_{this};
// Weak factory used to invalidate certain queued callbacks on reset().
// This is useful when doing video frame copies asynchronously since we
// want to discard video frames that might be received after the stream has
// been reset.
base::WeakPtrFactory<VideoRendererImpl> cancel_on_flush_weak_factory_{this};
} // namespace media