blob: 70b613669a13ffadf287529ab45cc4255f55b2cb [file] [log] [blame]
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <limits>
#include <memory>
#include "base/macros.h"
#include "base/synchronization/lock.h"
#include "base/time/time.h"
#include "media/base/audio_parameters.h"
#include "media/base/channel_mixer.h"
#include "media/base/multi_channel_resampler.h"
#include "services/audio/delay_buffer.h"
#include "services/audio/loopback_group_member.h"
#include "third_party/abseil-cpp/absl/types/optional.h"
namespace media {
class AudioBus;
} // namespace media
namespace audio {
// Thread-safe implementation of Snooper that records the audio from a
// GroupMember on one thread, and re-renders it to the desired output format on
// another thread. Since the data flow rates are known to be driven by different
// clocks (audio hardware clock versus system clock), the base::TimeTicks
// reference clock is used to detect drift and automatically correct for it to
// maintain proper synchronization.
// Throughout this class, there are sample counters (in terms of the input
// audio's sample rate) that are tracked/computed. They refer to the media
// timestamp of the audio flowing through specific parts of the processing
// pipeline: inbound from OnData() calls → through the delay buffer → through
// the resampler → and outbound via Render() calls:
// write position: The position of audio about to be written into the delay
// buffer. This is managed by OnData().
// read position: The position of audio about to be read from the delay
// buffer and pushed into the resampler. This is managed by
// ReadFromDelayBuffer().
// output position: The position of the audio about to come out of the
// resampler. This is computed within Render(). Note that
// this is a "virtual" position since it is in terms of the
// input audio's sample count, but refers to audio about to
// be generated in the output format (with a possibly
// different sample rate).
// Note that the media timestamps represented by the "positions," as well as the
// surrounding math operations, might seem backwards; but they are not. This is
// because the inbound audio is from a source that pre-renders audio for playout
// in the near future, while the outbound audio is audio that would have been
// played-out in the recent past.
class SnooperNode final : public LoopbackGroupMember::Snooper {
// Use sample counts as a precise measure of audio signal position and time
// duration.
using FrameTicks = int64_t;
// Contruct a SnooperNode that buffers input of one format and renders output
// in [possibly] another format.
SnooperNode(const media::AudioParameters& input_params,
const media::AudioParameters& output_params);
~SnooperNode() final;
// GroupMember::Snooper implementation. Inserts more data into the delay
// buffer.
void OnData(const media::AudioBus& input_bus,
base::TimeTicks reference_time,
double volume) final;
// Given the timing of recent OnData() calls and the |duration| of output that
// would be requested in a call to Render(), determine the latest possible
// |reference_time| for a Render() call that won't result in an underrun.
// Returns absl::nullopt while current conditions prohibit making a reliable
// suggestion.
absl::optional<base::TimeTicks> SuggestLatestRenderTime(FrameTicks duration);
// Renders more audio that was recorded from the GroupMember until
// |output_bus| is filled, resampling and remixing the channels if necessary.
// |reference_time| is used for detecting skip-ahead (i.e., a significant
// forward jump in the reference time) and also to maintain synchronization
// with the input.
void Render(base::TimeTicks reference_time, media::AudioBus* output_bus);
// Helper to store the new |correction_fps|, recompute the resampling I/O
// ratio, and reconfigure the resampler with the new ratio.
void UpdateCorrectionRate(int correction_fps);
// Called by the MultiChannelResampler to acquire more data from the delay
// buffer. This is invoked in the same call stack (and thread) as Render(),
// zero or more times as data is needed by the resampler.
void ReadFromDelayBuffer(int ignored, media::AudioBus* resampler_bus);
// Input and output audio parameters.
const media::AudioParameters input_params_;
const media::AudioParameters output_params_;
// Input and output AudioBus time durations, pre-computed from the input and
// output AudioParameters.
const base::TimeDelta input_bus_duration_;
const base::TimeDelta output_bus_duration_;
// The ratio between the input sampling rate and the output sampling rate. It
// is "perfect" because it assumes no clock skew. Corrections are applied to
// this to determine the actual resampler I/O ratio.
const double perfect_io_ratio_;
// Protects concurrent access to |buffer_| and the |write_position_| and
// |write_reference_time_|. All other members are either read-only, or are not
// accessed by multiple threads.
base::Lock lock_;
// Allows input data to be recorded and then read-back from any position
// later (by the resampler).
DelayBuffer buffer_; // Guarded by |lock_|.
// The next frame position at which to write into the delay buffer, and the
// TimeTicks representing its corresponding system clock timestamp.
FrameTicks write_position_; // Guarded by |lock_|.
base::TimeTicks write_reference_time_; // Guarded by |lock_|.
// Used by SuggestLatestRenderTime() to track whether OnData() has been called
// recently, and as a basis for its suggestion. Other methods should not
// depend on this value for anything.
base::TimeTicks checkpoint_time_;
// The next frame position from which to read from the delay buffer. This is
// the position of the frames about to be pushed into the resampler, not the
// position of frames about to be Render()'ed.
FrameTicks read_position_;
// The expected |reference_time| to be provided in the next call to Render().
// This is used to detect skip-ahead in the output, and compensate when
// necessary.
base::TimeTicks render_reference_time_;
// The additional number of frames currently being consumed by the resampler
// each second to correct for drift.
int correction_fps_;
// Resamples input audio that is read from the delay buffer. Even if the input
// and output have the same sampling rate, this is used to subtly stretch the
// audio signal to correct for drift.
media::MultiChannelResampler resampler_;
// Specifies whether channel mixing should occur before or after resampling,
// or is not needed. The strategy is chosen such that the minimal number of
// channels are resampled, as resampling is the more-expensive operation.
enum { kBefore, kAfter, kNone } const channel_mix_strategy_;
// Only used when the input channel layout differs from the output.
media::ChannelMixer channel_mixer_;
// Only allocated when using the channel mixer. When using the kAfter
// strategy, it is allocated just once, in the constructor, since its frame
// length is constant. When using the kBefore strategy, it is re-allocated
// whenever a larger one is needed and is reused thereafter.
std::unique_ptr<media::AudioBus> mix_bus_;
// An impossible value re-purposed to represent the "null" or "not set yet"
// condition for |read_position_| and |write_position_|.
static constexpr FrameTicks kNullPosition =
// The frame position where recording into the delay buffer always starts.
static constexpr FrameTicks kWriteStartPosition = 0;
} // namespace audio