blob: 4aa075d90b0b57d1db57bdc924facc8c2fe9306c [file] [log] [blame]
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/renderer/media/stream/media_stream_audio_processor.h"
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <limits>
#include <string>
#include <utility>
#include <vector>
#include "base/bind.h"
#include "base/command_line.h"
#include "base/feature_list.h"
#include "base/metrics/field_trial.h"
#include "base/metrics/field_trial_params.h"
#include "base/metrics/histogram_macros.h"
#include "base/optional.h"
#include "base/single_thread_task_runner.h"
#include "base/strings/string_number_conversions.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "content/public/common/content_client.h"
#include "content/public/common/content_features.h"
#include "content/public/renderer/content_renderer_client.h"
#include "content/renderer/media/webrtc/webrtc_audio_device_impl.h"
#include "media/base/audio_converter.h"
#include "media/base/audio_fifo.h"
#include "media/base/audio_parameters.h"
#include "media/base/channel_layout.h"
#include "media/webrtc/webrtc_switches.h"
#include "third_party/webrtc/api/audio/echo_canceller3_config.h"
#include "third_party/webrtc/api/audio/echo_canceller3_config_json.h"
#include "third_party/webrtc/api/audio/echo_canceller3_factory.h"
#include "third_party/webrtc/modules/audio_processing/include/audio_processing_statistics.h"
#include "third_party/webrtc/modules/audio_processing/typing_detection.h"
#include "third_party/webrtc_overrides/task_queue_factory.h"
namespace content {
using EchoCancellationType =
blink::AudioProcessingProperties::EchoCancellationType;
namespace {
using webrtc::AudioProcessing;
using NoiseSuppression = webrtc::AudioProcessing::Config::NoiseSuppression;
constexpr int kAudioProcessingNumberOfChannels = 1;
constexpr int kBuffersPerSecond = 100; // 10 ms per buffer.
AudioProcessing::ChannelLayout MapLayout(media::ChannelLayout media_layout) {
switch (media_layout) {
case media::CHANNEL_LAYOUT_MONO:
return AudioProcessing::kMono;
case media::CHANNEL_LAYOUT_STEREO:
return AudioProcessing::kStereo;
case media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC:
return AudioProcessing::kStereoAndKeyboard;
default:
NOTREACHED() << "Layout not supported: " << media_layout;
return AudioProcessing::kMono;
}
}
// This is only used for playout data where only max two channels is supported.
AudioProcessing::ChannelLayout ChannelsToLayout(int num_channels) {
switch (num_channels) {
case 1:
return AudioProcessing::kMono;
case 2:
return AudioProcessing::kStereo;
default:
NOTREACHED() << "Channels not supported: " << num_channels;
return AudioProcessing::kMono;
}
}
// Used by UMA histograms and entries shouldn't be re-ordered or removed.
enum AudioTrackProcessingStates {
AUDIO_PROCESSING_ENABLED = 0,
AUDIO_PROCESSING_DISABLED,
AUDIO_PROCESSING_IN_WEBRTC,
AUDIO_PROCESSING_MAX
};
void RecordProcessingState(AudioTrackProcessingStates state) {
UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates",
state, AUDIO_PROCESSING_MAX);
}
// Checks if the default minimum starting volume value for the AGC is overridden
// on the command line.
base::Optional<int> GetStartupMinVolumeForAgc() {
std::string min_volume_str(
base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
switches::kAgcStartupMinVolume));
int startup_min_volume;
if (min_volume_str.empty() ||
!base::StringToInt(min_volume_str, &startup_min_volume)) {
return base::Optional<int>();
}
return base::Optional<int>(startup_min_volume);
}
} // namespace
// Wraps AudioBus to provide access to the array of channel pointers, since this
// is the type webrtc::AudioProcessing deals in. The array is refreshed on every
// channel_ptrs() call, and will be valid until the underlying AudioBus pointers
// are changed, e.g. through calls to SetChannelData() or SwapChannels().
//
// All methods are called on one of the capture or render audio threads
// exclusively.
class MediaStreamAudioBus {
public:
MediaStreamAudioBus(int channels, int frames)
: bus_(media::AudioBus::Create(channels, frames)),
channel_ptrs_(new float*[channels]) {
// May be created in the main render thread and used in the audio threads.
DETACH_FROM_THREAD(thread_checker_);
}
void ReattachThreadChecker() {
DETACH_FROM_THREAD(thread_checker_);
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
}
media::AudioBus* bus() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
return bus_.get();
}
float* const* channel_ptrs() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
for (int i = 0; i < bus_->channels(); ++i) {
channel_ptrs_[i] = bus_->channel(i);
}
return channel_ptrs_.get();
}
private:
THREAD_CHECKER(thread_checker_);
std::unique_ptr<media::AudioBus> bus_;
std::unique_ptr<float* []> channel_ptrs_;
};
// Wraps AudioFifo to provide a cleaner interface to MediaStreamAudioProcessor.
// It avoids the FIFO when the source and destination frames match. All methods
// are called on one of the capture or render audio threads exclusively. If
// |source_channels| is larger than |destination_channels|, only the first
// |destination_channels| are kept from the source.
class MediaStreamAudioFifo {
public:
MediaStreamAudioFifo(int source_channels,
int destination_channels,
int source_frames,
int destination_frames,
int sample_rate)
: source_channels_(source_channels),
source_frames_(source_frames),
sample_rate_(sample_rate),
destination_(
new MediaStreamAudioBus(destination_channels, destination_frames)),
data_available_(false) {
DCHECK_GE(source_channels, destination_channels);
if (source_channels > destination_channels) {
audio_source_intermediate_ =
media::AudioBus::CreateWrapper(destination_channels);
}
if (source_frames != destination_frames) {
// Since we require every Push to be followed by as many Consumes as
// possible, twice the larger of the two is a (probably) loose upper bound
// on the FIFO size.
const int fifo_frames = 2 * std::max(source_frames, destination_frames);
fifo_.reset(new media::AudioFifo(destination_channels, fifo_frames));
}
// May be created in the main render thread and used in the audio threads.
DETACH_FROM_THREAD(thread_checker_);
}
void ReattachThreadChecker() {
DETACH_FROM_THREAD(thread_checker_);
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
destination_->ReattachThreadChecker();
}
void Push(const media::AudioBus& source, base::TimeDelta audio_delay) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DCHECK_EQ(source.channels(), source_channels_);
DCHECK_EQ(source.frames(), source_frames_);
const media::AudioBus* source_to_push = &source;
if (audio_source_intermediate_) {
for (int i = 0; i < destination_->bus()->channels(); ++i) {
audio_source_intermediate_->SetChannelData(
i,
const_cast<float*>(source.channel(i)));
}
audio_source_intermediate_->set_frames(source.frames());
source_to_push = audio_source_intermediate_.get();
}
if (fifo_) {
CHECK_LT(fifo_->frames(), destination_->bus()->frames());
next_audio_delay_ = audio_delay +
fifo_->frames() * base::TimeDelta::FromSeconds(1) / sample_rate_;
fifo_->Push(source_to_push);
} else {
CHECK(!data_available_);
source_to_push->CopyTo(destination_->bus());
next_audio_delay_ = audio_delay;
data_available_ = true;
}
}
// Returns true if there are destination_frames() of data available to be
// consumed, and otherwise false.
bool Consume(MediaStreamAudioBus** destination,
base::TimeDelta* audio_delay) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (fifo_) {
if (fifo_->frames() < destination_->bus()->frames())
return false;
fifo_->Consume(destination_->bus(), 0, destination_->bus()->frames());
*audio_delay = next_audio_delay_;
next_audio_delay_ -=
destination_->bus()->frames() * base::TimeDelta::FromSeconds(1) /
sample_rate_;
} else {
if (!data_available_)
return false;
*audio_delay = next_audio_delay_;
// The data was already copied to |destination_| in this case.
data_available_ = false;
}
*destination = destination_.get();
return true;
}
private:
THREAD_CHECKER(thread_checker_);
const int source_channels_; // For a DCHECK.
const int source_frames_; // For a DCHECK.
const int sample_rate_;
std::unique_ptr<media::AudioBus> audio_source_intermediate_;
std::unique_ptr<MediaStreamAudioBus> destination_;
std::unique_ptr<media::AudioFifo> fifo_;
// When using |fifo_|, this is the audio delay of the first sample to be
// consumed next from the FIFO. When not using |fifo_|, this is the audio
// delay of the first sample in |destination_|.
base::TimeDelta next_audio_delay_;
// True when |destination_| contains the data to be returned by the next call
// to Consume(). Only used when the FIFO is disabled.
bool data_available_;
};
MediaStreamAudioProcessor::MediaStreamAudioProcessor(
const blink::AudioProcessingProperties& properties,
WebRtcPlayoutDataSource* playout_data_source)
: render_delay_ms_(0),
audio_delay_stats_reporter_(kBuffersPerSecond),
playout_data_source_(playout_data_source),
main_thread_runner_(base::ThreadTaskRunnerHandle::Get()),
audio_mirroring_(false),
typing_detected_(false),
aec_dump_message_filter_(AecDumpMessageFilter::Get()),
stopped_(false) {
DCHECK(main_thread_runner_);
DETACH_FROM_THREAD(capture_thread_checker_);
DETACH_FROM_THREAD(render_thread_checker_);
InitializeAudioProcessingModule(properties);
// In unit tests not creating a message filter, |aec_dump_message_filter_|
// will be null. We can just ignore that. Other unit tests and browser tests
// ensure that we do get the filter when we should.
if (aec_dump_message_filter_.get())
aec_dump_message_filter_->AddDelegate(this);
}
MediaStreamAudioProcessor::~MediaStreamAudioProcessor() {
// TODO(miu): This class is ref-counted, shared among threads, and then
// requires itself to be destroyed on the main thread only?!?!? Fix this, and
// then remove the hack in WebRtcAudioSink::Adapter.
DCHECK(main_thread_runner_->BelongsToCurrentThread());
Stop();
}
void MediaStreamAudioProcessor::OnCaptureFormatChanged(
const media::AudioParameters& input_format) {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
// There is no need to hold a lock here since the caller guarantees that
// there is no more PushCaptureData() and ProcessAndConsumeData() callbacks
// on the capture thread.
InitializeCaptureFifo(input_format);
// Reset the |capture_thread_checker_| since the capture data will come from
// a new capture thread.
DETACH_FROM_THREAD(capture_thread_checker_);
}
void MediaStreamAudioProcessor::PushCaptureData(
const media::AudioBus& audio_source,
base::TimeDelta capture_delay) {
DCHECK_CALLED_ON_VALID_THREAD(capture_thread_checker_);
TRACE_EVENT1("audio", "MediaStreamAudioProcessor::PushCaptureData",
"delay (ms)", capture_delay.InMillisecondsF());
capture_fifo_->Push(audio_source, capture_delay);
}
bool MediaStreamAudioProcessor::ProcessAndConsumeData(
int volume,
bool key_pressed,
media::AudioBus** processed_data,
base::TimeDelta* capture_delay,
int* new_volume) {
DCHECK_CALLED_ON_VALID_THREAD(capture_thread_checker_);
DCHECK(processed_data);
DCHECK(capture_delay);
DCHECK(new_volume);
TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData");
MediaStreamAudioBus* process_bus;
if (!capture_fifo_->Consume(&process_bus, capture_delay))
return false;
// Use the process bus directly if audio processing is disabled.
MediaStreamAudioBus* output_bus = process_bus;
*new_volume = 0;
if (audio_processing_) {
output_bus = output_bus_.get();
*new_volume = ProcessData(process_bus->channel_ptrs(),
process_bus->bus()->frames(), *capture_delay,
volume, key_pressed, output_bus->channel_ptrs());
}
// Swap channels before interleaving the data.
if (audio_mirroring_ &&
output_format_.channel_layout() == media::CHANNEL_LAYOUT_STEREO) {
// Swap the first and second channels.
output_bus->bus()->SwapChannels(0, 1);
}
*processed_data = output_bus->bus();
return true;
}
void MediaStreamAudioProcessor::Stop() {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
if (stopped_)
return;
stopped_ = true;
if (aec_dump_message_filter_.get()) {
aec_dump_message_filter_->RemoveDelegate(this);
aec_dump_message_filter_ = nullptr;
}
if (!audio_processing_.get())
return;
audio_processing_.get()->UpdateHistogramsOnCallEnd();
blink::StopEchoCancellationDump(audio_processing_.get());
worker_queue_.reset(nullptr);
if (playout_data_source_) {
playout_data_source_->RemovePlayoutSink(this);
playout_data_source_ = nullptr;
}
}
const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const {
return input_format_;
}
const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const {
return output_format_;
}
void MediaStreamAudioProcessor::OnAecDumpFile(
const IPC::PlatformFileForTransit& file_handle) {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
base::File file = IPC::PlatformFileForTransitToFile(file_handle);
DCHECK(file.IsValid());
if (audio_processing_) {
if (!worker_queue_) {
worker_queue_ = std::make_unique<rtc::TaskQueue>(
CreateWebRtcTaskQueue(rtc::TaskQueue::Priority::LOW));
}
// Here tasks will be posted on the |worker_queue_|. It must be
// kept alive until StopEchoCancellationDump is called or the
// webrtc::AudioProcessing instance is destroyed.
blink::StartEchoCancellationDump(audio_processing_.get(), std::move(file),
worker_queue_.get());
} else {
file.Close();
}
}
void MediaStreamAudioProcessor::OnDisableAecDump() {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
if (audio_processing_)
blink::StopEchoCancellationDump(audio_processing_.get());
// Note that deleting an rtc::TaskQueue has to be done from the
// thread that created it.
worker_queue_.reset(nullptr);
}
void MediaStreamAudioProcessor::OnIpcClosing() {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
aec_dump_message_filter_ = nullptr;
}
// static
bool MediaStreamAudioProcessor::WouldModifyAudio(
const blink::AudioProcessingProperties& properties) {
// Note: This method should by kept in-sync with any changes to the logic in
// MediaStreamAudioProcessor::InitializeAudioProcessingModule().
if (properties.goog_audio_mirroring)
return true;
#if !defined(OS_IOS)
if (properties.EchoCancellationIsWebRtcProvided() ||
properties.goog_auto_gain_control) {
return true;
}
#endif
#if !defined(OS_IOS) && !defined(OS_ANDROID)
if (properties.goog_experimental_echo_cancellation ||
properties.goog_typing_noise_detection) {
return true;
}
#endif
if (properties.goog_noise_suppression ||
properties.goog_experimental_noise_suppression ||
properties.goog_highpass_filter) {
return true;
}
return false;
}
void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
int sample_rate,
int audio_delay_milliseconds) {
DCHECK_CALLED_ON_VALID_THREAD(render_thread_checker_);
DCHECK_GE(audio_bus->channels(), 1);
DCHECK_LE(audio_bus->channels(), 2);
int frames_per_10_ms = sample_rate / 100;
if (audio_bus->frames() != frames_per_10_ms) {
if (unsupported_buffer_size_log_count_ < 10) {
LOG(ERROR) << "MSAP::OnPlayoutData: Unsupported audio buffer size "
<< audio_bus->frames() << ", expected " << frames_per_10_ms;
++unsupported_buffer_size_log_count_;
}
return;
}
TRACE_EVENT1("audio", "MediaStreamAudioProcessor::OnPlayoutData",
"delay (ms)", audio_delay_milliseconds);
DCHECK_LT(audio_delay_milliseconds,
std::numeric_limits<base::subtle::Atomic32>::max());
base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds);
std::vector<const float*> channel_ptrs(audio_bus->channels());
for (int i = 0; i < audio_bus->channels(); ++i)
channel_ptrs[i] = audio_bus->channel(i);
// TODO(ajm): Should AnalyzeReverseStream() account for the
// |audio_delay_milliseconds|?
const int apm_error = audio_processing_->AnalyzeReverseStream(
channel_ptrs.data(), audio_bus->frames(), sample_rate,
ChannelsToLayout(audio_bus->channels()));
if (apm_error != webrtc::AudioProcessing::kNoError &&
apm_playout_error_code_log_count_ < 10) {
LOG(ERROR) << "MSAP::OnPlayoutData: AnalyzeReverseStream error="
<< apm_error;
++apm_playout_error_code_log_count_;
}
}
void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
// There is no need to hold a lock here since the caller guarantees that
// there is no more OnPlayoutData() callback on the render thread.
DETACH_FROM_THREAD(render_thread_checker_);
}
void MediaStreamAudioProcessor::OnRenderThreadChanged() {
DETACH_FROM_THREAD(render_thread_checker_);
DCHECK_CALLED_ON_VALID_THREAD(render_thread_checker_);
}
webrtc::AudioProcessorInterface::AudioProcessorStatistics
MediaStreamAudioProcessor::GetStats(bool has_remote_tracks) {
AudioProcessorStatistics stats;
stats.typing_noise_detected =
(base::subtle::Acquire_Load(&typing_detected_) != false);
stats.apm_statistics = audio_processing_->GetStatistics(has_remote_tracks);
return stats;
}
void MediaStreamAudioProcessor::InitializeAudioProcessingModule(
const blink::AudioProcessingProperties& properties) {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
DCHECK(!audio_processing_);
// Note: The audio mirroring constraint (i.e., swap left and right channels)
// is handled within this MediaStreamAudioProcessor and does not, by itself,
// require webrtc::AudioProcessing.
audio_mirroring_ = properties.goog_audio_mirroring;
#if defined(OS_ANDROID)
const bool goog_experimental_aec = false;
const bool goog_typing_detection = false;
#else
const bool goog_experimental_aec =
properties.goog_experimental_echo_cancellation;
const bool goog_typing_detection = properties.goog_typing_noise_detection;
#endif
// Return immediately if none of the goog constraints requiring
// webrtc::AudioProcessing are enabled.
if (!properties.EchoCancellationIsWebRtcProvided() &&
!goog_experimental_aec && !properties.goog_noise_suppression &&
!properties.goog_highpass_filter && !goog_typing_detection &&
!properties.goog_auto_gain_control &&
!properties.goog_experimental_noise_suppression) {
// Sanity-check: WouldModifyAudio() should return true iff
// |audio_mirroring_| is true.
DCHECK_EQ(audio_mirroring_, WouldModifyAudio(properties));
RecordProcessingState(AUDIO_PROCESSING_DISABLED);
return;
}
// Sanity-check: WouldModifyAudio() should return true because the above logic
// has determined webrtc::AudioProcessing will be used.
DCHECK(WouldModifyAudio(properties));
// Experimental options provided at creation.
webrtc::Config config;
// If the experimental AGC is enabled, check for overridden config params.
if (properties.goog_experimental_auto_gain_control) {
auto startup_min_volume = GetStartupMinVolumeForAgc();
auto* experimental_agc =
new webrtc::ExperimentalAgc(true, startup_min_volume.value_or(0));
experimental_agc->digital_adaptive_disabled =
base::FeatureList::IsEnabled(features::kWebRtcHybridAgc);
config.Set<webrtc::ExperimentalAgc>(experimental_agc);
#if defined(IS_CHROMECAST)
} else {
config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(false));
#endif // defined(IS_CHROMECAST)
}
// Create and configure the webrtc::AudioProcessing.
base::Optional<std::string> audio_processing_platform_config_json;
if (GetContentClient() && GetContentClient()->renderer()) {
audio_processing_platform_config_json =
GetContentClient()
->renderer()
->WebRTCPlatformSpecificAudioProcessingConfiguration();
}
webrtc::AudioProcessingBuilder ap_builder;
if (properties.EchoCancellationIsWebRtcProvided()) {
webrtc::EchoCanceller3Config aec3_config;
if (audio_processing_platform_config_json) {
aec3_config = webrtc::Aec3ConfigFromJsonString(
*audio_processing_platform_config_json);
bool config_parameters_already_valid =
webrtc::EchoCanceller3Config::Validate(&aec3_config);
RTC_DCHECK(config_parameters_already_valid);
}
ap_builder.SetEchoControlFactory(
std::unique_ptr<webrtc::EchoControlFactory>(
new webrtc::EchoCanceller3Factory(aec3_config)));
}
audio_processing_.reset(ap_builder.Create(config));
// Enable the audio processing components.
if (playout_data_source_) {
playout_data_source_->AddPlayoutSink(this);
}
// TODO(saza): When Chrome uses AGC2, handle all JSON config via the
// webrtc::AudioProcessing::Config, crbug.com/895814.
base::Optional<double> pre_amplifier_fixed_gain_factor,
gain_control_compression_gain_db;
blink::GetExtraGainConfig(audio_processing_platform_config_json,
&pre_amplifier_fixed_gain_factor,
&gain_control_compression_gain_db);
webrtc::AudioProcessing::Config apm_config = audio_processing_->GetConfig();
apm_config.high_pass_filter.enabled = properties.goog_highpass_filter;
if (properties.goog_auto_gain_control ||
properties.goog_experimental_auto_gain_control) {
bool use_hybrid_agc = false;
base::Optional<bool> use_peaks_not_rms;
base::Optional<int> saturation_margin;
if (properties.goog_experimental_auto_gain_control) {
use_hybrid_agc = base::FeatureList::IsEnabled(features::kWebRtcHybridAgc);
if (use_hybrid_agc) {
DCHECK(properties.goog_auto_gain_control)
<< "Cannot enable hybrid AGC when AGC is disabled.";
}
use_peaks_not_rms = base::GetFieldTrialParamByFeatureAsBool(
features::kWebRtcHybridAgc, "use_peaks_not_rms", false);
saturation_margin = base::GetFieldTrialParamByFeatureAsInt(
features::kWebRtcHybridAgc, "saturation_margin", -1);
}
blink::ConfigAutomaticGainControl(
&apm_config, properties.goog_auto_gain_control,
properties.goog_experimental_auto_gain_control, use_hybrid_agc,
use_peaks_not_rms, saturation_margin, gain_control_compression_gain_db);
}
blink::ConfigPreAmplifier(&apm_config, pre_amplifier_fixed_gain_factor);
if (goog_typing_detection) {
// TODO(xians): Remove this |typing_detector_| after the typing suppression
// is enabled by default.
typing_detector_.reset(new webrtc::TypingDetection());
blink::EnableTypingDetection(&apm_config, typing_detector_.get());
}
if (properties.goog_noise_suppression) {
blink::EnableNoiseSuppression(&apm_config, NoiseSuppression::kHigh);
}
if (properties.EchoCancellationIsWebRtcProvided()) {
blink::EnableEchoCancellation(&apm_config);
}
audio_processing_->ApplyConfig(apm_config);
RecordProcessingState(AUDIO_PROCESSING_ENABLED);
}
void MediaStreamAudioProcessor::InitializeCaptureFifo(
const media::AudioParameters& input_format) {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
DCHECK(input_format.IsValid());
input_format_ = input_format;
// TODO(crbug/881275): For now, we assume fixed parameters for the output when
// audio processing is enabled, to match the previous behavior. We should
// either use the input parameters (in which case, audio processing will
// convert at output) or ideally, have a backchannel from the sink to know
// what format it would prefer.
const int output_sample_rate = audio_processing_
?
#if defined(IS_CHROMECAST)
std::min(blink::kAudioProcessingSampleRate,
input_format.sample_rate())
#else
blink::kAudioProcessingSampleRate
#endif // defined(IS_CHROMECAST)
: input_format.sample_rate();
media::ChannelLayout output_channel_layout = audio_processing_ ?
media::GuessChannelLayout(kAudioProcessingNumberOfChannels) :
input_format.channel_layout();
// The output channels from the fifo is normally the same as input.
int fifo_output_channels = input_format.channels();
// Special case for if we have a keyboard mic channel on the input and no
// audio processing is used. We will then have the fifo strip away that
// channel. So we use stereo as output layout, and also change the output
// channels for the fifo.
if (input_format.channel_layout() ==
media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC &&
!audio_processing_) {
output_channel_layout = media::CHANNEL_LAYOUT_STEREO;
fifo_output_channels = ChannelLayoutToChannelCount(output_channel_layout);
}
// webrtc::AudioProcessing requires a 10 ms chunk size. We use this native
// size when processing is enabled. When disabled we use the same size as
// the source if less than 10 ms.
//
// TODO(ajm): This conditional buffer size appears to be assuming knowledge of
// the sink based on the source parameters. PeerConnection sinks seem to want
// 10 ms chunks regardless, while WebAudio sinks want less, and we're assuming
// we can identify WebAudio sinks by the input chunk size. Less fragile would
// be to have the sink actually tell us how much it wants (as in the above
// todo).
int processing_frames = input_format.sample_rate() / 100;
int output_frames = output_sample_rate / 100;
if (!audio_processing_ && input_format.frames_per_buffer() < output_frames) {
processing_frames = input_format.frames_per_buffer();
output_frames = processing_frames;
}
output_format_ = media::AudioParameters(
media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
output_channel_layout,
output_sample_rate,
output_frames);
capture_fifo_.reset(
new MediaStreamAudioFifo(input_format.channels(),
fifo_output_channels,
input_format.frames_per_buffer(),
processing_frames,
input_format.sample_rate()));
if (audio_processing_) {
output_bus_.reset(new MediaStreamAudioBus(output_format_.channels(),
output_frames));
}
}
int MediaStreamAudioProcessor::ProcessData(const float* const* process_ptrs,
int process_frames,
base::TimeDelta capture_delay,
int volume,
bool key_pressed,
float* const* output_ptrs) {
DCHECK(audio_processing_);
DCHECK_CALLED_ON_VALID_THREAD(capture_thread_checker_);
base::subtle::Atomic32 render_delay_ms =
base::subtle::Acquire_Load(&render_delay_ms_);
int64_t capture_delay_ms = capture_delay.InMilliseconds();
DCHECK_LT(capture_delay_ms,
std::numeric_limits<base::subtle::Atomic32>::max());
TRACE_EVENT2("audio", "MediaStreamAudioProcessor::ProcessData",
"capture_delay_ms", capture_delay_ms, "render_delay_ms",
render_delay_ms);
const int total_delay_ms = capture_delay_ms + render_delay_ms;
if (total_delay_ms > 300 && large_delay_log_count_ < 10) {
LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms
<< "ms; render delay: " << render_delay_ms << "ms";
++large_delay_log_count_;
}
audio_delay_stats_reporter_.ReportDelay(
capture_delay, base::TimeDelta::FromMilliseconds(render_delay_ms));
webrtc::AudioProcessing* ap = audio_processing_.get();
ap->set_stream_delay_ms(total_delay_ms);
DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel);
ap->set_stream_analog_level(volume);
ap->set_stream_key_pressed(key_pressed);
int err = ap->ProcessStream(
process_ptrs, process_frames, input_format_.sample_rate(),
MapLayout(input_format_.channel_layout()), output_format_.sample_rate(),
MapLayout(output_format_.channel_layout()), output_ptrs);
DCHECK_EQ(err, 0) << "ProcessStream() error: " << err;
if (typing_detector_) {
// Ignore remote tracks to avoid unnecessary stats computation.
auto voice_detected =
ap->GetStatistics(false /* has_remote_tracks */).voice_detected;
DCHECK(voice_detected.has_value());
bool typing_detected =
typing_detector_->Process(key_pressed, *voice_detected);
base::subtle::Release_Store(&typing_detected_, typing_detected);
}
main_thread_runner_->PostTask(
FROM_HERE,
base::BindOnce(&MediaStreamAudioProcessor::UpdateAecStats,
rtc::scoped_refptr<MediaStreamAudioProcessor>(this)));
// Return 0 if the volume hasn't been changed, and otherwise the new volume.
const int recommended_volume = ap->recommended_stream_analog_level();
return (recommended_volume == volume) ? 0 : recommended_volume;
}
void MediaStreamAudioProcessor::UpdateAecStats() {
DCHECK(main_thread_runner_->BelongsToCurrentThread());
}
} // namespace content