blob: 9ba2d7171a0698f2e683baeff02ba7ae7d338574 [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef SERVICES_AUDIO_INPUT_CONTROLLER_H_
#define SERVICES_AUDIO_INPUT_CONTROLLER_H_
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <string>
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/strings/string_piece.h"
#include "base/threading/thread_checker.h"
#include "base/timer/timer.h"
#include "build/build_config.h"
#include "media/base/audio_parameters.h"
#include "mojo/public/cpp/bindings/pending_receiver.h"
#include "mojo/public/cpp/bindings/pending_remote.h"
#include "mojo/public/cpp/bindings/receiver.h"
#include "mojo/public/cpp/bindings/remote.h"
#include "services/audio/public/mojom/audio_processing.mojom.h"
#include "services/audio/snoopable.h"
#include "services/audio/stream_monitor.h"
#include "services/audio/stream_monitor_coordinator.h"
namespace media {
class AudioBus;
class AudioInputStream;
class AudioManager;
#if defined(AUDIO_PROCESSING_IN_AUDIO_SERVICE)
class AudioProcessor;
#endif
class UserInputMonitor;
} // namespace media
namespace audio {
// Only do power monitoring for non-mobile platforms to save resources.
#if !defined(OS_ANDROID) && !defined(OS_IOS)
#define AUDIO_POWER_MONITORING
#endif
// All public methods of InputController must be called from the audio thread.
class InputController final : public StreamMonitor {
public:
// Error codes to make native logging more clear. These error codes are added
// to generic error strings to provide a higher degree of details.
// Changing these values can lead to problems when matching native debug
// logs with the actual cause of error.
enum ErrorCode {
// An unspecified error occured.
UNKNOWN_ERROR = 0,
// Failed to create an audio input stream.
STREAM_CREATE_ERROR, // = 1
// Failed to open an audio input stream.
STREAM_OPEN_ERROR, // = 2
// Native input stream reports an error. Exact reason differs between
// platforms.
STREAM_ERROR, // = 3
};
// An event handler that receives events from the InputController. The
// following methods are all called on the audio thread.
class EventHandler {
public:
// The initial "muted" state of the underlying stream is sent along with the
// OnCreated callback, to avoid the stream being treated as unmuted until an
// OnMuted callback has had time to be processed.
virtual void OnCreated(bool initially_muted) = 0;
virtual void OnError(ErrorCode error_code) = 0;
virtual void OnLog(base::StringPiece) = 0;
// Called whenever the muted state of the underlying stream changes.
virtual void OnMuted(bool is_muted) = 0;
protected:
virtual ~EventHandler() {}
};
// A synchronous writer interface used by InputController for
// synchronous writing.
class SyncWriter {
public:
virtual ~SyncWriter() {}
// Write certain amount of data from |data|.
virtual void Write(const media::AudioBus* data,
double volume,
bool key_pressed,
base::TimeTicks capture_time) = 0;
// Close this synchronous writer.
virtual void Close() = 0;
};
// enum used for determining what UMA stats to report.
enum StreamType {
VIRTUAL = 0,
HIGH_LATENCY = 1,
LOW_LATENCY = 2,
FAKE = 3,
};
~InputController() final;
media::AudioInputStream* stream_for_testing() { return stream_; }
// |user_input_monitor| is used for typing detection and can be NULL.
static std::unique_ptr<InputController> Create(
media::AudioManager* audio_manager,
EventHandler* event_handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
const std::string& device_id,
bool agc_is_enabled,
StreamMonitorCoordinator* stream_monitor_coordinator,
mojom::AudioProcessingConfigPtr processing_config);
// Starts recording using the created audio input stream.
void Record();
// Closes the audio input stream, freeing the associated resources. Must be
// called before destruction.
void Close();
// Sets the capture volume of the input stream. The value 0.0 corresponds
// to muted and 1.0 to maximum volume.
void SetVolume(double volume);
// Sets the output device which will be used to cancel audio from, if this
// input device supports echo cancellation.
void SetOutputDeviceForAec(const std::string& output_device_id);
bool ShouldRegisterWithStreamMonitorCoordinator() const;
// StreamMonitor implementation
void OnStreamActive(Snoopable* snoopable) override;
void OnStreamInactive(Snoopable* snoopable) override;
private:
// Used to log the result of capture startup.
// This was previously logged as a boolean with only the no callback and OK
// options. The enum order is kept to ensure backwards compatibility.
// Elements in this enum should not be deleted or rearranged; the only
// permitted operation is to add new elements before
// CAPTURE_STARTUP_RESULT_MAX and update CAPTURE_STARTUP_RESULT_MAX.
//
// The NO_DATA_CALLBACK enum has been replaced with NEVER_GOT_DATA,
// and there are also other histograms such as
// Media.Audio.InputStartupSuccessMac to cover issues similar
// to the ones the NO_DATA_CALLBACK was intended for.
enum CaptureStartupResult {
CAPTURE_STARTUP_OK = 0,
CAPTURE_STARTUP_CREATE_STREAM_FAILED = 1,
CAPTURE_STARTUP_OPEN_STREAM_FAILED = 2,
CAPTURE_STARTUP_NEVER_GOT_DATA = 3,
CAPTURE_STARTUP_STOPPED_EARLY = 4,
CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_STOPPED_EARLY,
};
#if defined(AUDIO_POWER_MONITORING)
// Used to log a silence report (see OnData).
// Elements in this enum should not be deleted or rearranged; the only
// permitted operation is to add new elements before SILENCE_STATE_MAX and
// update SILENCE_STATE_MAX.
// Possible silence state transitions:
// SILENCE_STATE_AUDIO_AND_SILENCE
// ^ ^
// SILENCE_STATE_ONLY_AUDIO SILENCE_STATE_ONLY_SILENCE
// ^ ^
// SILENCE_STATE_NO_MEASUREMENT
enum SilenceState {
SILENCE_STATE_NO_MEASUREMENT = 0,
SILENCE_STATE_ONLY_AUDIO = 1,
SILENCE_STATE_ONLY_SILENCE = 2,
SILENCE_STATE_AUDIO_AND_SILENCE = 3,
SILENCE_STATE_MAX = SILENCE_STATE_AUDIO_AND_SILENCE
};
#endif
#if defined(AUDIO_PROCESSING_IN_AUDIO_SERVICE)
class ProcessingHelper final : public mojom::AudioProcessorControls,
public Snoopable::Snooper {
public:
ProcessingHelper(
const media::AudioParameters& params,
media::AudioProcessingSettings processing_settings,
mojo::PendingReceiver<mojom::AudioProcessorControls> controls_receiver);
~ProcessingHelper() final;
// Snoopable::Snooper implementation
void OnData(const media::AudioBus& audio_bus,
base::TimeTicks reference_time,
double volume) final;
// mojom::AudioProcessorControls implementation.
void GetStats(GetStatsCallback callback) final;
void StartEchoCancellationDump(base::File file) final;
void StopEchoCancellationDump() final;
media::AudioProcessor* GetAudioProcessor();
// Starts monitoring |output_stream| instead of the currently monitored
// stream, if any.
void StartMonitoringStream(Snoopable* output_stream);
// Stops monitoring |output_stream|, provided it's the currently monitored
// stream.
void StopMonitoringStream(Snoopable* output_stream);
// Stops monitoring |monitored_output_stream_|, if not null.
void StopAllStreamMonitoring();
private:
// Starts and/or stops snooping and updates |monitored_output_stream_|
// appropriately.
void ChangeMonitoredStream(Snoopable* output_stream);
THREAD_CHECKER(owning_thread_);
const mojo::Receiver<mojom::AudioProcessorControls> receiver_;
const media::AudioParameters params_;
const std::unique_ptr<media::AudioProcessor> audio_processor_;
media::AudioParameters output_params_;
Snoopable* monitored_output_stream_ = nullptr;
std::unique_ptr<media::AudioBus> clamped_bus_;
};
#endif // defined(AUDIO_PROCESSING_IN_AUDIO_SERVICE)
InputController(EventHandler* handler,
SyncWriter* sync_writer,
media::UserInputMonitor* user_input_monitor,
const media::AudioParameters& params,
StreamType type,
StreamMonitorCoordinator* stream_monitor_coordinator,
mojom::AudioProcessingConfigPtr processing_config);
void DoCreate(media::AudioManager* audio_manager,
const media::AudioParameters& params,
const std::string& device_id,
bool enable_agc);
void DoReportError();
void DoLogAudioLevels(float level_dbfs, int microphone_volume_percent);
#if defined(AUDIO_POWER_MONITORING)
// Updates the silence state, see enum SilenceState above for state
// transitions.
void UpdateSilenceState(bool silence);
// Logs the silence state as UMA stat.
void LogSilenceState(SilenceState value);
#endif
// Logs the result of creating an InputController.
void LogCaptureStartupResult(CaptureStartupResult result);
// Logs whether an error was encountered suring the stream.
void LogCallbackError();
// Called by the stream with log messages.
void LogMessage(const std::string& message);
// Called on the hw callback thread. Checks for keyboard input if
// |user_input_monitor_| is set otherwise returns false.
bool CheckForKeyboardInput();
// Does power monitoring on supported platforms.
// Called on the hw callback thread.
// Returns true iff average power and mic volume was returned and should
// be posted to DoLogAudioLevels on the audio thread.
// Returns false if either power measurements are disabled or aren't needed
// right now (they're done periodically).
bool CheckAudioPower(const media::AudioBus* source,
double volume,
float* average_power_dbfs,
int* mic_volume_percent);
void CheckMutedState();
static StreamType ParamsToStreamType(const media::AudioParameters& params);
#if defined(AUDIO_PROCESSING_IN_AUDIO_SERVICE)
void UpdateVolumeAndAPMStats(base::Optional<double> new_volume);
#endif
// This class must be used on the audio manager thread.
THREAD_CHECKER(owning_thread_);
// Contains the InputController::EventHandler which receives state
// notifications from this class.
EventHandler* const handler_;
// Pointer to the audio input stream object.
// Only used on the audio thread.
media::AudioInputStream* stream_ = nullptr;
// SyncWriter is used only in low-latency mode for synchronous writing.
SyncWriter* const sync_writer_;
StreamType type_;
double max_volume_ = 0.0;
media::UserInputMonitor* const user_input_monitor_;
bool registered_to_coordinator_ = false;
StreamMonitorCoordinator* const stream_monitor_coordinator_;
mojom::AudioProcessingConfigPtr processing_config_;
#if defined(AUDIO_POWER_MONITORING)
// Whether the silence state and microphone levels should be checked and sent
// as UMA stats.
bool power_measurement_is_enabled_ = false;
// Updated each time a power measurement is performed.
base::TimeTicks last_audio_level_log_time_;
// The silence report sent as UMA stat at the end of a session.
SilenceState silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
#endif
size_t prev_key_down_count_ = 0;
// Time when the stream started recording.
base::TimeTicks stream_create_time_;
bool is_muted_ = false;
base::RepeatingTimer check_muted_state_timer_;
#if defined(AUDIO_PROCESSING_IN_AUDIO_SERVICE)
// Holds stats related to audio processing.
base::Optional<ProcessingHelper> processing_helper_;
#endif
class AudioCallback;
// Holds a pointer to the callback object that receives audio data from
// the lower audio layer. Valid only while 'recording' (between calls to
// stream_->Start() and stream_->Stop()).
// The value of this pointer is only set and read on the audio thread while
// the callbacks themselves occur on the hw callback thread. More details
// in the AudioCallback class in the cc file.
std::unique_ptr<AudioCallback> audio_callback_;
// A weak pointer factory that we use when posting tasks to the audio thread
// that we want to be automatically discarded after Close() has been called
// and that we do not want to keep the InputController instance alive
// beyond what is desired by the user of the instance. An example of where
// this is important is when we fire error notifications from the hw callback
// thread, post them to the audio thread. In that case, we do not want the
// error notification to keep the InputController alive for as long as
// the error notification is pending and then make a callback from an
// InputController that has already been closed.
// All outstanding weak pointers, are invalidated at the end of DoClose.
base::WeakPtrFactory<InputController> weak_ptr_factory_{this};
DISALLOW_COPY_AND_ASSIGN(InputController);
};
} // namespace audio
#endif // SERVICES_AUDIO_INPUT_CONTROLLER_H_