services/audio/snooper_node.cc

// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "services/audio/snooper_node.h"

#include <algorithm>
#include <cmath>

#include "base/bind.h"
#include "base/numerics/checked_math.h"
#include "base/trace_event/trace_event.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_timestamp_helper.h"

using Helper = media::AudioTimestampHelper;

namespace audio {

namespace {

// The delay buffer size is chosen to be a very conservative maximum, just to
// make sure there is an upper bound in-place so that the buffer won't grow
// indefinitely. In most normal cases, reads will cause the delay buffer to
// automatically prune its recording down to well under this maximum (e.g.,
// around 100 milliseconds of audio).
constexpr base::TimeDelta kDelayBufferSize =
    base::TimeDelta::FromMilliseconds(1000);

// A frequency at which people cannot discern tones that differ by 1 Hz. This is
// based on research that shows people can discern tones only when they are more
// than 1 Hz away from 500 Hz. Thus, assume people definitely can't discern
// tones 1 Hz away from 1000 Hz.
constexpr int kStepBasisHz = 1000;

// The number of frames the resampler should request at a time. Three kernel's
// worth is an arbitrary choice, but performs well since the lock guarding
// access to the delay buffer is only held a reasonably short time during the
// data extraction.
constexpr int kResamplerRequestSize = 3 * media::SincResampler::kKernelSize;

// Returns the deviation, around an estimated reference time, beyond which a
// SnooperNode considers a skip in input/output to have occurred.
base::TimeDelta GetReferenceTimeSkipThreshold(base::TimeDelta bus_duration) {
  return bus_duration / 2;
}

}  // namespace

// static
constexpr SnooperNode::FrameTicks SnooperNode::kNullPosition;

// static
constexpr SnooperNode::FrameTicks SnooperNode::kWriteStartPosition;

SnooperNode::SnooperNode(const media::AudioParameters& input_params,
                         const media::AudioParameters& output_params)
    : input_params_(input_params),
      output_params_(output_params),
      input_bus_duration_(
          Helper::FramesToTime(input_params_.frames_per_buffer(),
                               input_params_.sample_rate())),
      output_bus_duration_(
          Helper::FramesToTime(output_params_.frames_per_buffer(),
                               output_params_.sample_rate())),
      perfect_io_ratio_(static_cast<double>(input_params_.sample_rate()) /
                        output_params_.sample_rate()),
      buffer_(
          Helper::TimeToFrames(kDelayBufferSize, input_params_.sample_rate())),
      write_position_(kNullPosition),
      checkpoint_time_(base::TimeTicks::Min()),
      read_position_(kNullPosition),
      correction_fps_(0),
      resampler_(
          // For efficiency, a |channel_mix_strategy_| is chosen so that the
          // resampler is always processing the fewest number of channels.
          std::min(input_params_.channels(), output_params_.channels()),
          perfect_io_ratio_,
          kResamplerRequestSize,
          base::BindRepeating(&SnooperNode::ReadFromDelayBuffer,
                              base::Unretained(this))),
      channel_mix_strategy_(
          (input_params_.channel_layout() == output_params_.channel_layout())
              ? kNone
              : ((output_params_.channels() < input_params_.channels())
                     ? kBefore
                     : kAfter)),
      channel_mixer_(input_params_.channel_layout(),
                     output_params_.channel_layout()) {
  TRACE_EVENT2("audio", "SnooperNode::SnooperNode", "input_params",
               input_params.AsHumanReadableString(), "output_params",
               output_params.AsHumanReadableString());

  // Prime the resampler with silence to keep the calculations in Render()
  // simple.
  resampler_.PrimeWithSilence();

  // If channel mixing is to be performed after resampling, allocate a buffer to
  // hold the resampler's output.
  if (channel_mix_strategy_ == kAfter) {
    mix_bus_ = media::AudioBus::Create(input_params_.channels(),
                                       output_params_.frames_per_buffer());
  }
}

SnooperNode::~SnooperNode() = default;

void SnooperNode::OnData(const media::AudioBus& input_bus,
                         base::TimeTicks reference_time,
                         double volume) {
  DCHECK_EQ(input_bus.channels(), input_params_.channels());
  DCHECK_EQ(input_bus.frames(), input_params_.frames_per_buffer());

  TRACE_EVENT_WITH_FLOW2("audio", "SnooperNode::OnData", this,
                         TRACE_EVENT_FLAG_FLOW_IN | TRACE_EVENT_FLAG_FLOW_OUT,
                         "reference_time (bogo-μs)",
                         reference_time.since_origin().InMicroseconds(),
                         "write_position", write_position_);

  base::AutoLock scoped_lock(lock_);

  // If this is the first OnData() call, just set the starting read position.
  // Otherwise, check whether a gap (i.e., missing piece) in the recording flow
  // has occurred, and skip the write position forward if necessary.
  if (write_position_ == kNullPosition) {
    write_position_ = kWriteStartPosition;
  } else {
    const base::TimeDelta threshold =
        GetReferenceTimeSkipThreshold(input_bus_duration_);
    const base::TimeDelta delta = reference_time - write_reference_time_;
    if (delta < -threshold) {
      TRACE_EVENT_INSTANT1("audio", "SnooperNode Discards Input",
                           TRACE_EVENT_SCOPE_THREAD, "wait_time_remaining (μs)",
                           (-delta).InMicroseconds());
      // It's illegal to back-track the |write_position_| and/or attempt to
      // "rewrite history" in the delay buffer. Thus, simply drop input until it
      // catches up. Events such as this are generally only caused by device-
      // switching in audio::OutputController, where the delay timestamps may
      // shift. http://crbug.com/934770
      return;
    } else if (delta > threshold) {
      TRACE_EVENT_INSTANT1("audio", "SnooperNode Input Gap",
                           TRACE_EVENT_SCOPE_THREAD, "gap (μs)",
                           delta.InMicroseconds());
      // Skip the |write_position_| forward, which will create a zero-fill gap
      // in the delay buffer.
      write_position_ +=
          Helper::TimeToFrames(delta, input_params_.sample_rate());
    } else {
      // Normal case: Continue writing into the delay buffer at the current
      // |write_position_|.
      //
      // Note that, if input was being discarded (in the prior OnData() call),
      // there will be no "recovery adjustment" to the |write_position_|.
      // Instead, any significant jump in |write_reference_time_| will cause
      // Render() to gradually re-synchronize the audio. There will be no
      // zero-fill gap inserted into the delay buffer.
    }
  }

  buffer_.Write(write_position_, input_bus, volume);

  write_position_ += input_bus.frames();
  write_reference_time_ = reference_time + input_bus_duration_;
}

base::Optional<base::TimeTicks> SnooperNode::SuggestLatestRenderTime(
    FrameTicks duration) {
  DCHECK_GE(duration, 0);

  const base::TimeTicks last_checkpoint_time = checkpoint_time_;
  {
    base::AutoLock scoped_lock(lock_);
    if (write_position_ == kNullPosition) {
      return base::nullopt;  // OnData() never called yet.
    }
    checkpoint_time_ = write_reference_time_;
  }

  // Do not suggest any changes if OnData() has not been called since the last
  // call to this method. This may indicate an input discontinuity is occurring.
  if (checkpoint_time_ == last_checkpoint_time) {
    return base::nullopt;
  }

  // Suggest a render time by working backwards from the end time of the data
  // currently recorded in the delay buffer. Subtract from the end time: 1) the
  // maximum duration prebufferred in the resampler; 2) the duration to be
  // rendered; 3) a safety margin (to help avoid underruns when the machine is
  // under high stress).
  const base::TimeDelta max_resampler_prebuffer_duration = Helper::FramesToTime(
      kResamplerRequestSize + media::SincResampler::kKernelSize,
      input_params_.sample_rate());
  const base::TimeDelta render_duration =
      Helper::FramesToTime(duration, output_params_.sample_rate());
  const base::TimeDelta safety_margin =
      GetReferenceTimeSkipThreshold(render_duration);
  return checkpoint_time_ - max_resampler_prebuffer_duration - render_duration -
         safety_margin;
}

void SnooperNode::Render(base::TimeTicks reference_time,
                         media::AudioBus* output_bus) {
  DCHECK_EQ(output_bus->channels(), output_params_.channels());
  DCHECK_EQ(output_bus->frames(), output_params_.frames_per_buffer());

  TRACE_EVENT_WITH_FLOW1("audio", "SnooperNode::Render", this,
                         TRACE_EVENT_FLAG_FLOW_IN | TRACE_EVENT_FLAG_FLOW_OUT,
                         "reference_time (bogo-μs)",
                         reference_time.since_origin().InMicroseconds());

  // Use the difference in reference times between OnData() and Render() to
  // estimate the position of the audio about to come out of the resampler.
  lock_.Acquire();
  const FrameTicks estimated_output_position =
      (write_position_ == kNullPosition)
          ? kNullPosition
          : (write_position_ +
             Helper::TimeToFrames(reference_time - write_reference_time_,
                                  input_params_.sample_rate()));
  lock_.Release();

  // If recording has not started, just output silence.
  if (estimated_output_position == kNullPosition) {
    output_bus->Zero();
    return;
  }

  // If this is the first Render() call after recording started, just initialize
  // the starting read position. For all successive calls, adjust the resampler
  // to account for drift, and also handle any significant time gaps between
  // Render() calls.
  if (read_position_ == kNullPosition) {
    // Walk backwards from the estimated output position to initialize the read
    // position.
    read_position_ =
        estimated_output_position + std::lround(resampler_.BufferedFrames());
    DCHECK_EQ(correction_fps_, 0);
  } else {
    const base::TimeDelta threshold =
        GetReferenceTimeSkipThreshold(output_bus_duration_);
    const base::TimeDelta delta = reference_time - render_reference_time_;
    if (delta.magnitude() < threshold) {  // Normal case: No gap.
      // Compute the drift, which is the number of frames the resampler is
      // behind in reading from the delay buffer. This calculation also accounts
      // for the frames buffered within the resampler.
      const int64_t actual_output_position =
          read_position_ - std::lround(resampler_.BufferedFrames());
      const int drift = base::checked_cast<int>(estimated_output_position -
                                                actual_output_position);
      TRACE_COUNTER_ID1("audio", "SnooperNode Drift", this, drift);

      // The goal is to have zero drift, and the target is to achieve that goal
      // in approximately one second.
      const int target_correction_fps = drift;  // Drift divided by 1 second.

      // The minimum amount to step-up or step-down the correction rate. Using
      // this prevents excessive "churn" within the resampler, where otherwise
      // it would be recomputing its convolution kernel too often.
      const int fps_step = input_params_.sample_rate() / kStepBasisHz;
      DCHECK_GT(fps_step, 0);

      // Adjust the correction rate (and resampling ratio) if the above-computed
      // |target_correction_fps| is more than one |fps_step| different than the
      // current |correction_fps_|. Otherwise, leave the current rate unchanged,
      // to avoid reconfiguring the resampler too often.
      const int diff = target_correction_fps - correction_fps_;
      if (diff > fps_step || diff < -fps_step) {
        UpdateCorrectionRate(correction_fps_ + ((diff / fps_step) * fps_step));
      } else {
        // No correction necessary.
      }
    } else {  // Some type of rewind, fast-forward, or a rendering gap.
      TRACE_EVENT_INSTANT1("audio", "SnooperNode Render Skip",
                           TRACE_EVENT_SCOPE_THREAD, "delta (μs)",
                           delta.InMicroseconds());

      // Rather than flush and re-prime the resampler, just seek to its next
      // read-from position.
      read_position_ +=
          Helper::TimeToFrames(delta, input_params_.sample_rate());

      // This special event casts doubt on the validity of the current
      // correction rates. The system is likely to behave differently going
      // forward. Thus, set a zero correction rate.
      UpdateCorrectionRate(0);
    }

    TRACE_COUNTER_ID1("audio", "SnooperNode Correction FPS", this,
                      correction_fps_);
  }

  // Perform resampling and also channel mixing, if required. The resampler will
  // call ReadFromDelayBuffer(), as needed, to supply itself with more input
  // data; and this will move the |read_position_| forward.
  if (channel_mix_strategy_ == kAfter) {
    resampler_.Resample(mix_bus_->frames(), mix_bus_.get());
    channel_mixer_.Transform(mix_bus_.get(), output_bus);
  } else {
    resampler_.Resample(output_bus->frames(), output_bus);
  }

  render_reference_time_ = reference_time + output_bus_duration_;
}

void SnooperNode::UpdateCorrectionRate(int correction_fps) {
  correction_fps_ = correction_fps;
  const double ratio_adjustment =
      static_cast<double>(correction_fps_) / input_params_.sample_rate();
  DCHECK_GT(ratio_adjustment, -perfect_io_ratio_);
  resampler_.SetRatio(perfect_io_ratio_ + ratio_adjustment);
}

void SnooperNode::ReadFromDelayBuffer(int ignored,
                                      media::AudioBus* resampler_bus) {
  DCHECK_NE(read_position_, kNullPosition);
  const int frames_to_read = resampler_bus->frames();
  TRACE_EVENT2("audio", "SnooperNode::ReadFromDelayBuffer", "read_position",
               read_position_, "frames", frames_to_read);

  if (channel_mix_strategy_ == kBefore) {
    DCHECK_EQ(resampler_bus->channels(), output_params_.channels());

    // Reallocate the |mix_bus_| if needed.
    if (!mix_bus_ || mix_bus_->frames() < frames_to_read) {
      mix_bus_ = nullptr;  // Free memory before allocating more.
      mix_bus_ =
          media::AudioBus::Create(input_params_.channels(), frames_to_read);
    }

    // Do the read and also channel remix before resampling.
    lock_.Acquire();
    buffer_.Read(read_position_, frames_to_read, mix_bus_.get());
    lock_.Release();
    channel_mixer_.TransformPartial(mix_bus_.get(), frames_to_read,
                                    resampler_bus);
  } else {
    DCHECK_EQ(resampler_bus->channels(), input_params_.channels());
    lock_.Acquire();
    buffer_.Read(read_position_, frames_to_read, resampler_bus);
    lock_.Release();
  }

  read_position_ += frames_to_read;
}

}  // namespace audio