remoting/protocol/webrtc_frame_scheduler_simple.cc

// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "remoting/protocol/webrtc_frame_scheduler_simple.h"

#include <algorithm>

#include "remoting/protocol/frame_stats.h"
#include "remoting/protocol/webrtc_dummy_video_encoder.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"

namespace remoting {
namespace protocol {

namespace {

// Number of samples used to estimate processing time for the next frame.
const int kStatsWindow = 5;

const int kTargetFrameRate = 30;
constexpr base::TimeDelta kTargetFrameInterval =
    base::TimeDelta::FromMilliseconds(1000 / kTargetFrameRate);

// Target quantizer at which stop the encoding top-off.
const int kTargetQuantizerForVp8TopOff = 30;

const int64_t kPixelsPerMegapixel = 1000000;

// Minimum target bitrate per megapixel. The value is chosen experimentally such
// that when screen is not changing the codec converges to the target quantizer
// above in less than 10 frames.
const int kVp8MinimumTargetBitrateKbpsPerMegapixel = 2500;

// Threshold in number of updated pixels used to detect "big" frames. These
// frames update significant portion of the screen compared to the preceding
// frames. For these frames min quantizer may need to be adjusted in order to
// ensure that they get delivered to the client as soon as possible, in exchange
// for lower-quality image.
const int kBigFrameThresholdPixels = 300000;

// Estimated size (in bytes per megapixel) of encoded frame at target quantizer
// value (see kTargetQuantizerForVp8TopOff). Compression ratio varies depending
// on the image, so this is just a rough estimate. It's used to predict when
// encoded "big" frame may be too large to be delivered to the client quickly.
const int kEstimatedBytesPerMegapixel = 100000;

// Interval over which the bandwidth estimates is averaged to set target encoder
// bitrate.
constexpr base::TimeDelta kBandwidthAveragingInterval =
    base::TimeDelta::FromSeconds(1);

// Only update encoder bitrate when bandwidth changes by more than 33%. This
// value is chosen such that the codec is notified about significant changes in
// bandwidth, while ignoring bandwidth estimate noise. This is necessary because
// the encoder drops quality every time it's being reconfigured. When using VP8
// encoder in realtime mode encoded frame size correlates very poorly with the
// target bitrate, so it's not necessary to set target bitrate to match
// bandwidth exactly. Send bitrate is controlled more precisely by adjusting
// time intervals between frames (i.e. FPS).
const int kEncoderBitrateChangePercentage = 33;

int64_t GetRegionArea(const webrtc::DesktopRegion& region) {
  int64_t result = 0;
  for (webrtc::DesktopRegion::Iterator r(region); !r.IsAtEnd(); r.Advance()) {
    result += r.rect().width() * r.rect().height();
  }
  return result;
}

}  // namespace

WebrtcFrameSchedulerSimple::EncoderBitrateFilter::EncoderBitrateFilter() {}
WebrtcFrameSchedulerSimple::EncoderBitrateFilter::~EncoderBitrateFilter() {}

void WebrtcFrameSchedulerSimple::EncoderBitrateFilter::SetBandwidthEstimate(
    int bandwidth_kbps,
    base::TimeTicks now) {
  while (!bandwidth_samples_.empty() &&
         now - bandwidth_samples_.front().first > kBandwidthAveragingInterval) {
    bandwidth_samples_sum_ -= bandwidth_samples_.front().second;
    bandwidth_samples_.pop();
  }

  bandwidth_samples_.push(std::make_pair(now, bandwidth_kbps));
  bandwidth_samples_sum_ += bandwidth_kbps;

  UpdateTargetBitrate();
}

void WebrtcFrameSchedulerSimple::EncoderBitrateFilter::SetFrameSize(
    webrtc::DesktopSize size) {
  // TODO(sergeyu): This logic is applicable only to VP8. Reconsider it for VP9.
  minimum_bitrate_ =
      static_cast<int64_t>(kVp8MinimumTargetBitrateKbpsPerMegapixel) *
      size.width() * size.height() / 1000000LL;

  UpdateTargetBitrate();
}

int WebrtcFrameSchedulerSimple::EncoderBitrateFilter::GetTargetBitrateKbps()
    const {
  DCHECK_GT(current_target_bitrate_, 0);
  return current_target_bitrate_;
}

void WebrtcFrameSchedulerSimple::EncoderBitrateFilter::UpdateTargetBitrate() {
  if (bandwidth_samples_.empty()) {
    return;
  }

  int bandwidth_estimate = bandwidth_samples_sum_ / bandwidth_samples_.size();
  int target_bitrate = std::max(minimum_bitrate_, bandwidth_estimate);

  // Update encoder bitrate only when it changes by more than 30%. This is
  // necessary because the encoder resets internal state when it's reconfigured
  // and this causes visible drop in quality.
  if (current_target_bitrate_ == 0 ||
      std::abs(target_bitrate - current_target_bitrate_) >
          current_target_bitrate_ * kEncoderBitrateChangePercentage / 100) {
    current_target_bitrate_ = target_bitrate;
  }
}

WebrtcFrameSchedulerSimple::WebrtcFrameSchedulerSimple()
    : pacing_bucket_(LeakyBucket::kUnlimitedDepth, 0),
      frame_processing_delay_us_(kStatsWindow),
      updated_region_area_(kStatsWindow),
      weak_factory_(this) {}

WebrtcFrameSchedulerSimple::~WebrtcFrameSchedulerSimple() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

void WebrtcFrameSchedulerSimple::OnKeyFrameRequested() {
  DCHECK(thread_checker_.CalledOnValidThread());
  encoder_ready_ = true;
  key_frame_request_ = true;
  ScheduleNextFrame(base::TimeTicks::Now());
}

void WebrtcFrameSchedulerSimple::OnChannelParameters(int packet_loss,
                                                     base::TimeDelta rtt) {
  DCHECK(thread_checker_.CalledOnValidThread());

  rtt_estimate_ = rtt;
}

void WebrtcFrameSchedulerSimple::OnTargetBitrateChanged(int bandwidth_kbps) {
  DCHECK(thread_checker_.CalledOnValidThread());
  base::TimeTicks now = base::TimeTicks::Now();
  pacing_bucket_.UpdateRate(bandwidth_kbps * 1000 / 8, now);
  encoder_bitrate_.SetBandwidthEstimate(bandwidth_kbps, now);
  ScheduleNextFrame(now);
}

void WebrtcFrameSchedulerSimple::Start(
    WebrtcDummyVideoEncoderFactory* video_encoder_factory,
    const base::Closure& capture_callback) {
  DCHECK(thread_checker_.CalledOnValidThread());
  capture_callback_ = capture_callback;
  video_encoder_factory->SetVideoChannelStateObserver(
      weak_factory_.GetWeakPtr());
}

void WebrtcFrameSchedulerSimple::Pause(bool pause) {
  DCHECK(thread_checker_.CalledOnValidThread());

  paused_ = pause;
  if (paused_) {
    capture_timer_.Stop();
  } else {
    ScheduleNextFrame(base::TimeTicks::Now());
  }
}

bool WebrtcFrameSchedulerSimple::OnFrameCaptured(
    const webrtc::DesktopFrame* frame,
    WebrtcVideoEncoder::FrameParams* params_out) {
  DCHECK(thread_checker_.CalledOnValidThread());

  base::TimeTicks now = base::TimeTicks::Now();

  if ((!frame || frame->updated_region().is_empty())) {
    // If we've failed to capture a frame or captured an empty frame we still
    // need to encode and send the previous frame when top-off is active or a
    // key-frame was requested. But it makes sense only when we have a frame to
    // send, i.e. there is nothing to send if first capture request failed.
    bool resend_last_frame =
        captured_first_frame_ && (top_off_is_active_ || key_frame_request_);
    if (!resend_last_frame) {
      frame_pending_ = false;
      ScheduleNextFrame(now);
      return false;
    }
  }

  if (frame) {
    captured_first_frame_ = true;
    encoder_bitrate_.SetFrameSize(frame->size());
  }

  params_out->bitrate_kbps = encoder_bitrate_.GetTargetBitrateKbps();
  params_out->duration = kTargetFrameInterval;
  params_out->key_frame = key_frame_request_;
  key_frame_request_ = false;

  params_out->vpx_min_quantizer = 10;

  int64_t updated_area = 0;
  if (frame) {
    updated_area = params_out->key_frame
                       ? frame->size().width() * frame->size().height()
                       : GetRegionArea(frame->updated_region());
  }

  // If bandwidth is being underutilized then libvpx is likely to choose the
  // minimum allowed quantizer value, which means that encoded frame size may
  // be significantly bigger than the bandwidth allows. Detect this case and set
  // vpx_min_quantizer to 60. The quality will be topped off later.
  if (updated_area - updated_region_area_.Max() > kBigFrameThresholdPixels) {
    int expected_frame_size =
        updated_area * kEstimatedBytesPerMegapixel / kPixelsPerMegapixel;
    base::TimeDelta expected_send_delay = base::TimeDelta::FromMicroseconds(
        base::Time::kMicrosecondsPerSecond * expected_frame_size /
        pacing_bucket_.rate());
    if (expected_send_delay > kTargetFrameInterval) {
      params_out->vpx_min_quantizer = 60;
    }
  }

  updated_region_area_.Record(updated_area);

  params_out->vpx_max_quantizer = 63;

  params_out->clear_active_map = !top_off_is_active_;

  return true;
}

void WebrtcFrameSchedulerSimple::OnFrameEncoded(
    const WebrtcVideoEncoder::EncodedFrame* encoded_frame,
    HostFrameStats* frame_stats) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(frame_pending_);
  frame_pending_ = false;

  base::TimeTicks now = base::TimeTicks::Now();

  if (frame_stats) {
    // Calculate |send_pending_delay| before refilling |pacing_bucket_|.
    frame_stats->send_pending_delay =
        std::max(base::TimeDelta(), pacing_bucket_.GetEmptyTime() - now);
  }

  if (!encoded_frame || encoded_frame->data.empty()) {
    top_off_is_active_ = false;
  } else {
    pacing_bucket_.RefillOrSpill(encoded_frame->data.size(), now);

    frame_processing_delay_us_.Record(
        (now - last_capture_started_time_).InMicroseconds());

    // Top-off until the target quantizer value is reached.
    top_off_is_active_ =
        encoded_frame->quantizer > kTargetQuantizerForVp8TopOff;
  }

  ScheduleNextFrame(now);

  if (frame_stats) {
    frame_stats->rtt_estimate = rtt_estimate_;
    frame_stats->bandwidth_estimate_kbps = pacing_bucket_.rate() * 8 / 1000;
  }
}

void WebrtcFrameSchedulerSimple::ScheduleNextFrame(base::TimeTicks now) {
  DCHECK(thread_checker_.CalledOnValidThread());

  if (!encoder_ready_ || paused_ || pacing_bucket_.rate() == 0 ||
      capture_callback_.is_null() || frame_pending_) {
    return;
  }

  // If this is not the first frame then capture next frame after the previous
  // one has finished sending.
  base::TimeDelta expected_processing_time =
      base::TimeDelta::FromMicroseconds(frame_processing_delay_us_.Max());
  base::TimeTicks target_capture_time =
      pacing_bucket_.GetEmptyTime() - expected_processing_time;

  // Cap interval between frames to kTargetFrameInterval.
  if (!last_capture_started_time_.is_null()) {
    target_capture_time = std::max(
        target_capture_time, last_capture_started_time_ + kTargetFrameInterval);
  }

  if (target_capture_time < now) {
    target_capture_time = now;
  }

  capture_timer_.Start(FROM_HERE, target_capture_time - now,
                       base::Bind(&WebrtcFrameSchedulerSimple::CaptureNextFrame,
                                  base::Unretained(this)));
}

void WebrtcFrameSchedulerSimple::CaptureNextFrame() {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!frame_pending_);
  last_capture_started_time_ = base::TimeTicks::Now();
  frame_pending_ = true;
  capture_callback_.Run();
}

}  // namespace protocol
}  // namespace remoting