remoting/test/test_video_renderer.cc

// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "remoting/test/test_video_renderer.h"

#include <stdint.h>
#include <utility>

#include "base/bind.h"
#include "base/callback_helpers.h"
#include "base/logging.h"
#include "base/macros.h"
#include "base/memory/ptr_util.h"
#include "base/synchronization/lock.h"
#include "base/threading/thread.h"
#include "base/threading/thread_task_runner_handle.h"
#include "remoting/codec/video_decoder.h"
#include "remoting/codec/video_decoder_verbatim.h"
#include "remoting/codec/video_decoder_vpx.h"
#include "remoting/proto/video.pb.h"
#include "remoting/test/rgb_value.h"
#include "remoting/test/video_frame_writer.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
#include "third_party/webrtc/modules/desktop_capture/shared_desktop_frame.h"

namespace {

// Used to account for frame resizing and lossy encoding error in percentage.
// The average color usually only varies by 1 on each channel, so 0.01 is large
// enough to allow variations while not being flaky for false negative cases.
const double kMaxColorError = 0.01;

}  // namespace

namespace remoting {
namespace test {

// Implements video decoding functionality.
class TestVideoRenderer::Core {
 public:
  Core();
  ~Core();

  // Initializes the internal structures of the class.
  void Initialize();

  // Used to decode video packets.
  void ProcessVideoPacket(std::unique_ptr<VideoPacket> packet,
                          base::OnceClosure done);

  // Initialize a decoder to decode video packets.
  void SetCodecForDecoding(const protocol::ChannelConfig::Codec codec);

  // Returns a copy of the current frame.
  std::unique_ptr<webrtc::DesktopFrame> GetCurrentFrameForTest() const;

  // Set expected image pattern for comparison and the callback will be called
  // when the pattern is matched.
  void ExpectAverageColorInRect(
      const webrtc::DesktopRect& expected_rect,
      const RGBValue& expected_avg_color,
      const base::Closure& image_pattern_matched_callback);

  // Turn on/off saving video frames to disk.
  void save_frame_data_to_disk(bool save_frame_data_to_disk) {
    save_frame_data_to_disk_ = save_frame_data_to_disk;
  }

 private:
  // Returns average color of pixels fall within |rect| on the current frame.
  RGBValue CalculateAverageColorValue(const webrtc::DesktopRect& rect) const;

  // Compares |candidate_avg_value| to |expected_avg_color_|.
  // Returns true if the root mean square of the errors in the R, G and B
  // components does not exceed a given limit.
  bool ExpectedAverageColorIsMatched(const RGBValue& candidate_avg_value) const;

  // Used to ensure Core methods are called on the same thread.
  base::ThreadChecker thread_checker_;

  // Used to decode video packets.
  std::unique_ptr<VideoDecoder> decoder_;

  // Used to post tasks back to main thread.
  scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;

  // Protects access to |frame_|.
  mutable base::Lock lock_;

  // Used to store decoded video frame.
  std::unique_ptr<webrtc::SharedDesktopFrame> frame_;

  // Used to store the expected image pattern.
  webrtc::DesktopRect expected_rect_;
  RGBValue expected_avg_color_;

  // Used to store the callback when expected pattern is matched.
  base::Closure image_pattern_matched_callback_;

  // Used to identify whether saving frame frame data to disk.
  bool save_frame_data_to_disk_;

  // Used to dump video frames and generate image patterns.
  VideoFrameWriter video_frame_writer;

  DISALLOW_COPY_AND_ASSIGN(Core);
};

TestVideoRenderer::Core::Core()
    : main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
      save_frame_data_to_disk_(false) {
  thread_checker_.DetachFromThread();
}

TestVideoRenderer::Core::~Core() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

void TestVideoRenderer::Core::Initialize() {
  DCHECK(thread_checker_.CalledOnValidThread());
}

void TestVideoRenderer::Core::SetCodecForDecoding(
    const protocol::ChannelConfig::Codec codec) {
  DCHECK(thread_checker_.CalledOnValidThread());

  if (decoder_) {
    LOG(WARNING) << "Decoder is set more than once";
  }

  switch (codec) {
    case protocol::ChannelConfig::CODEC_VP8: {
      VLOG(1) << "Test Video Renderer will use VP8 decoder";
      decoder_ = VideoDecoderVpx::CreateForVP8();
      break;
    }
    case protocol::ChannelConfig::CODEC_VP9: {
      VLOG(1) << "Test Video Renderer will use VP9 decoder";
      decoder_ = VideoDecoderVpx::CreateForVP9();
      break;
    }
    case protocol::ChannelConfig::CODEC_VERBATIM: {
      VLOG(1) << "Test Video Renderer will use VERBATIM decoder";
      decoder_.reset(new VideoDecoderVerbatim());
      break;
    }
    default: {
      NOTREACHED() << "Unsupported codec: " << codec;
    }
  }
}

std::unique_ptr<webrtc::DesktopFrame>
TestVideoRenderer::Core::GetCurrentFrameForTest() const {
  base::AutoLock auto_lock(lock_);
  DCHECK(frame_);
  return base::WrapUnique(webrtc::BasicDesktopFrame::CopyOf(*frame_));
}

void TestVideoRenderer::Core::ProcessVideoPacket(
    std::unique_ptr<VideoPacket> packet,
    base::OnceClosure done) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(decoder_);
  DCHECK(packet);

  VLOG(2) << "TestVideoRenderer::Core::ProcessVideoPacket() Called";

  // Screen size is attached on the first packet as well as when the
  // host screen is resized.
  if (packet->format().has_screen_width() &&
      packet->format().has_screen_height()) {
    webrtc::DesktopSize source_size(packet->format().screen_width(),
                                    packet->format().screen_height());
    if (!frame_ || !frame_->size().equals(source_size)) {
      base::AutoLock auto_lock(lock_);
      frame_.reset(webrtc::SharedDesktopFrame::Wrap(
          new webrtc::BasicDesktopFrame(source_size)));
    }
  }

  // Render the result into a new DesktopFrame instance that shares buffer with
  // |frame_|. updated_region() will be updated for |new_frame|, but not for
  // |frame_|.
  std::unique_ptr<webrtc::DesktopFrame> new_frame(frame_->Share());

  {
    base::AutoLock auto_lock(lock_);
    if (!decoder_->DecodePacket(*packet, new_frame.get())) {
      LOG(ERROR) << "Decoder::DecodePacket() failed.";
      return;
    }
  }

  main_task_runner_->PostTask(FROM_HERE, std::move(done));

  if (save_frame_data_to_disk_) {
    std::unique_ptr<webrtc::DesktopFrame> frame(
        webrtc::BasicDesktopFrame::CopyOf(*frame_));
    video_frame_writer.HighlightRectInFrame(frame.get(), expected_rect_);
    video_frame_writer.WriteFrameToDefaultPath(*frame);
  }

  // Check to see if a image pattern matched reply is passed in, and whether
  // the |expected_rect_| falls within the current frame.
  if (image_pattern_matched_callback_.is_null() ||
      expected_rect_.right() > frame_->size().width() ||
      expected_rect_.bottom() > frame_->size().height()) {
    return;
  }
  // Compare the expected image pattern with the corresponding rectangle
  // region
  // on the current frame.
  RGBValue accumulating_avg_value = CalculateAverageColorValue(expected_rect_);
  if (ExpectedAverageColorIsMatched(accumulating_avg_value)) {
    main_task_runner_->PostTask(
        FROM_HERE, base::ResetAndReturn(&image_pattern_matched_callback_));
  }
}

void TestVideoRenderer::Core::ExpectAverageColorInRect(
    const webrtc::DesktopRect& expected_rect,
    const RGBValue& expected_avg_color,
    const base::Closure& image_pattern_matched_callback) {
  DCHECK(thread_checker_.CalledOnValidThread());

  expected_rect_ = expected_rect;
  expected_avg_color_ = expected_avg_color;
  image_pattern_matched_callback_ = image_pattern_matched_callback;
}

RGBValue TestVideoRenderer::Core::CalculateAverageColorValue(
    const webrtc::DesktopRect& rect) const {
  int red_sum = 0;
  int green_sum = 0;
  int blue_sum = 0;

  // Loop through pixels that fall within |accumulating_rect_| to obtain the
  // average color value.
  for (int y = rect.top(); y < rect.bottom(); ++y) {
    uint8_t* frame_pos =
        frame_->data() + (y * frame_->stride() +
                          rect.left() * webrtc::DesktopFrame::kBytesPerPixel);

    // Pixels of decoded video frame are presented in ARGB format.
    for (int x = 0; x < rect.width(); ++x) {
      red_sum += frame_pos[2];
      green_sum += frame_pos[1];
      blue_sum += frame_pos[0];
      frame_pos += 4;
    }
  }

  int area = rect.width() * rect.height();
  RGBValue rgb_value(red_sum / area, green_sum / area, blue_sum / area);
  return rgb_value;
}

bool TestVideoRenderer::Core::ExpectedAverageColorIsMatched(
    const RGBValue& candidate_avg_value) const {
  double error_sum_squares = 0;
  double red_error = expected_avg_color_.red - candidate_avg_value.red;
  double green_error = expected_avg_color_.green - candidate_avg_value.green;
  double blue_error = expected_avg_color_.blue - candidate_avg_value.blue;
  error_sum_squares = red_error * red_error + green_error * green_error +
                      blue_error * blue_error;
  error_sum_squares /= (255.0 * 255.0);

  return sqrt(error_sum_squares / 3) < kMaxColorError;
}

TestVideoRenderer::TestVideoRenderer()
    : video_decode_thread_(
        new base::Thread("TestVideoRendererVideoDecodingThread")),
      weak_factory_(this) {
  DCHECK(thread_checker_.CalledOnValidThread());

  core_.reset(new Core());
  if (!video_decode_thread_->Start()) {
    LOG(ERROR) << "Cannot start TestVideoRenderer";
  } else {
    video_decode_task_runner_ = video_decode_thread_->task_runner();
    video_decode_task_runner_->PostTask(
        FROM_HERE,
        base::BindOnce(&Core::Initialize, base::Unretained(core_.get())));
  }
}

TestVideoRenderer::~TestVideoRenderer() {
  DCHECK(thread_checker_.CalledOnValidThread());

  video_decode_task_runner_->DeleteSoon(FROM_HERE, core_.release());

  // The thread's message loop will run until it runs out of work.
  video_decode_thread_->Stop();
}

bool TestVideoRenderer::Initialize(
    const ClientContext& client_context,
    protocol::FrameStatsConsumer* stats_consumer) {
  return true;
}

void TestVideoRenderer::OnSessionConfig(const protocol::SessionConfig& config) {
  DCHECK(thread_checker_.CalledOnValidThread());

  VLOG(2) << "TestVideoRenderer::OnSessionConfig() Called";
  protocol::ChannelConfig::Codec codec = config.video_config().codec;
  SetCodecForDecoding(codec);
}

protocol::VideoStub* TestVideoRenderer::GetVideoStub() {
  DCHECK(thread_checker_.CalledOnValidThread());

  VLOG(2) << "TestVideoRenderer::GetVideoStub() Called";
  return this;
}

protocol::FrameConsumer* TestVideoRenderer::GetFrameConsumer() {
  DCHECK(thread_checker_.CalledOnValidThread());
  NOTREACHED();
  return nullptr;
}
protocol::FrameStatsConsumer* TestVideoRenderer::GetFrameStatsConsumer() {
  return nullptr;
}

void TestVideoRenderer::ProcessVideoPacket(
    std::unique_ptr<VideoPacket> video_packet,
    base::OnceClosure done) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(video_decode_task_runner_) << "Failed to start video decode thread";

  if (video_packet->has_data() && video_packet->data().size() != 0) {
    VLOG(2) << "process video packet is called!";

    // Post video process task to the video decode thread.
    base::OnceClosure process_video_task =
        base::BindOnce(&TestVideoRenderer::Core::ProcessVideoPacket,
                       base::Unretained(core_.get()), std::move(video_packet),
                       std::move(done));
    video_decode_task_runner_->PostTask(FROM_HERE,
                                        std::move(process_video_task));
  } else {
    // Log at a high verbosity level as we receive empty packets frequently and
    // they can clutter up the debug output if the level is set too low.
    VLOG(3) << "Empty Video Packet received.";
    std::move(done).Run();
  }
}

void TestVideoRenderer::SetCodecForDecoding(
    const protocol::ChannelConfig::Codec codec) {
  DCHECK(thread_checker_.CalledOnValidThread());

  VLOG(2) << "TestVideoRenderer::SetDecoder() Called";
  video_decode_task_runner_->PostTask(
      FROM_HERE, base::BindOnce(&Core::SetCodecForDecoding,
                                base::Unretained(core_.get()), codec));
}

std::unique_ptr<webrtc::DesktopFrame>
TestVideoRenderer::GetCurrentFrameForTest() const {
  DCHECK(thread_checker_.CalledOnValidThread());

  return core_->GetCurrentFrameForTest();
}

void TestVideoRenderer::ExpectAverageColorInRect(
    const webrtc::DesktopRect& expected_rect,
    const RGBValue& expected_avg_color,
    const base::Closure& image_pattern_matched_callback) {
  DCHECK(thread_checker_.CalledOnValidThread());
  DCHECK(!expected_rect.is_empty()) << "Expected rect cannot be empty";

  DVLOG(2) << "TestVideoRenderer::SetImagePatternAndMatchedCallback() Called";
  video_decode_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&Core::ExpectAverageColorInRect,
                     base::Unretained(core_.get()), expected_rect,
                     expected_avg_color, image_pattern_matched_callback));
}

void TestVideoRenderer::SaveFrameDataToDisk(bool save_frame_data_to_disk) {
  DCHECK(thread_checker_.CalledOnValidThread());

  video_decode_task_runner_->PostTask(
      FROM_HERE,
      base::BindOnce(&Core::save_frame_data_to_disk,
                     base::Unretained(core_.get()), save_frame_data_to_disk));
}

}  // namespace test
}  // namespace remoting