device/vr/android/arcore/ar_compositor_frame_sink.cc

// Copyright 2021 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "device/vr/android/arcore/ar_compositor_frame_sink.h"

#include "base/bind.h"
#include "base/logging.h"
#include "base/memory/raw_ptr.h"
#include "base/task/bind_post_task.h"
#include "components/viz/common/quads/compositor_frame.h"
#include "components/viz/common/quads/surface_draw_quad.h"
#include "components/viz/common/quads/texture_draw_quad.h"
#include "components/viz/host/host_display_client.h"
#include "components/viz/host/renderer_settings_creation.h"
#include "device/vr/android/arcore/ar_image_transport.h"
#include "device/vr/android/web_xr_presentation_state.h"
#include "device/vr/public/cpp/xr_frame_sink_client.h"
#include "ui/android/window_android.h"
#include "ui/gfx/video_types.h"
#include "ui/gl/gl_bindings.h"

namespace {
class ArCoreHostDisplayClient : public viz::HostDisplayClient {
 public:
  explicit ArCoreHostDisplayClient(ui::WindowAndroid* root_window)
      : HostDisplayClient(gfx::kNullAcceleratedWidget),
        root_window_(root_window) {
    // TODO(https://crbug.com/1194775): Ideally, we'd DCHECK here, but the UTs
    // don't create a root_window.
  }

  ~ArCoreHostDisplayClient() override = default;

  void DidCompleteSwapWithSize(const gfx::Size& pixel_size) override {}

  void OnContextCreationResult(gpu::ContextResult context_result) override {}

  void SetWideColorEnabled(bool enabled) override {
    if (root_window_) {
      root_window_->SetWideColorEnabled(enabled);
    }
  }

  void SetPreferredRefreshRate(float refresh_rate) override {
    if (root_window_) {
      root_window_->SetPreferredRefreshRate(refresh_rate);
    }
  }

 private:
  raw_ptr<ui::WindowAndroid> root_window_;
};
}  // namespace

namespace device {
ArCompositorFrameSink::ArCompositorFrameSink(
    scoped_refptr<base::SingleThreadTaskRunner> gl_thread_task_runner,
    BeginFrameCallback on_begin_frame,
    CompositorReceivedFrameCallback on_compositor_received_frame,
    RenderingFinishedCallback on_rendering_finished,
    CanIssueNewFrameCallback on_can_issue_new_frame)
    : gl_thread_task_runner_(gl_thread_task_runner),
      on_begin_frame_(on_begin_frame),
      on_compositor_received_frame_(on_compositor_received_frame),
      on_rendering_finished_(on_rendering_finished),
      on_can_issue_new_frame_(on_can_issue_new_frame) {
  DCHECK(ArImageTransport::UseSharedBuffer())
      << "ArCompositorFrameSink only works with Shared Buffers";
  DCHECK(gl_thread_task_runner_);
}

ArCompositorFrameSink::~ArCompositorFrameSink() {
  DCHECK(IsOnGlThread());
  CloseBindingsIfOpen();
}

bool ArCompositorFrameSink::IsOnGlThread() const {
  return gl_thread_task_runner_->BelongsToCurrentThread();
}

viz::FrameSinkId ArCompositorFrameSink::FrameSinkId() {
  if (!is_initialized_)
    return {};
  return xr_frame_sink_client_->FrameSinkId();
}

void ArCompositorFrameSink::Initialize(
    gpu::SurfaceHandle surface_handle,
    ui::WindowAndroid* root_window,
    const gfx::Size& frame_size,
    device::XrFrameSinkClient* xr_frame_sink_client,
    DomOverlaySetup dom_setup,
    base::OnceCallback<void(bool)> on_initialized,
    base::OnceClosure on_bindings_disconnect) {
  DCHECK(IsOnGlThread());
  DCHECK(!is_initialized_);
  DCHECK(xr_frame_sink_client);
  DVLOG(1) << __func__;

  // Store the passed in values.
  display_client_ = std::make_unique<ArCoreHostDisplayClient>(root_window);
  frame_size_ = frame_size;
  xr_frame_sink_client_ = xr_frame_sink_client;
  on_initialized_ = std::move(on_initialized);
  on_bindings_disconnect_ = std::move(on_bindings_disconnect);

  auto root_params = viz::mojom::RootCompositorFrameSinkParams::New();

  // Create interfaces for a root CompositorFrameSink.
  // sink_remote_ and client_receiver are the bits to talk to/from the FrameSink
  root_params->compositor_frame_sink =
      sink_remote_.BindNewEndpointAndPassReceiver();
  sink_receiver_.Bind(root_params->compositor_frame_sink_client
                          .InitWithNewPipeAndPassReceiver());
  root_params->display_private =
      display_private_.BindNewEndpointAndPassReceiver();
  root_params->display_client =
      display_client_->GetBoundRemote(gl_thread_task_runner_);

  root_params->widget = surface_handle;
  root_params->gpu_compositing = true;
  root_params->renderer_settings = viz::CreateRendererSettings();

  // We can still get Frame callbacks if we don't specify this, but ideally we
  // want to tie our framerate to that of ARCore as much as we can, so by
  // setting ourselves up as an ExternalBeginFrame Controller, we can do that.
  root_params->external_begin_frame_controller =
      frame_controller_remote_.BindNewEndpointAndPassReceiver();

  // Ensure that we'll have a valid LocalSurfaceId.
  allocator_.GenerateId();

  xr_frame_sink_client_->InitializeRootCompositorFrameSink(
      std::move(root_params), dom_setup,
      base::BindPostTask(
          gl_thread_task_runner_,
          base::BindOnce(&ArCompositorFrameSink::OnRootCompositorFrameSinkReady,
                         weak_ptr_factory_.GetWeakPtr(), dom_setup)));

  // Note that since we own these remotes, Unretained(this) is okay, since they
  // will be destroyed (and thus unable to call the disconnect handler), before
  // we are.
  display_private_.set_disconnect_handler(base::BindOnce(
      &ArCompositorFrameSink::OnBindingsDisconnect, base::Unretained(this)));
  sink_remote_.set_disconnect_handler(base::BindOnce(
      &ArCompositorFrameSink::OnBindingsDisconnect, base::Unretained(this)));
  frame_controller_remote_.set_disconnect_handler(base::BindOnce(
      &ArCompositorFrameSink::OnBindingsDisconnect, base::Unretained(this)));
  sink_receiver_.set_disconnect_handler(base::BindOnce(
      &ArCompositorFrameSink::OnBindingsDisconnect, base::Unretained(this)));
}

void ArCompositorFrameSink::OnRootCompositorFrameSinkReady(
    DomOverlaySetup dom_setup) {
  DVLOG(1) << __func__;
  DCHECK(IsOnGlThread());
  DCHECK(!is_initialized_);

  if (dom_setup != DomOverlaySetup::kNone) {
    // If the frame sink client doesn't have a valid SurfaceId at this point,
    // then the compositor hierarchy did not get set up, which means that we'll
    // be unable to composite DOM content.
    absl::optional<viz::SurfaceId> dom_surface =
        xr_frame_sink_client_->GetDOMSurface();
    if (dom_surface && dom_surface->is_valid()) {
      should_composite_dom_overlay_ = true;
    } else if (dom_setup == DomOverlaySetup::kRequired) {
      std::move(on_initialized_).Run(false);
      return;
    }
  }

  display_private_->Resize(frame_size_);
  display_private_->SetDisplayVisible(true);
  sink_remote_->SetNeedsBeginFrame(true);

  is_initialized_ = true;
  std::move(on_initialized_).Run(true);
}

void ArCompositorFrameSink::RequestBeginFrame(base::TimeDelta interval,
                                              base::TimeTicks deadline) {
  DCHECK(IsOnGlThread());
  DCHECK(is_initialized_);
  // Note that this is enforced on the other side of the mojom call, so this
  // DCHECK just helps us fail a little bit earlier.
  DCHECK(can_issue_new_begin_frame_)
      << "Previous frame must finish being submitted before a new frame can be "
         "requested.";
  DVLOG(3) << __func__;

  // Note that the Unretained(this) below is okay, since if this is destroyed,
  // the remote will be closed and the callback is guaranteed to not be run.
  frame_controller_remote_->IssueExternalBeginFrame(
      viz::BeginFrameArgs::Create(BEGINFRAME_FROM_HERE, UINT64_MAX,
                                  next_begin_frame_id_++,
                                  base::TimeTicks::Now(), deadline, interval,
                                  viz::BeginFrameArgs::NORMAL),
      true,
      base::BindOnce(&ArCompositorFrameSink::OnFrameSubmitAck,
                     base::Unretained(this)));
  can_issue_new_begin_frame_ = false;
}

void ArCompositorFrameSink::SubmitFrame(WebXrFrame* xr_frame,
                                        FrameType frame_type) {
  DVLOG(3) << __func__;
  DCHECK(IsOnGlThread());
  DCHECK(is_initialized_);
  DCHECK(xr_frame);
  sink_remote_->SubmitCompositorFrame(allocator_.GetCurrentLocalSurfaceId(),
                                      CreateFrame(xr_frame, frame_type),
                                      absl::optional<viz::HitTestRegionList>(),
                                      /*trace_time=*/0);
}

void ArCompositorFrameSink::DidNotProduceFrame(WebXrFrame* xr_frame) {
  DVLOG(3) << __func__;
  DCHECK(IsOnGlThread());
  DCHECK(is_initialized_);
  DCHECK(xr_frame);
  DCHECK(xr_frame->begin_frame_args);
  sink_remote_->DidNotProduceFrame(
      viz::BeginFrameAck(*xr_frame->begin_frame_args, /*has_damage=*/false));
}

void ArCompositorFrameSink::DidReceiveCompositorFrameAck(
    std::vector<viz::ReturnedResource> resources) {
  DVLOG(3) << __func__;
  // Notify that we've received the Ack for this frame first. It may be that the
  // most recently submitted frame is also dropped, so updating the parent that
  // we acknowledged the frame can help to keep their logic more consistent.
  on_compositor_received_frame_.Run();

  // Resources can come back either via this method or ReclaimResources.
  // This method indicates that the most recently submitted frame has been
  // received; while ReclaimResources will generally be called to free up the
  // resources. However, it's all timing dependent as to which one gets called
  // with the actual freed resources.
  DVLOG(3) << __func__ << " Reclaiming Resources";
  ReclaimResources(std::move(resources));
}

void ArCompositorFrameSink::ReclaimResources(
    std::vector<viz::ReturnedResource> resources) {
  DVLOG(3) << __func__ << " resources.size()=" << resources.size();
  for (const auto& resource : resources) {
    DVLOG(3) << __func__ << " Reclaimed: " << resource.id;
    if (resource.id == viz::kInvalidResourceId)
      continue;

    auto it = id_to_frame_map_.find(resource.id);
    DCHECK(it != id_to_frame_map_.end());
    auto* rendering_frame = it->second;

    // While we now know that this resource is associated with this frame, we
    // don't know which buffer it is associated with, and we need to ensure that
    // we've got all of the resources associated with a frame cleared before we
    // actually clear the frame. First determine which buffer this ResourceId
    // was associated with and then clear it.
    if (resource.id == rendering_frame->shared_buffer->id) {
      rendering_frame->shared_buffer->id = viz::kInvalidResourceId;
    }
    if (resource.id == rendering_frame->camera_image_shared_buffer->id) {
      rendering_frame->camera_image_shared_buffer->id = viz::kInvalidResourceId;
    }

    // In order to keep our map size small we can remove this association as it
    // is no longer needed.
    id_to_frame_map_.erase(it);

    // Even though we've gotten the ReturnedResource, that just means that all
    // of the needed work has been queued up, we need to actually track the sync
    // token to determine when the frame is *actually* done. Given that each
    // frame can have multiple buffers associated with it, we'll store the token
    // until we get all of the buffers associated with the frame returned.
    rendering_frame->reclaimed_sync_tokens.push_back(resource.sync_token);

    // Once we've cleared all of the buffers on the frame that were passed to
    // viz, we can tell our parent that the frame is ready to be reclaimed
    // (though they will need to wait for the tokens if there are any associated
    // with the frame). Note that we don't need to check elsewhere as if a frame
    // was submitted, it has at least the shared buffer for the camera.
    if (rendering_frame->shared_buffer->id == viz::kInvalidResourceId &&
        rendering_frame->camera_image_shared_buffer->id ==
            viz::kInvalidResourceId) {
      on_rendering_finished_.Run(rendering_frame);
    }
  }
}

void ArCompositorFrameSink::OnBeginFrame(
    const viz::BeginFrameArgs& args,
    const viz::FrameTimingDetailsMap& timing_details) {
  on_begin_frame_.Run(args, timing_details);
}

void ArCompositorFrameSink::OnFrameSubmitAck(const viz::BeginFrameAck& ack) {
  DVLOG(3) << __func__;
  can_issue_new_begin_frame_ = true;
  on_can_issue_new_frame_.Run();
}

void ArCompositorFrameSink::CloseBindingsIfOpen() {
  DVLOG(1) << __func__;
  display_private_.reset();
  sink_remote_.reset();
  frame_controller_remote_.reset();
  sink_receiver_.reset();
  display_client_.reset();
}

void ArCompositorFrameSink::OnBindingsDisconnect() {
  DVLOG(1) << __func__;

  is_initialized_ = false;
  CloseBindingsIfOpen();
  if (on_bindings_disconnect_) {
    std::move(on_bindings_disconnect_).Run();
  }
}

viz::CompositorFrame ArCompositorFrameSink::CreateFrame(WebXrFrame* xr_frame,
                                                        FrameType frame_type) {
  DCHECK(IsOnGlThread());
  DCHECK(is_initialized_);
  DCHECK(xr_frame);
  DCHECK(xr_frame->begin_frame_args)
      << "Never received a BeginFrame for this frame";
  DVLOG(3) << __func__ << " frame_id=" << xr_frame->index
           << " frame_type=" << frame_type;

  viz::CompositorFrame frame;
  frame.metadata.begin_frame_ack =
      viz::BeginFrameAck(*xr_frame->begin_frame_args, true);
  frame.metadata.device_scale_factor = 1.f;
  frame.metadata.frame_token = ++next_frame_token_;

  // As the root compositor, we're always the first render pass.
  const viz::CompositorRenderPassId kRenderPassId{1};

  // Because the Camera Image is the fullscreen, and the AR Content is also
  // rendered fullscreen, we're outputting and saying that the "damage" (e.g.
  // changed region), is the full screen, or in other words, the size of the
  // camera image.
  gfx::Rect output_rect(frame_size_);
  gfx::Rect damage_rect = output_rect;

  auto render_pass = viz::CompositorRenderPass::Create();
  render_pass->SetNew(kRenderPassId, output_rect, damage_rect,
                      gfx::Transform());

  // The order that these are added corresponds to their Z-order, so add things
  // that should be in the front first.
  // This means that we want to add:
  // 1) The DOM content
  // 2) The GL(WebXr) content from the renderer
  // 3) The camera image

  // First the DOM, if it's enabled
  if (should_composite_dom_overlay_) {
    auto dom_surface_id = xr_frame_sink_client_->GetDOMSurface();
    bool can_composite_dom_overlay =
        dom_surface_id && dom_surface_id->is_valid();
    DVLOG(3)
        << __func__
        << " Attempting to composite DOMOverlay, can_composite_dom_overlay="
        << can_composite_dom_overlay;
    if (can_composite_dom_overlay) {
      viz::SharedQuadState* dom_quad_state =
          render_pass->CreateAndAppendSharedQuadState();
      dom_quad_state->SetAll(
          gfx::Transform(),
          /*quad_layer_rect=*/output_rect,
          /*visible_layer_rect=*/output_rect, gfx::MaskFilterInfo(),
          /*clip_rect=*/absl::nullopt, /*are_contents_opaque=*/false,
          /*opacity=*/1.f, SkBlendMode::kSrcOver, /*sorting_context_id=*/0);

      viz::SurfaceDrawQuad* dom_quad =
          render_pass->CreateAndAppendDrawQuad<viz::SurfaceDrawQuad>();
      dom_quad->SetNew(dom_quad_state, gfx::Rect(output_rect.size()),
                       gfx::Rect(output_rect.size()),
                       viz::SurfaceRange(*dom_surface_id), SK_ColorTRANSPARENT,
                       /*stretch_content_to_fill_bounds=*/true);
    }
  }

  // Setup some variables for the SharedQuadState that are the same for the
  // Camera/Renderer
  float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};

  // Next add the Renderer Content
  if (frame_type == FrameType::kHasWebXrContent) {
    WebXrSharedBuffer* renderer_buffer = xr_frame->shared_buffer.get();
    renderer_buffer->id = resource_id_generator_.GenerateNextId();
    id_to_frame_map_[renderer_buffer->id] = xr_frame;

    viz::SharedQuadState* xr_content_quad_state =
        render_pass->CreateAndAppendSharedQuadState();
    xr_content_quad_state->SetAll(
        gfx::Transform(),
        /*quad_layer_rect=*/output_rect,
        /*visible_layer_rect=*/output_rect, gfx::MaskFilterInfo(),
        /*clip_rect=*/absl::nullopt, /*are_contents_opaque=*/false,
        /*opacity=*/1.f, SkBlendMode::kSrcOver, /*sorting_context_id=*/0);

    viz::TextureDrawQuad* xr_content_quad =
        render_pass->CreateAndAppendDrawQuad<viz::TextureDrawQuad>();
    xr_content_quad->SetNew(
        xr_content_quad_state,
        /*rect=*/output_rect,
        /*visible_rect=*/output_rect,
        /*needs_blending=*/true, renderer_buffer->id,
        /*premultiplied_alpha=*/true,
        /*uv_top_left=*/xr_frame->bounds_left.origin(),
        /*uv_bottom_right=*/xr_frame->bounds_left.bottom_right(),
        /*background_color=*/SK_ColorTRANSPARENT, opacity,
        /*y_flipped=*/true,
        /*nearest_neighbor=*/false,
        /*secure_output_only=*/false, gfx::ProtectedVideoType::kClear);

    auto renderer_resource = viz::TransferableResource::MakeGL(
        renderer_buffer->mailbox_holder.mailbox,
        /*filter=*/GL_LINEAR, renderer_buffer->mailbox_holder.texture_target,
        renderer_buffer->mailbox_holder.sync_token, renderer_buffer->size,
        /*is_overlay_candidate=*/false);

    renderer_resource.id = renderer_buffer->id;
    id_to_frame_map_[renderer_buffer->id] = xr_frame;

    frame.resource_list.push_back(renderer_resource);
  }

  WebXrSharedBuffer* camera_buffer = xr_frame->camera_image_shared_buffer.get();
  camera_buffer->id = resource_id_generator_.GenerateNextId();

  viz::SharedQuadState* camera_quad_state =
      render_pass->CreateAndAppendSharedQuadState();
  camera_quad_state->SetAll(
      gfx::Transform(),
      /*quad_layer_rect=*/output_rect,
      /*visible_layer_rect=*/output_rect, gfx::MaskFilterInfo(),
      /*clip_rect=*/absl::nullopt, /*are_contents_opaque=*/true,
      /*opacity=*/1.f, SkBlendMode::kSrcOver, /*sorting_context_id=*/0);

  viz::TextureDrawQuad* camera_quad =
      render_pass->CreateAndAppendDrawQuad<viz::TextureDrawQuad>();
  // UV from 0,0 to 1,1 because the camera texture is fullscreen.
  camera_quad->SetNew(camera_quad_state,
                      /*rect=*/output_rect,
                      /*visible_rect=*/output_rect,
                      /*needs_blending=*/true, camera_buffer->id,
                      /*premultiplied_alpha=*/true,
                      /*uv_top_left=*/gfx::PointF(0.f, 0.f),
                      /*uv_bottom_right=*/gfx::PointF(1.f, 1.f),
                      /*background_color=*/SK_ColorTRANSPARENT, opacity,
                      /*y_flipped=*/true,
                      /*nearest_neighbor=*/false,
                      /*secure_output_only=*/false,
                      gfx::ProtectedVideoType::kClear);

  // Additionally append to the resource_list
  auto camera_resource = viz::TransferableResource::MakeGL(
      camera_buffer->mailbox_holder.mailbox,
      /*filter=*/GL_LINEAR, camera_buffer->mailbox_holder.texture_target,
      camera_buffer->mailbox_holder.sync_token, camera_buffer->size,
      /*is_overlay_candidate=*/false);

  camera_resource.id = camera_buffer->id;
  id_to_frame_map_[camera_buffer->id] = xr_frame;

  frame.resource_list.push_back(camera_resource);

  frame.render_pass_list.push_back(std::move(render_pass));

  return frame;
}

}  // namespace device