blob: 09a39933953ebf7ed53cafa7433560a147ff80d5 [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "gpu/command_buffer/service/sync_point_manager.h"
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include "base/bind.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/ref_counted.h"
#include "base/single_thread_task_runner.h"
namespace gpu {
namespace {
void RunOnThread(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
base::OnceClosure callback) {
if (task_runner->BelongsToCurrentThread()) {
std::move(callback).Run();
} else {
task_runner->PostTask(FROM_HERE, std::move(callback));
}
}
} // namespace
SyncPointOrderData::OrderFence::OrderFence(
uint32_t order,
uint64_t release,
scoped_refptr<SyncPointClientState> state,
uint64_t callback_id)
: order_num(order),
fence_release(release),
client_state(std::move(state)),
callback_id(callback_id) {}
SyncPointOrderData::OrderFence::OrderFence(const OrderFence& other) = default;
SyncPointOrderData::OrderFence::~OrderFence() = default;
SyncPointOrderData::SyncPointOrderData(SyncPointManager* sync_point_manager,
SequenceId sequence_id)
: sync_point_manager_(sync_point_manager), sequence_id_(sequence_id) {}
SyncPointOrderData::~SyncPointOrderData() {
DCHECK(destroyed_);
}
void SyncPointOrderData::Destroy() {
// Because of circular references between the SyncPointOrderData and
// SyncPointClientState, we must remove the references on destroy. Releasing
// the fence syncs in the order fence queue would be redundant at this point
// because they are assumed to be released on the destruction of the
// SyncPointClientState.
{
base::AutoLock auto_lock(lock_);
DCHECK(!destroyed_);
destroyed_ = true;
while (!order_fence_queue_.empty())
order_fence_queue_.pop();
}
// Call DestroyedSyncPointOrderData outside the lock to prevent deadlock.
sync_point_manager_->DestroyedSyncPointOrderData(sequence_id_);
}
uint32_t SyncPointOrderData::GenerateUnprocessedOrderNumber() {
base::AutoLock auto_lock(lock_);
DCHECK(!destroyed_);
last_unprocessed_order_num_ = sync_point_manager_->GenerateOrderNumber();
unprocessed_order_nums_.push(last_unprocessed_order_num_);
return last_unprocessed_order_num_;
}
void SyncPointOrderData::BeginProcessingOrderNumber(uint32_t order_num) {
DCHECK(processing_thread_checker_.CalledOnValidThread());
DCHECK_GE(order_num, current_order_num_);
// Use thread-safe accessors here because |processed_order_num_| and
// |unprocessed_order_num_| are protected by a lock.
DCHECK_GT(order_num, processed_order_num());
DCHECK_LE(order_num, unprocessed_order_num());
current_order_num_ = order_num;
paused_ = false;
}
void SyncPointOrderData::PauseProcessingOrderNumber(uint32_t order_num) {
DCHECK(processing_thread_checker_.CalledOnValidThread());
DCHECK_EQ(current_order_num_, order_num);
DCHECK(!paused_);
paused_ = true;
}
void SyncPointOrderData::FinishProcessingOrderNumber(uint32_t order_num) {
DCHECK(processing_thread_checker_.CalledOnValidThread());
DCHECK_EQ(current_order_num_, order_num);
DCHECK(!paused_);
// Catch invalid waits which were waiting on fence syncs that do not exist.
// When we end processing an order number, we should release any fence syncs
// which were suppose to be released during this order number.
// Release without the lock to avoid possible deadlocks.
std::vector<OrderFence> ensure_releases;
{
base::AutoLock auto_lock(lock_);
DCHECK_GT(order_num, processed_order_num_);
processed_order_num_ = order_num;
DCHECK(!unprocessed_order_nums_.empty());
DCHECK_EQ(order_num, unprocessed_order_nums_.front());
unprocessed_order_nums_.pop();
uint32_t next_order_num = 0;
if (!unprocessed_order_nums_.empty())
next_order_num = unprocessed_order_nums_.front();
while (!order_fence_queue_.empty()) {
const OrderFence& order_fence = order_fence_queue_.top();
// It's possible for the fence's order number to equal next order number.
// This happens when the wait was enqueued with an order number greater
// than the last unprocessed order number. So don't release the fence yet.
if (!next_order_num || order_fence.order_num < next_order_num) {
ensure_releases.push_back(order_fence);
order_fence_queue_.pop();
continue;
}
break;
}
}
for (OrderFence& order_fence : ensure_releases) {
order_fence.client_state->EnsureWaitReleased(order_fence.fence_release,
order_fence.callback_id);
}
}
uint64_t SyncPointOrderData::ValidateReleaseOrderNumber(
scoped_refptr<SyncPointClientState> client_state,
uint32_t wait_order_num,
uint64_t fence_release) {
base::AutoLock auto_lock(lock_);
if (destroyed_)
return 0;
// We should have unprocessed order numbers which could potentially release
// this fence.
if (unprocessed_order_nums_.empty())
return 0;
// We should have an unprocessed order number lower than the wait order
// number for the wait to be valid. It's not possible for wait order number to
// equal next unprocessed order number, but we handle that defensively.
if (wait_order_num <= unprocessed_order_nums_.front())
return 0;
// So far it could be valid, but add an order fence guard to be sure it
// gets released eventually.
uint32_t expected_order_num =
std::min(unprocessed_order_nums_.back(), wait_order_num);
uint64_t callback_id = ++current_callback_id_;
order_fence_queue_.push(OrderFence(expected_order_num, fence_release,
std::move(client_state), callback_id));
return callback_id;
}
SyncPointClientState::ReleaseCallback::ReleaseCallback(
uint64_t release,
base::OnceClosure callback,
uint64_t callback_id)
: release_count(release),
callback_closure(std::move(callback)),
callback_id(callback_id) {}
SyncPointClientState::ReleaseCallback::ReleaseCallback(
ReleaseCallback&& other) = default;
SyncPointClientState::ReleaseCallback::~ReleaseCallback() = default;
SyncPointClientState::SyncPointClientState(
SyncPointManager* sync_point_manager,
scoped_refptr<SyncPointOrderData> order_data,
CommandBufferNamespace namespace_id,
CommandBufferId command_buffer_id)
: sync_point_manager_(sync_point_manager),
order_data_(std::move(order_data)),
namespace_id_(namespace_id),
command_buffer_id_(command_buffer_id) {}
SyncPointClientState::~SyncPointClientState() {
DCHECK_EQ(UINT64_MAX, fence_sync_release_);
}
void SyncPointClientState::Destroy() {
// Release all fences on destruction.
ReleaseFenceSyncHelper(UINT64_MAX);
DCHECK(sync_point_manager_); // not destroyed
sync_point_manager_->DestroyedSyncPointClientState(namespace_id_,
command_buffer_id_);
sync_point_manager_ = nullptr;
}
bool SyncPointClientState::Wait(const SyncToken& sync_token,
base::OnceClosure callback) {
DCHECK(sync_point_manager_); // not destroyed
// Validate that this Wait call is between BeginProcessingOrderNumber() and
// FinishProcessingOrderNumber(), or else we may deadlock.
DCHECK(order_data_->IsProcessingOrderNumber());
return sync_point_manager_->Wait(sync_token, order_data_->sequence_id(),
order_data_->current_order_num(),
std::move(callback));
}
bool SyncPointClientState::WaitNonThreadSafe(
const SyncToken& sync_token,
scoped_refptr<base::SingleThreadTaskRunner> task_runner,
base::OnceClosure callback) {
return Wait(sync_token,
base::BindOnce(&RunOnThread, task_runner, std::move(callback)));
}
bool SyncPointClientState::IsFenceSyncReleased(uint64_t release) {
base::AutoLock lock(fence_sync_lock_);
return release <= fence_sync_release_;
}
bool SyncPointClientState::WaitForRelease(uint64_t release,
uint32_t wait_order_num,
base::OnceClosure callback) {
// Lock must be held the whole time while we validate otherwise it could be
// released while we are checking.
base::AutoLock auto_lock(fence_sync_lock_);
// Already released, do not run the callback.
if (release <= fence_sync_release_)
return false;
uint64_t callback_id =
order_data_->ValidateReleaseOrderNumber(this, wait_order_num, release);
if (callback_id) {
// Add the callback which will be called upon release.
release_callback_queue_.emplace(release, std::move(callback), callback_id);
return true;
}
DLOG(ERROR) << "Client waiting on non-existent sync token";
return false;
}
void SyncPointClientState::ReleaseFenceSync(uint64_t release) {
// Validate that this Release call is between BeginProcessingOrderNumber() and
// FinishProcessingOrderNumber(), or else we may deadlock.
DCHECK(order_data_->IsProcessingOrderNumber());
ReleaseFenceSyncHelper(release);
}
void SyncPointClientState::ReleaseFenceSyncHelper(uint64_t release) {
// Call callbacks without the lock to avoid possible deadlocks.
std::vector<base::OnceClosure> callback_list;
{
base::AutoLock auto_lock(fence_sync_lock_);
DLOG_IF(ERROR, release <= fence_sync_release_)
<< "Client submitted fence releases out of order.";
fence_sync_release_ = release;
while (!release_callback_queue_.empty() &&
release_callback_queue_.top().release_count <= release) {
ReleaseCallback& release_callback =
const_cast<ReleaseCallback&>(release_callback_queue_.top());
callback_list.emplace_back(std::move(release_callback.callback_closure));
release_callback_queue_.pop();
}
}
for (base::OnceClosure& closure : callback_list)
std::move(closure).Run();
}
void SyncPointClientState::EnsureWaitReleased(uint64_t release,
uint64_t callback_id) {
// Call callbacks without the lock to avoid possible deadlocks.
base::OnceClosure callback;
{
base::AutoLock auto_lock(fence_sync_lock_);
if (release <= fence_sync_release_)
return;
std::vector<ReleaseCallback> popped_callbacks;
popped_callbacks.reserve(release_callback_queue_.size());
while (!release_callback_queue_.empty() &&
release_callback_queue_.top().release_count <= release) {
ReleaseCallback& top_item =
const_cast<ReleaseCallback&>(release_callback_queue_.top());
if (top_item.release_count == release &&
top_item.callback_id == callback_id) {
// Call the callback, and discard this item from the callback queue.
callback = std::move(top_item.callback_closure);
} else {
// Store the item to be placed back into the callback queue later.
popped_callbacks.emplace_back(std::move(top_item));
}
release_callback_queue_.pop();
}
// Add back in popped items.
for (ReleaseCallback& popped_callback : popped_callbacks) {
release_callback_queue_.emplace(std::move(popped_callback));
}
}
if (callback) {
// This effectively releases the wait without releasing the fence.
DLOG(ERROR) << "Client did not release sync token as expected";
std::move(callback).Run();
}
}
SyncPointManager::SyncPointManager() {
// Order number 0 is treated as invalid, so increment the generator and return
// positive order numbers in GenerateOrderNumber() from now on.
order_num_generator_.GetNext();
}
SyncPointManager::~SyncPointManager() {
DCHECK(order_data_map_.empty());
for (const ClientStateMap& client_state_map : client_state_maps_)
DCHECK(client_state_map.empty());
}
scoped_refptr<SyncPointOrderData> SyncPointManager::CreateSyncPointOrderData() {
base::AutoLock auto_lock(lock_);
SequenceId sequence_id = SequenceId::FromUnsafeValue(next_sequence_id_++);
scoped_refptr<SyncPointOrderData> order_data =
new SyncPointOrderData(this, sequence_id);
DCHECK(!order_data_map_.count(sequence_id));
order_data_map_.insert(std::make_pair(sequence_id, order_data));
return order_data;
}
void SyncPointManager::DestroyedSyncPointOrderData(SequenceId sequence_id) {
base::AutoLock auto_lock(lock_);
DCHECK(order_data_map_.count(sequence_id));
order_data_map_.erase(sequence_id);
}
scoped_refptr<SyncPointClientState>
SyncPointManager::CreateSyncPointClientState(
CommandBufferNamespace namespace_id,
CommandBufferId command_buffer_id,
SequenceId sequence_id) {
scoped_refptr<SyncPointOrderData> order_data =
GetSyncPointOrderData(sequence_id);
scoped_refptr<SyncPointClientState> client_state = new SyncPointClientState(
this, order_data, namespace_id, command_buffer_id);
{
base::AutoLock auto_lock(lock_);
DCHECK_GE(namespace_id, 0);
DCHECK_LT(static_cast<size_t>(namespace_id), arraysize(client_state_maps_));
DCHECK(!client_state_maps_[namespace_id].count(command_buffer_id));
client_state_maps_[namespace_id].insert(
std::make_pair(command_buffer_id, client_state));
}
return client_state;
}
void SyncPointManager::DestroyedSyncPointClientState(
CommandBufferNamespace namespace_id,
CommandBufferId command_buffer_id) {
base::AutoLock auto_lock(lock_);
DCHECK_GE(namespace_id, 0);
DCHECK_LT(static_cast<size_t>(namespace_id), arraysize(client_state_maps_));
DCHECK(client_state_maps_[namespace_id].count(command_buffer_id));
client_state_maps_[namespace_id].erase(command_buffer_id);
}
bool SyncPointManager::IsSyncTokenReleased(const SyncToken& sync_token) {
scoped_refptr<SyncPointClientState> release_state = GetSyncPointClientState(
sync_token.namespace_id(), sync_token.command_buffer_id());
if (release_state)
return release_state->IsFenceSyncReleased(sync_token.release_count());
return true;
}
SequenceId SyncPointManager::GetSyncTokenReleaseSequenceId(
const SyncToken& sync_token) {
scoped_refptr<SyncPointClientState> client_state = GetSyncPointClientState(
sync_token.namespace_id(), sync_token.command_buffer_id());
if (client_state)
return client_state->sequence_id();
return SequenceId();
}
uint32_t SyncPointManager::GetProcessedOrderNum() const {
base::AutoLock auto_lock(lock_);
uint32_t processed_order_num = 0;
for (const auto& kv : order_data_map_) {
processed_order_num =
std::max(processed_order_num, kv.second->processed_order_num());
}
return processed_order_num;
}
uint32_t SyncPointManager::GetUnprocessedOrderNum() const {
base::AutoLock auto_lock(lock_);
uint32_t unprocessed_order_num = 0;
for (const auto& kv : order_data_map_) {
unprocessed_order_num =
std::max(unprocessed_order_num, kv.second->unprocessed_order_num());
}
return unprocessed_order_num;
}
bool SyncPointManager::Wait(const SyncToken& sync_token,
SequenceId sequence_id,
uint32_t wait_order_num,
base::OnceClosure callback) {
// Waits on the same sequence can cause deadlocks.
if (sequence_id == GetSyncTokenReleaseSequenceId(sync_token))
return false;
scoped_refptr<SyncPointClientState> release_state = GetSyncPointClientState(
sync_token.namespace_id(), sync_token.command_buffer_id());
if (release_state &&
release_state->WaitForRelease(sync_token.release_count(), wait_order_num,
std::move(callback))) {
return true;
}
// Do not run callback if wait is invalid.
return false;
}
bool SyncPointManager::WaitNonThreadSafe(
const SyncToken& sync_token,
SequenceId sequence_id,
uint32_t wait_order_num,
scoped_refptr<base::SingleThreadTaskRunner> task_runner,
base::OnceClosure callback) {
return Wait(sync_token, sequence_id, wait_order_num,
base::BindOnce(&RunOnThread, task_runner, std::move(callback)));
}
bool SyncPointManager::WaitOutOfOrder(const SyncToken& trusted_sync_token,
base::OnceClosure callback) {
// No order number associated with the current execution context, using
// UINT32_MAX will just assume the release is in the SyncPointClientState's
// order numbers to be executed. Null sequence id will be ignored for the
// deadlock early out check.
return Wait(trusted_sync_token, SequenceId(), UINT32_MAX,
std::move(callback));
}
uint32_t SyncPointManager::GenerateOrderNumber() {
return order_num_generator_.GetNext();
}
scoped_refptr<SyncPointClientState> SyncPointManager::GetSyncPointClientState(
CommandBufferNamespace namespace_id,
CommandBufferId command_buffer_id) {
if (namespace_id >= 0) {
DCHECK_LT(static_cast<size_t>(namespace_id), arraysize(client_state_maps_));
base::AutoLock auto_lock(lock_);
ClientStateMap& client_state_map = client_state_maps_[namespace_id];
auto it = client_state_map.find(command_buffer_id);
if (it != client_state_map.end())
return it->second;
}
return nullptr;
}
scoped_refptr<SyncPointOrderData> SyncPointManager::GetSyncPointOrderData(
SequenceId sequence_id) {
base::AutoLock auto_lock(lock_);
auto it = order_data_map_.find(sequence_id);
if (it != order_data_map_.end())
return it->second;
return nullptr;
}
} // namespace gpu