patch/0013-cl-khr-command-buffer.patch

From b8b5301dc98d90333682a4d26d69a2a601793177 Mon Sep 17 00:00:00 2001
From: TFLite Patcher <tflite@localhost>
Date: Fri, 18 Oct 2024 18:13:53 +0800
Subject: [PATCH] Use cl_khr_command_buffer extension with clvk

PATCH_NAME=cl-khr-command-buffer
---
 tensorflow/lite/delegates/gpu/cl/api.cc       |  2 +-
 .../delegates/gpu/cl/inference_context.cc     | 66 ++++++++++++++++---
 .../lite/delegates/gpu/cl/inference_context.h |  8 +++
 3 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/tensorflow/lite/delegates/gpu/cl/api.cc b/tensorflow/lite/delegates/gpu/cl/api.cc
index dc9a8d3a..da97b755 100644
--- a/tensorflow/lite/delegates/gpu/cl/api.cc
+++ b/tensorflow/lite/delegates/gpu/cl/api.cc
@@ -586,7 +586,7 @@ class InferenceRunnerImpl : public CLInferenceRunner {
     // but currently it does not, see the bug for details. Once fixed, this
     // code should be in an else clause of the above if statement.
     RETURN_IF_ERROR(context_->AddToQueue(queue_));
-    clFlush(queue_->queue());
+    context_->FlushQueue(queue_);
     return absl::OkStatus();
   }
 
diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.cc b/tensorflow/lite/delegates/gpu/cl/inference_context.cc
index 32cb168b..3fe6e345 100644
--- a/tensorflow/lite/delegates/gpu/cl/inference_context.cc
+++ b/tensorflow/lite/delegates/gpu/cl/inference_context.cc
@@ -316,6 +316,13 @@ absl::Status InferenceContext::InitFromGpuModel(
   shared_buffers_parent_ptr_ = shared_buffer;
   RETURN_IF_ERROR(AllocateMemory(*gpu_model, env->GetDevicePtr()->GetInfo(),
                                  &create_info, &env->context()));
+
+  gpu_info_ = env->device().GetInfo();
+  if (gpu_info_.opencl_info.IsCLVK() &&
+      gpu_info_.SupportsExtension("cl_khr_command_buffer")) {
+    use_command_buffer_ = true;
+  }
+
   InitFromGpuModel(gpu_model);
 
   CreationContext creation_context;
@@ -368,8 +375,6 @@ absl::Status InferenceContext::InitFromGpuModel(
     external_tensor.second = nullptr;
   }
 
-  gpu_info_ = env->device().GetInfo();
-
   if (serialized_model) {
     auto encoded_fb = Encode(*env->GetDevicePtr(), *env->program_cache(),
                              gpu_model_fb, &builder);
@@ -470,6 +475,9 @@ void InferenceContext::InitFromGpuModel(GpuModel* gpu_model) {
   for (const auto& output : gpu_model->output_ids_and_refs) {
     output_ids_.push_back(output.first);
   }
+  if (use_command_buffer_ && command_buffer_ready_) {
+    command_buffer_needs_reset_ = true;
+  }
   nodes_.resize(gpu_model->nodes.size());
   for (int i = 0; i < gpu_model->nodes.size(); ++i) {
     nodes_[i].cl_operation.Init(std::move(gpu_model->nodes[i].gpu_operation));
@@ -840,6 +848,42 @@ void InferenceContext::PrepareExternal() {
   }
 }
 
+absl::Status InferenceContext::AddCommandBufferToQueue(CLCommandQueue* queue) {
+  cl_int err;
+  if (command_buffer_needs_reset_) {
+    command_buffer_ready_ = false;
+    err = clReleaseCommandBufferKHR(command_buffer_);
+    if (err != CL_SUCCESS) {
+      return absl::InternalError("clReleaseCommandBufferKHR failed");
+    }
+  }
+  cl_command_queue cl_queue = queue->queue();
+  if (!command_buffer_ready_) {
+    command_buffer_ = clCreateCommandBufferKHR(1, &cl_queue, nullptr, &err);
+    if (err != CL_SUCCESS) {
+      return absl::InternalError("clCreateCommandBufferKHR failed");
+    }
+    RETURN_IF_ERROR(AddToCommanBuffer(command_buffer_));
+    err = clFinalizeCommandBufferKHR(command_buffer_);
+    if (err != CL_SUCCESS) {
+      return absl::InternalError("clFinalizeCommandBufferKHR failed");
+    }
+    command_buffer_ready_ = true;
+  }
+  err = clEnqueueCommandBufferKHR(1, &cl_queue, command_buffer_, 0, nullptr,
+                                  nullptr);
+  if (err != CL_SUCCESS) {
+    return absl::InternalError("clEnqueueCommandBufferKHR failed");
+  }
+  return absl::OkStatus();
+}
+
+void InferenceContext::FlushQueue(CLCommandQueue* queue) {
+  if (!gpu_info_.opencl_info.IsCLVK()) {
+    clFlush(queue->queue());
+  }
+}
+
 absl::Status InferenceContext::AddToQueue(CLCommandQueue* queue) {
   if (recordable_queue_ && recordable_queue_->IsSupported()) {
     return recordable_queue_->Execute(queue);
@@ -851,13 +895,17 @@ absl::Status InferenceContext::AddToQueue(CLCommandQueue* queue) {
     RETURN_IF_ERROR(
         queue->EnqueueEvent(&execution_hints_.prev_enqueue_start_point));
   }
-  int counter = 0;
-  for (auto& node : nodes_) {
-    RETURN_IF_ERROR(node.cl_operation.AddToQueue(queue));
-    counter++;
-    if (execution_hints_.flush_periodically &&
-        counter % execution_hints_.flush_period == 0) {
-      clFlush(queue->queue());
+  if (use_command_buffer_) {
+    RETURN_IF_ERROR(AddCommandBufferToQueue(queue));
+  } else {
+    int counter = 0;
+    for (auto& node : nodes_) {
+      RETURN_IF_ERROR(node.cl_operation.AddToQueue(queue));
+      counter++;
+      if (execution_hints_.flush_periodically &&
+          counter % execution_hints_.flush_period == 0) {
+        clFlush(queue->queue());
+      }
     }
   }
   if (execution_hints_.need_flush) {
diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.h b/tensorflow/lite/delegates/gpu/cl/inference_context.h
index e964b618..b900b4e1 100644
--- a/tensorflow/lite/delegates/gpu/cl/inference_context.h
+++ b/tensorflow/lite/delegates/gpu/cl/inference_context.h
@@ -84,6 +84,7 @@ class InferenceContext {
       Environment* env, std::vector<uint8_t>* serialized_model = nullptr);
 
   absl::Status AddToQueue(CLCommandQueue* queue);
+  void FlushQueue(CLCommandQueue* queue);
   absl::Status Profile(ProfilingCommandQueue* queue, ProfilingInfo* result);
   // for profiling and memory statistics
   uint64_t GetSizeOfMemoryAllocatedForIntermediateTensors() const;
@@ -155,6 +156,8 @@ class InferenceContext {
   absl::Status ClarifyTimeWithCommandBuffer(ProfilingCommandQueue* queue,
                                             ProfilingInfo* result);
 
+  absl::Status AddCommandBufferToQueue(CLCommandQueue *queue);
+
   struct ExecutionHints {
     bool need_flush = false;
 
@@ -203,6 +206,11 @@ class InferenceContext {
 
   std::unique_ptr<RecordableQueue> recordable_queue_ = nullptr;
 
+  bool use_command_buffer_ = false;
+  bool command_buffer_needs_reset_ = false;
+  bool command_buffer_ready_ = false;
+  cl_command_buffer_khr command_buffer_;
+
   GpuInfo gpu_info_;
 };