tflite: Reuse the delegate instance to reduce overhead

BUG=b:342537877
TEST=bazel run --config=host_clang '//tool:op_compat_test' \
       -- --stable_delegate_settings_file=$(realpath xnnpack.json)

Change-Id: Id5cb91c89da32b35a529e18cdf03a364b0415d19
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/tflite/+/5568601
Commit-Queue: Shik Chen <shik@chromium.org>
Tested-by: Shik Chen <shik@chromium.org>
Reviewed-by: Ching-Kang Yen <chingkang@chromium.org>

common/test_util/stable_delegate_env.cc

diff --git a/common/test_util/stable_delegate_env.cc b/common/test_util/stable_delegate_env.cc
index ae4b4af..435cde5 100644
--- a/common/test_util/stable_delegate_env.cc
+++ b/common/test_util/stable_delegate_env.cc
@@ -56,7 +56,18 @@
   return std::move(provided_delegates[0].delegate);
 }
 
+TfLiteDelegate* StableDelegateEnvironment::GetDelegate() {
+  if (cached_delegate_ == nullptr) {
+    cached_delegate_ = CreateDelegate();
+  }
+  return cached_delegate_.get();
+}
+
 void StableDelegateEnvironment::SetUp() {
   // Ensure that the delegate can be created before running any test.
   ASSERT_NE(CreateDelegate(), nullptr);
 }
+
+void StableDelegateEnvironment::TearDown() {
+  cached_delegate_.reset();
+}

common/test_util/stable_delegate_env.h

diff --git a/common/test_util/stable_delegate_env.h b/common/test_util/stable_delegate_env.h
index 7040725..bdc1b96 100644
--- a/common/test_util/stable_delegate_env.h
+++ b/common/test_util/stable_delegate_env.h
@@ -26,14 +26,21 @@
 
   bool InitFromCommandLine(int* argc, char** argv);
 
+  // Creates a new delegate instance. The caller owns the delegate.
   TfLiteDelegatePtr CreateDelegate();
 
+  // Retrieves a delegate instance, possibly reusing a cached instance. The
+  // testing environment owns the delegate.
+  TfLiteDelegate* GetDelegate();
+
  protected:
   void SetUp() override;
+  void TearDown() override;
 
  private:
   ToolParams params_;
   ProvidedDelegateList delegate_list_;
+  TfLiteDelegatePtr cached_delegate_ = tflite::tools::CreateNullDelegate();
 };
 
 #endif  // COMMON_TEST_UTIL_STABLE_DELEGATE_ENV_H_

tool/op_compat_test.cc

diff --git a/tool/op_compat_test.cc b/tool/op_compat_test.cc
index 4c6c766..2b47b70 100644
--- a/tool/op_compat_test.cc
+++ b/tool/op_compat_test.cc
@@ -82,13 +82,16 @@
 class ModelTest : public testing::Test {
  protected:
   void TestModel(const SingleOpModelBuilder& model_builder) {
-    auto delegate = g_env->CreateDelegate();
+    // Call GetDelegate() instead of CreateDelegate() here to reuse the delegate
+    // instance and reduce the overhead of creating/destorying the delegate
+    // itself.
+    auto delegate = g_env->GetDelegate();
     ASSERT_NE(delegate, nullptr);
 
     // Run model with/without the delegate and compare the result.
     std::vector<std::vector<double>> actual, expected;
     random_input_data_.clear();
-    RunModel(model_builder.Build(), delegate.get(), actual);
+    RunModel(model_builder.Build(), delegate, actual);
     RunModel(model_builder.Build(), nullptr, expected);
     ASSERT_EQ(actual.size(), expected.size());
     for (size_t i = 0; i < actual.size(); i++) {