tflite: Add Fp16Eq matcher Some delegates may internally use fp16 for fp32 arithmetic, making the default EXPECT_FLOAT_EQ too strict for such cases. Since we are going to check fp16 results in many places, it would be nice for us to have a single fp16 matchers similar to EXPECT_FLOAT_EQ. This CL implements a new Fp16Eq matcher based on the implementation in tensorflow [1] to provide a more precise and easy comparison. BUG=b:374245241 TEST=bazelisk run --config='asan' -c dbg --config=host_clang '//common/test_util:fp16_compare_test' [1] https://github.com/tensorflow/tensorflow/blob/689a4291c92835bc3d7b51adb3540b87432557d0/tensorflow/lite/kernels/test_util.cc#L121-L135 Change-Id: I6660ceab524ef84cae1c6d0e98bced7c2e51b14f Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/tflite/+/6370771 Reviewed-by: Shik Chen <shik@chromium.org> Tested-by: Tommy Chiang <ototot@google.com> Commit-Queue: Tommy Chiang <ototot@google.com>
diff --git a/common/BUILD.bazel b/common/BUILD.bazel index d252424..1ebadf4 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel
@@ -168,6 +168,7 @@ ":async_driver", ":simple_model_builder", "//android:hardware_buffer", + "//common/test_util:fp16_compare", "//common/test_util:stable_delegate_env", "@com_google_absl//absl/random", "@com_google_googletest//:gtest", @@ -197,6 +198,7 @@ ":simple_model_builder", ":sync_driver", "//common/test_util:aligned_tflite_buffer_allocator", + "//common/test_util:fp16_compare", "//common/test_util:stable_delegate_env", "@com_google_absl//absl/random", "@com_google_googletest//:gtest",
diff --git a/common/async_delegate_test.cc b/common/async_delegate_test.cc index 8a489b9..2db4a1d 100644 --- a/common/async_delegate_test.cc +++ b/common/async_delegate_test.cc
@@ -4,8 +4,8 @@ * found in the LICENSE file. */ +#include <gmock/gmock.h> #include <gtest/gtest.h> - #include <vector> #include "absl/random/random.h" @@ -13,6 +13,7 @@ #include "common/async_driver.h" #include "common/scoped_ahwb.h" #include "common/simple_model_builder.h" +#include "common/test_util/fp16_compare.h" #include "common/test_util/stable_delegate_env.h" #include "tensorflow/lite/core/c/builtin_op_data.h" @@ -20,16 +21,6 @@ StableDelegateEnvironment* g_env = nullptr; -// Use a relaxed tolerance when comparing the computation results, as the -// primary goal of this test is to verify the asynchronous kernel API flow. -// Some delegates may internally use fp16 for fp32 arithmetic, making the -// default EXPECT_FLOAT_EQ too strict for such cases. -// TODO(shik): If we need checking fp16 results in more places, consider -// implementing a generic fp16 GMock matcher that computes the difference in -// ULPs (Units in the Last Place), similar to the current implementation of -// EXPECT_FLOAT_EQ, to provide a more precise comparison. -constexpr float kEps = 1e-2; - class AsyncDelegateTest : public testing::TestWithParam<int> {}; TEST_P(AsyncDelegateTest, Inference) { @@ -134,9 +125,7 @@ ASSERT_EQ(driver->Invoke(), kTfLiteOk); auto output = driver->GetOutputTensorData<float>("d"); - for (int i = 0; i < n; ++i) { - EXPECT_NEAR(output[i], d_data[i], kEps); - } + EXPECT_THAT(output, testing::ElementsAreArray(ArrayFp16Eq(d_data))); } }
diff --git a/common/sync_driver_test.cc b/common/sync_driver_test.cc index 64860a0..54d3927 100644 --- a/common/sync_driver_test.cc +++ b/common/sync_driver_test.cc
@@ -4,14 +4,15 @@ * found in the LICENSE file. */ +#include <gmock/gmock.h> #include <gtest/gtest.h> - #include <vector> #include "absl/random/random.h" #include "common/simple_model_builder.h" #include "common/sync_driver.h" #include "common/test_util/aligned_tflite_buffer_allocator.h" +#include "common/test_util/fp16_compare.h" #include "common/test_util/stable_delegate_env.h" #include "tensorflow/lite/core/c/builtin_op_data.h" @@ -19,16 +20,6 @@ StableDelegateEnvironment* g_env = nullptr; -// Use a relaxed tolerance when comparing the computation results, as the -// primary goal of this test is to verify the asynchronous kernel API flow. -// Some delegates may internally use fp16 for fp32 arithmetic, making the -// default EXPECT_FLOAT_EQ too strict for such cases. -// TODO(ototot): If we need checking fp16 results in more places, consider -// implementing a generic fp16 GMock matcher that computes the difference in -// ULPs (Units in the Last Place), similar to the current implementation of -// EXPECT_FLOAT_EQ, to provide a more precise comparison. -constexpr float kEps = 1e-2; - class AsyncDelegateTest : public testing::TestWithParam<int> {}; TEST_P(AsyncDelegateTest, Inference) { @@ -106,9 +97,7 @@ ASSERT_EQ(driver->Invoke(), kTfLiteOk); auto output = driver->GetOutputTensorData<float>("d"); - for (int i = 0; i < n; ++i) { - EXPECT_NEAR(output[i], d_data[i], kEps); - } + EXPECT_THAT(output, testing::ElementsAreArray(ArrayFp16Eq(d_data))); } }
diff --git a/common/test_util/BUILD.bazel b/common/test_util/BUILD.bazel index b1be3ba..9810fdb 100644 --- a/common/test_util/BUILD.bazel +++ b/common/test_util/BUILD.bazel
@@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") package( default_visibility = ["//:__subpackages__"], @@ -40,3 +40,23 @@ "@org_tensorflow//tensorflow/lite:util", ], ) + +cc_library( + name = "fp16_compare", + testonly = True, + srcs = ["fp16_compare.cc"], + hdrs = ["fp16_compare.h"], + deps = [ + "@com_google_googletest//:gtest", + ], +) + +cc_binary( + name = "fp16_compare_test", + testonly = True, + srcs = ["fp16_compare_test.cc"], + deps = [ + ":fp16_compare", + "@com_google_googletest//:gtest", + ], +)
diff --git a/common/test_util/fp16_compare.cc b/common/test_util/fp16_compare.cc new file mode 100644 index 0000000..db13a41 --- /dev/null +++ b/common/test_util/fp16_compare.cc
@@ -0,0 +1,117 @@ +/* + * Copyright 2025 The ChromiumOS Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "common/test_util/fp16_compare.h" + +#include <gmock/gmock-matchers.h> +#include <algorithm> +#include <bit> +#include <cmath> +#include <cstdint> +#include <ostream> +#include <vector> + +namespace { + +// Converts an integer from the sign-and-magnitude representation to +// the biased representation. More precisely, let N be 2 to the +// power of (kBitCount - 1), an integer x is represented by the +// unsigned number x + N. +// +// For instance, +// +// -N + 1 (the most negative number representable using +// sign-and-magnitude) is represented by 1; +// 0 is represented by N; and +// N - 1 (the biggest number representable using +// sign-and-magnitude) is represented by 2N - 1. +// +// Read https://en.wikipedia.org/wiki/Signed_number_representations +// for more details on signed number representations. +uint32_t SignAndMagnitudeToBiased(uint32_t sam) { + constexpr uint32_t kSignBitMask = 1u << 31; + if (kSignBitMask & sam) { + // sam represents a negative number. + return ~sam + 1; + } else { + // sam represents a positive number. + return kSignBitMask | sam; + } +} + +// Given two numbers in the sign-and-magnitude representation, +// returns the distance between them as an unsigned number. +uint32_t DistanceBetweenSignAndMagnitudeNumbers(uint32_t sam1, uint32_t sam2) { + uint32_t biased1 = SignAndMagnitudeToBiased(sam1); + uint32_t biased2 = SignAndMagnitudeToBiased(sam2); + return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); +} + +// Returns true if and only if lhs is at most max_ulps ULP's away from rhs. +// In particular, this function: +// +// - returns true if both numbers are NAN. +// - returns false if exact one of numbers is NAN. +// - treats really large numbers as almost equal to infinity. +// - thinks +0.0 and -0.0 are 0 ULP's apart. +bool AlmostEquals(float lhs, float rhs, uint32_t max_ulps) { + if (std::isnan(lhs) || std::isnan(rhs)) { + return std::isnan(lhs) && std::isnan(rhs); + } + + return DistanceBetweenSignAndMagnitudeNumbers(std::bit_cast<uint32_t>(lhs), + std::bit_cast<uint32_t>(rhs)) <= + max_ulps; +} + +class Fp16EqMatcher { + public: + using is_gtest_matcher = void; + + explicit Fp16EqMatcher(float expected) : expected(expected) {} + + // Compare the actual value and the expected value stored during + // construction. Omit the explanation opportunity and rely on + // Describe{Negation}To. + bool MatchAndExplain(float actual, std::ostream*) const { + // FP16 only has 10 bits precision while FP32 has 23 bits precision. Thus, + // to check if results of FP16 are almost equal, we could check the result + // is within 4 * 2^13 ULPs of FP32, which equals to 4 ULPs of FP16. + constexpr uint32_t fp16_ulps_in_fp32 = 4 * (1 << 13); + // The minimum exponent of FP16 is 2^-14, which means the minimum ULP of + // FP16 is 2^-24. Therefore, when expected is less than 2^-14, i.e. a + // subnormal FP16 number, the minimum ULP of FP16 should be used instead of + // ULP of FP32. + if (std::abs(expected) < 0x1p-14) { + return std::abs(actual - expected) <= 4 * 0x1p-24; + } + return AlmostEquals(actual, expected, fp16_ulps_in_fp32); + } + + // Describes the property of a value matching this matcher. + void DescribeTo(std::ostream* os) const { *os << "is " << expected; } + + // Describes the property of a value NOT matching this matcher. + void DescribeNegationTo(std::ostream* os) const { + *os << "is not " << expected; + } + + private: + const float expected; +}; + +} // namespace + +testing::Matcher<float> Fp16Eq(float expected) { + return Fp16EqMatcher(expected); +} + +std::vector<testing::Matcher<float>> ArrayFp16Eq(std::span<float> values) { + std::vector<testing::Matcher<float>> matchers; + matchers.reserve(values.size()); + std::ranges::transform(values, std::back_inserter(matchers), Fp16Eq); + return matchers; +}
diff --git a/common/test_util/fp16_compare.h b/common/test_util/fp16_compare.h new file mode 100644 index 0000000..d9a3413 --- /dev/null +++ b/common/test_util/fp16_compare.h
@@ -0,0 +1,24 @@ +/* + * Copyright 2025 The ChromiumOS Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef COMMON_TEST_UTIL_FP16_COMPARE_H_ +#define COMMON_TEST_UTIL_FP16_COMPARE_H_ + +#include <gmock/gmock-matchers.h> +#include <span> +#include <vector> + +// Similar to FloatEq and DoubleEq, this matcher use ULP(Units in the Last +// Place)-based comparison to compare the given floating number in the fp16 +// sense. In other words, this automatically picks a reasonable error bound +// based on the absolute value of the expected value. +testing::Matcher<float> Fp16Eq(float expected); + +// A helper function to help convert the provided values into a series of gmock +// matchers. +std::vector<testing::Matcher<float>> ArrayFp16Eq(std::span<float> values); + +#endif // COMMON_TEST_UTIL_FP16_COMPARE_H_
diff --git a/common/test_util/fp16_compare_test.cc b/common/test_util/fp16_compare_test.cc new file mode 100644 index 0000000..0916415 --- /dev/null +++ b/common/test_util/fp16_compare_test.cc
@@ -0,0 +1,45 @@ +/* + * Copyright 2025 The ChromiumOS Authors + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include <gtest/gtest.h> + +#include "common/test_util/fp16_compare.h" + +// Adapted from +// https://github.com/tensorflow/tensorflow/blob/b0b4ec040b9b8f4b7fefdb5d7c3695349dae1d9d/tensorflow/lite/kernels/test_util_test.cc#L124-L177 +TEST(Fp16CompareTest, Eq) { + // Minimum number that FP16 could represent. When the expected is a subnormal + // FP16 number, i.e. its exponent is the minimum, -14, this is the ULP used. + // Given minimum exponent is -14 and fraction has 10 bits, the true minimum + // of FP16 is 2^(-14-10) = 2^(-24). + constexpr float fp16_true_min = 0x1p-24; + + // FP16 has 10 bits for tha fraction part, so the ULP error is between + // 2^-10 / 2 and 2^-10 relative error. Since we emulate a FP16 ULP by 2^13 + // FP32 ULPs, rounding error is negligible. So the tolerated relative error + // of 4 ULPs is roughly between 4 * 2^-10 / 2 and 4 * 2^-10 ~= 0.195% and + // 0.39%. + // 0.15% relative error should be tolerated by 4 ULPs in FP16. + EXPECT_THAT(0.1f, Fp16Eq(0.10015)); + EXPECT_THAT(100.f, Fp16Eq(100.15)); + EXPECT_THAT(-1.f, Fp16Eq(-1.0015)); + EXPECT_THAT(0.f, Fp16Eq(4 * fp16_true_min)); + EXPECT_THAT(0.f, Fp16Eq(-4 * fp16_true_min)); + // NaN equals to NaN. + EXPECT_THAT(std::nanf(""), Fp16Eq(std::nanf(""))); + + // 0.4% relative error should not be tolerated by 4 ULPs in FP16. + EXPECT_THAT(0.1f, Not(Fp16Eq(0.1004))); + EXPECT_THAT(100.f, Not(Fp16Eq(100.4))); + EXPECT_THAT(-1.f, Not(Fp16Eq(-1.004))); + EXPECT_THAT(0.f, Not(Fp16Eq(5 * fp16_true_min))); + EXPECT_THAT(0.f, Not(Fp16Eq(-5 * fp16_true_min))); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}