Avoid OOM during input prasing by limiting the object size reservation.

PiperOrigin-RevId: 624202250

fuzztest/internal/serialization.cc

diff --git a/fuzztest/internal/serialization.cc b/fuzztest/internal/serialization.cc
index e6c50ae..fe4fae7 100644
--- a/fuzztest/internal/serialization.cc
+++ b/fuzztest/internal/serialization.cc
@@ -285,6 +285,8 @@
       uint64_t vec_size;
       std::memcpy(&vec_size, buf.str, sizeof(vec_size));
       buf.Advance(sizeof(vec_size));
+      // This could happen for malformed inputs.
+      if (vec_size > buf.size) return false;
       auto& v = obj.value.emplace<std::vector<IRObject>>();
       v.reserve(vec_size);
       for (uint64_t i = 0; i < vec_size; ++i) {

fuzztest/internal/serialization_test.cc

diff --git a/fuzztest/internal/serialization_test.cc b/fuzztest/internal/serialization_test.cc
index 17e7da8..c84da1d 100644
--- a/fuzztest/internal/serialization_test.cc
+++ b/fuzztest/internal/serialization_test.cc
@@ -325,6 +325,18 @@
               Optional(SubsAre(ValueIs<uint64_t>(0))));
 }
 
+TEST(BinaryFormatSerializerTest, MalformedObjectSizeIsRejectedWithoutOOM) {
+  static constexpr char kGoodInput[] =
+      "FUZZTESTv1b\x04\x01\x00\x00\x00\x00\x00\x00\x00\x00";
+  static constexpr char kBadInput[] =
+      "FUZZTESTv1b\x04\xff\xff\xff\xff\xff\xff\xff\xff\x00";
+  EXPECT_THAT(IRObject::FromString({kGoodInput, sizeof(kGoodInput) - 1}),
+              Optional(SubsAre(ValueIs<std::monostate>({}))));
+  // Expect grace failure instead of OOM error.
+  EXPECT_EQ(IRObject::FromString({kBadInput, sizeof(kBadInput) - 1}),
+            std::nullopt);
+}
+
 TEST(IRToCorpus, SpecializationIsBackwardCompatible) {
   EXPECT_THAT(
       (IRObject{std::vector<IRObject>{IRObject{1}, IRObject{2}, IRObject{3}}}