write_superframe_index: return 0 if buffer is full

write_superframe_index() should return the number of bytes written to
ctx->pending_cx_data. If ctx->pending_cx_data is full,
write_superframe_index() doesn't write the optional superframe index, so
it should return 0 in this case. Add an assertion that would have
detected this bug. Add and clarify comments for code related to this
bug.

Also fix the buffer full check. The check should not assume that
ctx->pending_cx_data is equal to ctx->cx_data, and the check had an
off-by-one error.

The bug was introduced when write_superframe_index() was added in the
following CLs:
https://chromium-review.googlesource.com/c/webm/libvpx/+/44659
https://chromium-review.googlesource.com/c/webm/libvpx/+/45268

Bug: oss-fuzz:476466137
Change-Id: Ie113568cf25acc73f8af640a3c51cfdb5b900613

vp9/vp9_cx_iface.c

diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c
index 83f45b0..ab9c582 100644
--- a/vp9/vp9_cx_iface.c
+++ b/vp9/vp9_cx_iface.c
@@ -8,7 +8,9 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <assert.h>
 #include <limits.h>
+#include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
@@ -122,6 +124,7 @@
   VP9_COMP *cpi;
   unsigned char *cx_data;
   size_t cx_data_sz;
+  // pending_cx_data either is a null pointer or points into the cx_data buffer.
   unsigned char *pending_cx_data;
   size_t pending_cx_data_sz;
   int pending_frame_count;
@@ -1253,8 +1256,12 @@
 
   // Write the index
   index_sz = 2 + (mag + 1) * ctx->pending_frame_count;
-  if (ctx->pending_cx_data_sz + index_sz < ctx->cx_data_sz) {
-    uint8_t *x = ctx->pending_cx_data + ctx->pending_cx_data_sz;
+  unsigned char *cx_data_end = ctx->cx_data + ctx->cx_data_sz;
+  unsigned char *pending_cx_data_end =
+      ctx->pending_cx_data + ctx->pending_cx_data_sz;
+  ptrdiff_t space_remaining = cx_data_end - pending_cx_data_end;
+  if (index_sz <= space_remaining) {
+    uint8_t *x = pending_cx_data_end;
     int i, j;
 #ifdef TEST_SUPPLEMENTAL_SUPERFRAME_DATA
     uint8_t marker_test = 0xc0;
@@ -1285,6 +1292,8 @@
 #ifdef TEST_SUPPLEMENTAL_SUPERFRAME_DATA
     index_sz += index_sz_test;
 #endif
+  } else {
+    index_sz = 0;
   }
   return index_sz;
 }
@@ -1613,9 +1622,12 @@
               ctx->pending_frame_sizes[ctx->pending_frame_count++] = size;
             ctx->pending_frame_magnitude |= size;
             ctx->pending_cx_data_sz += size;
-            // write the superframe only for the case when
-            if (!ctx->output_cx_pkt_cb.output_cx_pkt)
+            // write the superframe only for the case when the callback function
+            // for getting per-layer packets is not registered.
+            if (!ctx->output_cx_pkt_cb.output_cx_pkt) {
               size += write_superframe_index(ctx);
+              assert(size <= cx_data_sz);
+            }
             pkt.data.frame.buf = ctx->pending_cx_data;
             pkt.data.frame.sz = ctx->pending_cx_data_sz;
             ctx->pending_cx_data = NULL;