vp8: Fix integer overflow in vp8_yv12_realloc_frame_buffer

test/test.mk

@@ -227,6 +227,7 @@ endif # VP9
 
 ## Multi-codec / unconditional whitebox tests.
 
+LIBVPX_TEST_SRCS-yes += yv12config_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_ENCODERS) += sad_test.cc
 ifneq (, $(filter yes, $(HAVE_NEON) $(HAVE_SSE2) $(HAVE_MSA)))
 LIBVPX_TEST_SRCS-$(CONFIG_ENCODERS) += sum_squares_test.cc

test/yv12config_test.cc

@@ -0,0 +1,120 @@
+/*
+ *  Copyright (c) 2026 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+// Tests for the YV12 frame buffer allocation helpers in vpx_scale. The frame
+// sizes here are computed independently in 64-bit so they cannot wrap.
+
+#include <stdint.h>
+
+#include <cstring>
+
+#include "gtest/gtest.h"
+#include "vpx_scale/yv12config.h"
+
+namespace {
+
+struct FrameLayout {
+  uint64_t frame_size;
+  uint64_t y_offset;
+  uint64_t u_offset;
+  uint64_t v_offset;
+};
+
+// The buffer layout vp8_yv12_realloc_frame_buffer() must produce, evaluated
+// in 64-bit as ground truth.
+FrameLayout ComputeLayout(int width, int height, int border) {
+  const uint64_t aligned_width = ((uint64_t)width + 15) & ~(uint64_t)15;
+  const uint64_t aligned_height = ((uint64_t)height + 15) & ~(uint64_t)15;
+  const uint64_t y_stride =
+      ((aligned_width + 2 * (uint64_t)border) + 31) & ~(uint64_t)31;
+  const uint64_t yplane_size = (aligned_height + 2 * (uint64_t)border) * y_stride;
+  const uint64_t uv_stride = y_stride >> 1;
+  const uint64_t uvplane_size =
+      (aligned_height / 2 + (uint64_t)border) * uv_stride;
+  FrameLayout layout;
+  layout.frame_size = yplane_size + 2 * uvplane_size;
+  layout.y_offset = (uint64_t)border * y_stride + (uint64_t)border;
+  layout.u_offset =
+      yplane_size + (uint64_t)(border / 2) * uv_stride + (uint64_t)(border / 2);
+  layout.v_offset = yplane_size + uvplane_size +
+                    (uint64_t)(border / 2) * uv_stride +
+                    (uint64_t)(border / 2);
+  return layout;
+}
+
+void ExpectExactLayout(const YV12_BUFFER_CONFIG &ybf, const FrameLayout &layout) {
+  EXPECT_EQ(layout.frame_size, (uint64_t)ybf.frame_size);
+  EXPECT_EQ(ybf.buffer_alloc + layout.y_offset, ybf.y_buffer);
+  EXPECT_EQ(ybf.buffer_alloc + layout.u_offset, ybf.u_buffer);
+  EXPECT_EQ(ybf.buffer_alloc + layout.v_offset, ybf.v_buffer);
+}
+
+// Normal frames must still allocate and report the exact frame size.
+TEST(YV12ConfigTest, Vp8NormalFrames) {
+  static const int kDims[][2] = { { 16, 16 }, { 352, 288 }, { 16383, 16383 } };
+  for (size_t i = 0; i < sizeof(kDims) / sizeof(kDims[0]); ++i) {
+    const int width = kDims[i][0];
+    const int height = kDims[i][1];
+    YV12_BUFFER_CONFIG ybf;
+    memset(&ybf, 0, sizeof(ybf));
+    ASSERT_EQ(0, vp8_yv12_alloc_frame_buffer(&ybf, width, height,
+                                             VP8BORDERINPIXELS));
+    ExpectExactLayout(ybf, ComputeLayout(width, height, VP8BORDERINPIXELS));
+    vp8_yv12_de_alloc_frame_buffer(&ybf);
+  }
+}
+
+// Dimensions whose true plane sizes overflow 32-bit arithmetic must never be
+// accepted with a wrapped (undersized) allocation: either the call fails, or
+// the buffer is the full 64-bit size.
+TEST(YV12ConfigTest, Vp8WrappingSizesNotUndersized) {
+  static const int kWrap[][3] = {
+    // width, height, border.
+    { 65536, 65536, 32 },    // yplane_size wraps to 8392704, true 4303360000.
+    { 46341, 46341, 32 },    // yplane_size wraps to -2139779584.
+    { 100000, 100000, 32 },  // yplane_size wraps to 1422869504.
+  };
+  for (size_t i = 0; i < sizeof(kWrap) / sizeof(kWrap[0]); ++i) {
+    const int width = kWrap[i][0];
+    const int height = kWrap[i][1];
+    const int border = kWrap[i][2];
+    YV12_BUFFER_CONFIG ybf;
+    memset(&ybf, 0, sizeof(ybf));
+    const int ret = vp8_yv12_alloc_frame_buffer(&ybf, width, height, border);
+    if (ret != 0) {
+      EXPECT_EQ(NULL, ybf.buffer_alloc);
+      continue;
+    }
+    ExpectExactLayout(ybf, ComputeLayout(width, height, border));
+    vp8_yv12_de_alloc_frame_buffer(&ybf);
+  }
+}
+
+// A huge border passes the "multiple of 32" rule but wraps the plane size
+// arithmetic to a tiny allocation; it must be rejected outright.
+TEST(YV12ConfigTest, Vp8OversizedBorderRejected) {
+  YV12_BUFFER_CONFIG ybf;
+  memset(&ybf, 0, sizeof(ybf));
+  EXPECT_NE(0, vp8_yv12_alloc_frame_buffer(&ybf, 32, 32, 1 << 29));
+  EXPECT_EQ(NULL, ybf.buffer_alloc);
+}
+
+// Negative dimensions and borders must be rejected.
+TEST(YV12ConfigTest, Vp8NegativeArgsRejected) {
+  YV12_BUFFER_CONFIG ybf;
+  memset(&ybf, 0, sizeof(ybf));
+  EXPECT_NE(0, vp8_yv12_alloc_frame_buffer(&ybf, -16, 16, 32));
+  EXPECT_NE(0, vp8_yv12_alloc_frame_buffer(&ybf, 16, -16, 32));
+  EXPECT_NE(0, vp8_yv12_alloc_frame_buffer(&ybf, 16, 16, -32));
+  EXPECT_NE(0, vp8_yv12_alloc_frame_buffer(&ybf, 16, 16, 33));
+  EXPECT_EQ(NULL, ybf.buffer_alloc);
+}
+
+}  // namespace

vpx_scale/generic/yv12config.c

@@ -50,21 +50,57 @@ int vp8_yv12_de_alloc_frame_buffer(YV12_BUFFER_CONFIG *ybf) {
 
 int vp8_yv12_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width,
                                   int height, int border) {
+#if CONFIG_SIZE_LIMIT
+  if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;
+#endif
+
+  /* Bound the dimensions and the border so the stride arithmetic below
+   * cannot overflow a signed int: with width, height, border <= INT_MAX / 4
+   * the largest int intermediate, aligned_width + 2 * border + 31, is
+   * bounded by 3 * (INT_MAX / 4) + 46 < INT_MAX (the plane sizes themselves
+   * are computed in 64-bit). */
+  if (width < 0 || height < 0 || width > INT_MAX / 4 || height > INT_MAX / 4)
+    return -1;
+
+  /* Only support allocating buffers that have a border that's a multiple
+   * of 32. The border restriction is required to get 16-byte alignment of
+   * the start of the chroma rows without introducing an arbitrary gap
+   * between planes, which would break the semantics of things like
+   * vpx_img_set_rect(). Negative borders are rejected here as well: they
+   * would place the plane pointers outside the buffer. */
+  if (border < 0 || (border & 0x1f) || border > INT_MAX / 4) return -3;
+
   if (ybf) {
     int aligned_width = (width + 15) & ~15;
     int aligned_height = (height + 15) & ~15;
     int y_stride = ((aligned_width + 2 * border) + 31) & ~31;
-    int yplane_size = (aligned_height + 2 * border) * y_stride;
+    const uint64_t yplane_size =
+        (aligned_height + 2 * border) * (uint64_t)y_stride;
     int uv_width = aligned_width >> 1;
     int uv_height = aligned_height >> 1;
     /** There is currently a bunch of code which assumes
      *  uv_stride == y_stride/2, so enforce this here. */
     int uv_stride = y_stride >> 1;
-    int uvplane_size = (uv_height + border) * uv_stride;
-    const size_t frame_size = yplane_size + 2 * uvplane_size;
+    const uint64_t uvplane_size = (uv_height + border) * (uint64_t)uv_stride;
+    const uint64_t frame_size = yplane_size + 2 * uvplane_size;
+
+#if defined(VPX_MAX_ALLOCABLE_MEMORY)
+    // The decoder may allocate REF_FRAMES frame buffers in the frame buffer
+    // pool. Bound the total amount of allocated memory as if these REF_FRAMES
+    // frame buffers were allocated in a single allocation.
+    if (frame_size > VPX_MAX_ALLOCABLE_MEMORY / REF_FRAMES) return -1;
+#endif  // VPX_MAX_ALLOCABLE_MEMORY
+
+#if UINT64_MAX > SIZE_MAX
+    // frame_size is stored in buffer_alloc_sz, which is a size_t. If it won't
+    // fit, fail early.
+    if (frame_size > SIZE_MAX) {
+      return -1;
+    }
+#endif
 
     if (!ybf->buffer_alloc) {
-      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size);
+      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
       if (!ybf->buffer_alloc) {
         ybf->buffer_alloc_sz = 0;
         return -1;
@@ -74,21 +110,14 @@ int vp8_yv12_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width,
       // This memset is needed for fixing the issue of using uninitialized
       // value in msan test. It will cause a perf loss, so only do this for
       // msan test.
-      memset(ybf->buffer_alloc, 0, frame_size);
+      memset(ybf->buffer_alloc, 0, (size_t)frame_size);
 #endif
 #endif
-      ybf->buffer_alloc_sz = frame_size;
+      ybf->buffer_alloc_sz = (size_t)frame_size;
     }
 
     if (ybf->buffer_alloc_sz < frame_size) return -1;
 
-    /* Only support allocating buffers that have a border that's a multiple
-     * of 32. The border restriction is required to get 16-byte alignment of
-     * the start of the chroma rows without introducing an arbitrary gap
-     * between planes, which would break the semantics of things like
-     * vpx_img_set_rect(). */
-    if (border & 0x1f) return -3;
-
     ybf->y_crop_width = width;
     ybf->y_crop_height = height;
     ybf->y_width = aligned_width;
@@ -106,13 +135,17 @@ int vp8_yv12_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width,
     ybf->alpha_stride = 0;
 
     ybf->border = border;
-    ybf->frame_size = frame_size;
+    ybf->frame_size = (size_t)frame_size;
 
-    ybf->y_buffer = ybf->buffer_alloc + (border * y_stride) + border;
+    ybf->y_buffer =
+        ybf->buffer_alloc + (size_t)((uint64_t)border * y_stride + border);
     ybf->u_buffer =
-        ybf->buffer_alloc + yplane_size + (border / 2 * uv_stride) + border / 2;
-    ybf->v_buffer = ybf->buffer_alloc + yplane_size + uvplane_size +
-                    (border / 2 * uv_stride) + border / 2;
+        ybf->buffer_alloc +
+        (size_t)(yplane_size + (uint64_t)(border / 2) * uv_stride + border / 2);
+    ybf->v_buffer =
+        ybf->buffer_alloc +
+        (size_t)(yplane_size + uvplane_size + (uint64_t)(border / 2) * uv_stride +
+                 border / 2);
     ybf->alpha_buffer = NULL;
 
     ybf->corrupted = 0; /* assume not currupted by errors */