commit 1344a2e947c749d231141a295327e5b99b444d63
author Pascal Massimino <skal@google.com> Fri Jun 29 17:15:47 2018
committer Pascal Massimino <skal@google.com> Fri Jun 29 18:02:17 2018
tree 42fa970ba495b435c9238b1fee2eebb14feacd51
parent be738c6d396fa5a272c1b209be4379a7532debfe

fix alpha-filtering crash when image width is larger than radius

(we also limit radius based on height too, for good measure, although it's not an asan bug)

fixes oss-fuzz issue #9105

Change-Id: Ie0d79dd81480dc4e2b653b7e992e5cdcd3dfa834

src/utils/quant_levels_dec_utils.c

diff --git a/src/utils/quant_levels_dec_utils.c b/src/utils/quant_levels_dec_utils.c
index 3818a78..f65b6cd 100644
--- a/src/utils/quant_levels_dec_utils.c
+++ b/src/utils/quant_levels_dec_utils.c
@@ -261,9 +261,15 @@
 
 int WebPDequantizeLevels(uint8_t* const data, int width, int height, int stride,
                          int strength) {
-  const int radius = 4 * strength / 100;
+  int radius = 4 * strength / 100;
+
   if (strength < 0 || strength > 100) return 0;
   if (data == NULL || width <= 0 || height <= 0) return 0;  // bad params
+
+  // limit the filter size to not exceed the image dimensions
+  if (2 * radius + 1 > width) radius = (width - 1) >> 1;
+  if (2 * radius + 1 > height) radius = (height - 1) >> 1;
+
   if (radius > 0) {
     SmoothParams p;
     memset(&p, 0, sizeof(p));