Impose the residual range to be valid for ANS

ALso add a last chance WebP in case we have configurations that
fail because the residuals are out of range.

Change-Id: Ia97eba447bfa49f0c7dc3fd393efac91f7742e67
Reviewed-on: https://chromium-review.googlesource.com/c/codecs/libwebp2/+/6184800
Tested-by: WebM Builds <builds@webmproject.org>
Reviewed-by: Maryla Ustarroz-Calonge <maryla@google.com>
diff --git a/src/common/symbols.cc b/src/common/symbols.cc
index 814595c..9db41fb 100644
--- a/src/common/symbols.cc
+++ b/src/common/symbols.cc
@@ -23,7 +23,6 @@
 #include <array>
 #include <cassert>
 #include <cstdint>
-#include <limits>
 
 #include "src/common/color_precision.h"
 #include "src/common/constants.h"
@@ -612,6 +611,15 @@
   Segment operator|(const Segment& other) const {
     return Segment{std::min(min, other.min), std::max(max, other.max)};
   }
+  // This is the only operation that can fail, if the two intervals do not
+  // intersect.
+  WP2_NO_DISCARD
+  bool IntersectAndUpdate(const Segment& other) {
+    if (other.min > max || other.max < min) return false;
+    min = std::max(min, other.min);
+    max = std::min(max, other.max);
+    return true;
+  }
   int32_t min, max;
 };
 }  // namespace WP2L
@@ -625,10 +633,11 @@
 }  // namespace WP2
 
 namespace WP2L {
-void GetARGBRanges(const std::array<TransformHeader,
-                                    kPossibleTransformCombinationSize>& headers,
-                   WP2SampleFormat format, int32_t minima_range[4],
-                   int32_t maxima_range[4], uint32_t num_transforms) {
+WP2Status GetARGBRanges(
+    const std::array<TransformHeader, kPossibleTransformCombinationSize>&
+        headers,
+    WP2SampleFormat format, int32_t minima_range[4], int32_t maxima_range[4],
+    uint32_t num_transforms) {
   Segment segments[4];
   // Define the original range according to the pixel format.
   const int32_t num_bits_alpha = ::WP2::kAlphaBits;
@@ -658,13 +667,14 @@
                           "invalid kMaxAbsResidual8 value");
             static_assert(2 * kMaxAbsResidual10 + 1 <= ::WP2::kANSMaxRange,
                           "invalid kMaxAbsResidual10 value");
-            segments[c] = Segment{std::max(segments[c].min, -abs_value_max),
-                                  std::min(segments[c].max, abs_value_max)};
+            if (!segments[c].IntersectAndUpdate(
+                    Segment{-abs_value_max, abs_value_max})) {
+              return WP2_STATUS_INVALID_PARAMETER;
+            }
             if (segments[c].min > segments[c].max) {
               // Should not happen.
               assert(false);
-              segments[c].min = segments[c].max =
-                  std::numeric_limits<int32_t>::max();
+              return WP2_STATUS_INVALID_PARAMETER;
             }
           }
         }
@@ -748,11 +758,20 @@
         assert(false);
     }
   }
+
+  // Make sure we fit in the ANS ranges [0,(1 << WP2::kANSMaxRangeBits)-1].
+  constexpr int32_t kMaxRange = 1 << WP2::kANSMaxRangeBits;
   for (uint32_t c = 0; c < 4; ++c) {
+    // Assign to the output.
+    if (!segments[c].IntersectAndUpdate(
+            Segment{-kMaxRange / 2, kMaxRange / 2 - 1})) {
+      return WP2_STATUS_INVALID_PARAMETER;
+    }
     minima_range[c] = segments[c].min;
     maxima_range[c] = segments[c].max;
     assert(minima_range[c] <= maxima_range[c]);
   }
+  return WP2_STATUS_OK;
 }
 
 }  // namespace WP2L
diff --git a/src/common/symbols.h b/src/common/symbols.h
index e60d0f5..ce80b3f 100644
--- a/src/common/symbols.h
+++ b/src/common/symbols.h
@@ -718,11 +718,11 @@
 // unused if no color indexing transform is present).
 // If 'transforms' is not one of the official sets of transforms you can specify
 // its length with 'num_transforms'.
-void GetARGBRanges(const std::array<TransformHeader,
-                                    kPossibleTransformCombinationSize>& headers,
-                   WP2SampleFormat format, int32_t minima_range[4],
-                   int32_t maxima_range[4],
-                   uint32_t num_transforms = kPossibleTransformCombinationSize);
+WP2Status GetARGBRanges(
+    const std::array<TransformHeader, kPossibleTransformCombinationSize>&
+        headers,
+    WP2SampleFormat format, int32_t minima_range[4], int32_t maxima_range[4],
+    uint32_t num_transforms = kPossibleTransformCombinationSize);
 
 }  // namespace WP2L
 
diff --git a/src/dec/lossless/losslessi_dec.cc b/src/dec/lossless/losslessi_dec.cc
index e5acd9c..e2a7784 100644
--- a/src/dec/lossless/losslessi_dec.cc
+++ b/src/dec/lossless/losslessi_dec.cc
@@ -292,8 +292,8 @@
       std::array<TransformHeader, kPossibleTransformCombinationSize> headers;
       for (int i = 0; i < ind; ++i) headers[i] = transforms_[i].header_;
 
-      GetARGBRanges(headers, hdr_.symbols_info_.SampleFormat(), minima_range,
-                    maxima_range, ind);
+      WP2_CHECK_STATUS(GetARGBRanges(headers, hdr_.symbols_info_.SampleFormat(),
+                                     minima_range, maxima_range, ind));
     }
     int16_t* predicted = nullptr;
 #ifndef WP2_REDUCE_BINARY_SIZE
@@ -1112,8 +1112,8 @@
       headers[t] = transforms_[t].header_;
     }
 
-    GetARGBRanges(headers, hdr_.symbols_info_.SampleFormat(), minima_range,
-                  maxima_range);
+    WP2_CHECK_STATUS(GetARGBRanges(headers, hdr_.symbols_info_.SampleFormat(),
+                                   minima_range, maxima_range));
     // Read the ARGB ranges. As the encoder tries to reduce the ranges of ARGB
     // values around zero, they are more efficiently stored with prefix coding.
     for (Symbol sym : {kSymbolA, kSymbolR, kSymbolG, kSymbolB}) {
diff --git a/src/enc/lossless/analysisl_enc.cc b/src/enc/lossless/analysisl_enc.cc
index 16fd7ef..01f4460 100644
--- a/src/enc/lossless/analysisl_enc.cc
+++ b/src/enc/lossless/analysisl_enc.cc
@@ -703,6 +703,13 @@
     }
     c.lz77s_types_to_try_size = n_lz77s;
   }
+
+  // Add a last chance config that minimizes the range of the residuals by
+  // having no transforms.
+  WP2_CHECK_ALLOC_OK(configs->push_back(configs->back()));
+  for (auto& t : configs->back().transforms) t.type = TransformType::kNum;
+  configs->back().is_skippable = true;
+
   return WP2_STATUS_OK;
 }
 
diff --git a/src/enc/lossless/losslessi_enc.cc b/src/enc/lossless/losslessi_enc.cc
index 96e9534..f112bcb 100644
--- a/src/enc/lossless/losslessi_enc.cc
+++ b/src/enc/lossless/losslessi_enc.cc
@@ -928,8 +928,9 @@
         case TransformType::kPredictorWSub:
           // Define the minimum/maximum values before applying the predictors.
           int32_t minima_range[4], maxima_range[4];
-          GetARGBRanges(config->transforms, symbols_info_.SampleFormat(),
-                        minima_range, maxima_range, /*num_transforms=*/i);
+          WP2_CHECK_STATUS(
+              GetARGBRanges(config->transforms, symbols_info_.SampleFormat(),
+                            minima_range, maxima_range, /*num_transforms=*/i));
           WP2_CHECK_STATUS(ApplyPredictor(minima_range, maxima_range,
                                           config->HasSubModes(), progress_local,
                                           config->transform_bits, enc, dicts));
@@ -1073,12 +1074,16 @@
     }
     // Find the ranges in which they should belong according to the transforms.
     int32_t minima_range[4], maxima_range[4];
-    GetARGBRanges(config->transforms, symbols_info->SampleFormat(),
-                  minima_range, maxima_range);
+    WP2_CHECK_STATUS(GetARGBRanges(config->transforms,
+                                   symbols_info->SampleFormat(), minima_range,
+                                   maxima_range));
     for (uint32_t c = 0; c < 4; ++c) {
-      assert(minima_range[c] <= minima[c]);
       assert(minima[c] <= maxima[c]);
-      assert(minima[c] <= maxima_range[c]);
+      if (minima[c] < minima_range[c] || maxima[c] > maxima_range[c]) {
+        // Invalidate the encoding because it does not fit within the range.
+        enc->Reset();
+        return WP2_STATUS_OK;
+      }
     }
     // Write the ARGB ranges.
     for (Symbol sym : {kSymbolA, kSymbolR, kSymbolG, kSymbolB}) {
@@ -1171,6 +1176,10 @@
     const CrunchConfig& config = crunch_configs[i_config];
     const ProgressRange config_progress(full_encode_progress,
                                         1. / crunch_configs.size());
+    if (config.is_skippable && best_cost < std::numeric_limits<float>::max()) {
+      WP2_CHECK_STATUS(config_progress.AdvanceBy(1.));
+      continue;
+    }
     const ProgressRange transforms_progress(config_progress, 0.5);
     const ProgressRange encode_progress(config_progress, 0.5);
 
@@ -1194,6 +1203,12 @@
         use_previous_transforms ? &crunch_configs[i_config - 1] : nullptr,
         cache_transforms, transforms_progress, &crunch_configs[i_config],
         &symbols_info, &enc, &dicts));
+    // If enc has been emptied, encoding could not be performed.
+    if (enc.NumTokens() == 0) {
+      use_previous_transforms = false;
+      WP2_CHECK_STATUS(progress.AdvanceBy(1.));
+      continue;
+    }
     use_previous_transforms = cache_transforms;
 
     // -------------------------------------------------------------------------
diff --git a/src/enc/lossless/losslessi_enc.h b/src/enc/lossless/losslessi_enc.h
index d09a17a..fe3fd35 100644
--- a/src/enc/lossless/losslessi_enc.h
+++ b/src/enc/lossless/losslessi_enc.h
@@ -263,6 +263,9 @@
   bool group4_use_move_to_front;
   // How to sort the palette.
   Palette::Sorting palette_sorting_type;
+  // Whether the config can be skipped if one has already been found to be
+  // valid.
+  bool is_skippable = false;
   // Returns whether one of the transforms is kPredictorWSub.
   bool HasSubModes() const {
     for (uint32_t i = 0; i < kPossibleTransformCombinationSize; ++i) {