commit ae81d71862a422deb1f5287a0b2bea701883dcdd
author Jordan Cook <jordan.cook@pioneer.com> Wed Feb 23 00:44:05 2022
committer Jordan Cook <jordan.cook@pioneer.com> Wed Feb 23 00:44:05 2022
tree 7fc1a2f087a4bb5623a2ca4893a189e55fdfca32
parent 98582645a595dd2472f6f14f15b772aff822393d

Skip normalizing a JSON request body if it's excessively large (>10MB) due to performance impact

HISTORY.md

diff --git a/HISTORY.md b/HISTORY.md
index 6b59cec..7df2cac 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -3,6 +3,7 @@
 ## 0.9.3 (Unreleased)
 * Fix handling BSON serializer differences between pymongo's `bson` and standalone `bson` codec.
 * Fix request normalization for request body with a list as a JSON root
+* Skip normalizing a JSON request body if it's excessively large (>10MB) due to performance impact
 
 ## 0.9.2 (2022-02-15)
 * Fix serialization in filesystem backend with binary content that is also valid UTF-8

requests_cache/cache_keys.py

diff --git a/requests_cache/cache_keys.py b/requests_cache/cache_keys.py
index 394d422..256e331 100644
--- a/requests_cache/cache_keys.py
+++ b/requests_cache/cache_keys.py
@@ -22,9 +22,13 @@
     from .models import AnyPreparedRequest, AnyRequest, CachedResponse
 
 __all__ = ['create_key', 'normalize_request']
+
 # Request headers that are always excluded from cache keys, but not redacted from cached responses
 DEFAULT_EXCLUDE_HEADERS = {'Cache-Control', 'If-None-Match', 'If-Modified-Since'}
 
+# Maximum JSON request body size that will be normalized
+MAX_NORM_BODY_SIZE = 10 * 1024 * 1025
+
 ParamList = Optional[Iterable[str]]
 RequestContent = Union[Mapping, str, bytes]
 
@@ -158,13 +162,12 @@
     return encode(filtered_body)
 
 
-# TODO: Skip this for a very large response body?
 def normalize_json_body(
     original_body: Union[str, bytes], ignored_parameters: ParamList
 ) -> Union[str, bytes]:
     """Normalize and filter a request body with serialized JSON data"""
 
-    if len(original_body) == 0:
+    if len(original_body) == 0 or len(original_body) > MAX_NORM_BODY_SIZE:
         return original_body
 
     try:

tests/unit/test_cache_keys.py

diff --git a/tests/unit/test_cache_keys.py b/tests/unit/test_cache_keys.py
index ce4e1fa..dd79bf6 100644
--- a/tests/unit/test_cache_keys.py
+++ b/tests/unit/test_cache_keys.py
@@ -1,10 +1,12 @@
 """The cache_keys module is mostly covered indirectly via other tests.
 This just contains tests for some extra edge cases not covered elsewhere.
 """
+import json
+
 import pytest
 from requests import PreparedRequest, Request
 
-from requests_cache.cache_keys import create_key, normalize_request
+from requests_cache.cache_keys import MAX_NORM_BODY_SIZE, create_key, normalize_request
 
 CACHE_KEY = 'e8cb526891875e37'
 
@@ -94,6 +96,19 @@
     assert normalize_request(request, ignored_parameters=['param']).body == request.data
 
 
+def test_normalize_request__ovsersized_body():
+    body = {'param': '1', 'content': '0' * MAX_NORM_BODY_SIZE}
+    encoded_body = json.dumps(body).encode('utf-8')
+
+    request = Request(
+        method='GET',
+        url='https://img.site.com/base/img.jpg',
+        json=body,
+        headers={'Content-Type': 'application/octet-stream'},
+    )
+    assert normalize_request(request, ignored_parameters=['param']).body == encoded_body
+
+
 def test_remove_ignored_headers__empty():
     request = PreparedRequest()
     request.prepare(