WIP: implement hashed_parameters
diff --git a/docs/user_guide/matching.md b/docs/user_guide/matching.md index c495d14..e3291a4 100644 --- a/docs/user_guide/matching.md +++ b/docs/user_guide/matching.md
@@ -82,6 +82,41 @@ If you want to match _all_ request headers, you can use `match_headers=True`. +(hashed-params)= +## Hashed Parameter Matching +In some cases, you may need to cache responses separately per user without storing +plaintext credentials in the cache. For example, a multi-user application calling +`GET /api/current_user` where the response varies only by the `Authorization` header. + +The `hashed_parameters` option solves this by **hashing** parameter values for cache key +differentiation while **redacting** them in stored responses: +```python +>>> session = CachedSession(hashed_parameters=['Authorization']) +>>> # These two requests are cached separately +>>> session.get('https://api.example.com/me', headers={'Authorization': 'Bearer user-1-token'}) +>>> session.get('https://api.example.com/me', headers={'Authorization': 'Bearer user-2-token'}) +>>> # This is a cache hit and returns the response for user 1 +>>> r = session.get('https://api.example.com/me', headers={'Authorization': 'Bearer user-1-token'}) +>>> assert r.from_cache is True +>>> # The token itself is not stored in the cache +>>> assert r.request.headers['Authorization'] == 'REDACTED' +``` + +```{note} +`hashed_parameters` takes precedence over `ignored_parameters`. Since `Authorization` is in +`ignored_parameters` by default, you only need to set `hashed_parameters=['Authorization']`; there +is no need to also remove it from `ignored_parameters`. +``` + +This works for any request parameter or header. For example, you could also use it with URL +parameters: +```python +>>> session = CachedSession(hashed_parameters=['api_key']) +>>> session.get('https://api.example.com/data', params={'api_key': 'key-1'}) +>>> session.get('https://api.example.com/data', params={'api_key': 'key-2'}) +``` + + (custom-matching)= ## Custom Request Matching If you need more advanced behavior, you can implement your own custom request matching.
diff --git a/requests_cache/backends/base.py b/requests_cache/backends/base.py index b302d15..2b7bab2 100644 --- a/requests_cache/backends/base.py +++ b/requests_cache/backends/base.py
@@ -88,7 +88,9 @@ """ cache_key = cache_key or self.create_key(response.request) cached_response = CachedResponse.from_response(response, expires=expires) - cached_response = redact_response(cached_response, self._settings.ignored_parameters) + cached_response = redact_response( + cached_response, self._settings.ignored_parameters, self._settings.hashed_parameters + ) self.responses[cache_key] = cached_response # Save redirect aliases, unless this is a revalidation (i.e., it was saved previously) @@ -121,6 +123,7 @@ ignored_parameters=self._settings.ignored_parameters, content_root_key=self._settings.content_root_key, match_headers=match_headers or self._settings.match_headers, + hashed_parameters=self._settings.hashed_parameters, serializer=self.responses.serializer, **kwargs, )
diff --git a/requests_cache/cache_keys.py b/requests_cache/cache_keys.py index 4ff408f..9f717ca 100644 --- a/requests_cache/cache_keys.py +++ b/requests_cache/cache_keys.py
@@ -58,6 +58,7 @@ match_headers: Union[ParamList, bool] = False, serializer: Any = None, content_root_key: Optional[str] = None, + hashed_parameters: ParamList = None, **request_kwargs, ) -> str: """Create a normalized cache key based on a request object @@ -66,11 +67,20 @@ request: Request object to generate a cache key from ignored_parameters: Request parameters, headers, and/or JSON body params to exclude match_headers: Match only the specified headers, or ``True`` to match all headers - request_kwargs: Additional keyword arguments for :py:func:`~requests.request` + serializer: Serializer name or instance content_root_key: root element in the request body to apply ignored_parameters to + hashed_parameters: Request parameters and/or headers whose values should be hashed + instead of redacted, for cache key differentiation + request_kwargs: Additional keyword arguments for :py:func:`~requests.request` """ + # Auto-include hashed_parameters in match_headers so they contribute to the cache key + if hashed_parameters and match_headers is not True: + existing = list(match_headers) if match_headers else [] + merged = {h.lower() for h in existing} | {h.lower() for h in hashed_parameters} + match_headers = sorted(merged) + # Normalize and gather all relevant request info to match against - request = normalize_request(request, ignored_parameters, content_root_key) + request = normalize_request(request, ignored_parameters, content_root_key, hashed_parameters) key_parts = [ request.method or '', request.url, @@ -114,6 +124,7 @@ request: AnyRequest, ignored_parameters: ParamList = None, content_root_key: Optional[str] = None, + hashed_parameters: ParamList = None, ) -> AnyPreparedRequest: """Normalize and remove ignored parameters from request URL, body, and headers. This is used for both: @@ -125,6 +136,8 @@ request: Request object to normalize ignored_parameters: Request parameters, headers, and/or JSON body params to exclude content_root_key: root element in the request body to apply ignored_parameters to + hashed_parameters: Request parameters and/or headers whose values should be hashed + instead of redacted, for cache key differentiation """ if isinstance(request, Request): # For a multipart POST request that hasn't been prepared, we need to patch the form boundary @@ -136,18 +149,22 @@ norm_request.method = (norm_request.method or '').upper() norm_request.url = normalize_url(norm_request.url or '', ignored_parameters) - norm_request.headers = normalize_headers(norm_request.headers, ignored_parameters) + norm_request.headers = normalize_headers( + norm_request.headers, ignored_parameters, hashed_parameters + ) norm_request.body = normalize_body(norm_request, ignored_parameters, content_root_key) return norm_request def normalize_headers( - headers: MutableMapping[str, str], ignored_parameters: ParamList = None + headers: MutableMapping[str, str], + ignored_parameters: ParamList = None, + hashed_parameters: ParamList = None, ) -> CaseInsensitiveDict: """Sort and filter request headers, and normalize minor variations in multi-value headers""" headers = {k: decode(v) for (k, v) in headers.items()} - if ignored_parameters: - headers = filter_sort_dict(headers, ignored_parameters) + if ignored_parameters or hashed_parameters: + headers = filter_sort_dict(headers, ignored_parameters, hashed_parameters) for k, v in headers.items(): if ',' in v: values = [v.strip() for v in v.lower().split(',') if v.strip()] @@ -234,18 +251,27 @@ return query_str -def redact_response(response: CachedResponse, ignored_parameters: ParamList) -> CachedResponse: - """Redact any ignored parameters (potentially containing sensitive info) from a cached request""" - if ignored_parameters: - response.url = filter_url(response.url, ignored_parameters) - response.request.url = filter_url(response.request.url, ignored_parameters) - response.headers = CaseInsensitiveDict( - filter_sort_dict(response.headers, ignored_parameters) - ) +def redact_response( + response: CachedResponse, + ignored_parameters: ParamList, + hashed_parameters: ParamList = None, +) -> CachedResponse: + """Redact any ignored parameters (potentially containing sensitive info) from a cached request. + + Both ``ignored_parameters`` and ``hashed_parameters`` are redacted in stored responses. + ``hashed_parameters`` are only hashed during cache key generation, not in stored data. + """ + # Merge hashed_parameters into ignored set for redaction + all_redacted = set(ignored_parameters or []) | set(hashed_parameters or []) + if all_redacted: + redact_list = sorted(all_redacted) + response.url = filter_url(response.url, redact_list) + response.request.url = filter_url(response.request.url, redact_list) + response.headers = CaseInsensitiveDict(filter_sort_dict(response.headers, redact_list)) response.request.headers = CaseInsensitiveDict( - filter_sort_dict(response.request.headers, ignored_parameters) + filter_sort_dict(response.request.headers, redact_list) ) - response.request.body = normalize_body(response.request, ignored_parameters) + response.request.body = normalize_body(response.request, redact_list) return response @@ -257,17 +283,46 @@ def filter_sort_dict( - data: Mapping[str, str], ignored_parameters: ParamList = None + data: Mapping[str, str], + ignored_parameters: ParamList = None, + hashed_parameters: ParamList = None, ) -> Dict[str, str]: # Note: Any ignored_parameters present will have their values replaced instead of removing the # parameter, so the cache key will still match whether the parameter was present or not. + # hashed_parameters take precedence: values are replaced with a SHA-256 hash for cache key + # differentiation, while still being redacted in stored responses (handled by redact_response). ignored_parameters = set(ignored_parameters or []) - return {k: ('REDACTED' if k in ignored_parameters else v) for k, v in sorted(data.items())} + hashed_parameters = set(hashed_parameters or []) + # hashed_parameters win over ignored_parameters + ignored_parameters -= hashed_parameters + + def _transform_value(k: str, v: str) -> str: + if k in hashed_parameters: + return sha256(encode(v)).hexdigest() + if k in ignored_parameters: + return 'REDACTED' + return v + + return {k: _transform_value(k, v) for k, v in sorted(data.items())} -def filter_sort_multidict(data: KVList, ignored_parameters: ParamList = None) -> KVList: +def filter_sort_multidict( + data: KVList, + ignored_parameters: ParamList = None, + hashed_parameters: ParamList = None, +) -> KVList: ignored_parameters = set(ignored_parameters or []) - return [(k, 'REDACTED' if k in ignored_parameters else v) for k, v in sorted(data)] + hashed_parameters = set(hashed_parameters or []) + ignored_parameters -= hashed_parameters + + def _transform_value(k: str, v: str) -> str: + if k in hashed_parameters: + return sha256(encode(v)).hexdigest() + if k in ignored_parameters: + return 'REDACTED' + return v + + return [(k, _transform_value(k, v)) for k, v in sorted(data)] def filter_sort_list(data: List, ignored_parameters: ParamList = None) -> List:
diff --git a/requests_cache/policy/settings.py b/requests_cache/policy/settings.py index 71c48c6..222d561 100644 --- a/requests_cache/policy/settings.py +++ b/requests_cache/policy/settings.py
@@ -32,6 +32,7 @@ disabled: bool = field(default=False) expire_after: ExpirationTime = field(default=None) filter_fn: FilterCallback = field(default=None) + hashed_parameters: Iterable[str] = field(factory=tuple) ignored_parameters: Iterable[str] = field(default=DEFAULT_IGNORED_PARAMS) key_fn: KeyCallback = field(default=None) match_headers: Union[Iterable[str], bool] = field(default=False)
diff --git a/requests_cache/session.py b/requests_cache/session.py index 050baac..132eb0a 100644 --- a/requests_cache/session.py +++ b/requests_cache/session.py
@@ -53,6 +53,7 @@ allowable_codes: Iterable[int] = DEFAULT_STATUS_CODES, allowable_methods: Iterable[str] = DEFAULT_METHODS, always_revalidate: bool = False, + hashed_parameters: Iterable[str] = (), ignored_parameters: Iterable[str] = DEFAULT_IGNORED_PARAMS, match_headers: Union[Iterable[str], bool] = False, filter_fn: Optional[FilterCallback] = None, @@ -71,6 +72,7 @@ allowable_codes=allowable_codes, allowable_methods=allowable_methods, always_revalidate=always_revalidate, + hashed_parameters=hashed_parameters, ignored_parameters=ignored_parameters, match_headers=match_headers, filter_fn=filter_fn, @@ -380,6 +382,9 @@ is not expired match_headers: Request headers to match, when `Vary` response header is not available. May be a list of headers, or ``True`` to match all. + hashed_parameters: Request parameters and/or headers whose values should be hashed for + cache key differentiation but redacted in stored responses. Useful for per-user caching + without storing plaintext credentials. Takes precedence over ``ignored_parameters``. ignored_parameters: Request parameters, headers, and/or JSON body params to exclude from both request matching and cached request data read_only: Read existing cached responses, but do not write any new responses to the cache
diff --git a/tests/unit/test_cache_keys.py b/tests/unit/test_cache_keys.py index eec2c63..c426353 100644 --- a/tests/unit/test_cache_keys.py +++ b/tests/unit/test_cache_keys.py
@@ -13,6 +13,7 @@ from requests_cache.cache_keys import ( MAX_NORM_BODY_SIZE, create_key, + filter_sort_dict, normalize_headers, normalize_request, redact_response, @@ -262,3 +263,109 @@ headers={'foo': 'bar'}, ) assert normalize_request(request.prepare(), ignored_parameters=None).headers == request.headers + + +def test_create_key__hashed_parameters__same_value(): + """Requests with the same hashed parameter value should get the same cache key""" + request_1 = Request( + method='GET', + url='https://example.com/api/me', + headers={'Authorization': 'Bearer same-token'}, + ) + request_2 = Request( + method='GET', + url='https://example.com/api/me', + headers={'Authorization': 'Bearer same-token'}, + ) + key_1 = create_key(request_1, hashed_parameters=['Authorization']) + key_2 = create_key(request_2, hashed_parameters=['Authorization']) + assert key_1 == key_2 + + +@pytest.mark.parametrize( + 'kwargs, expect_different', + [ + pytest.param( + {'hashed_parameters': ['Authorization']}, + True, + id='hashed', + ), + pytest.param( + {'hashed_parameters': ['Authorization'], 'ignored_parameters': ['Authorization']}, + True, + id='hashed_precedence_over_ignored', + ), + pytest.param( + {'hashed_parameters': ['Authorization'], 'match_headers': ['Accept-Language']}, + True, + id='hashed_with_explicit_match_headers', + ), + pytest.param( + {'ignored_parameters': ['Authorization']}, + False, + id='ignored_only', + ), + ], +) +def test_create_key__hashed_parameters(kwargs, expect_different): + """With different Authorization values, cache keys should differ when hashed_parameters + is set, and match when only ignored_parameters is set (control case).""" + request_1 = Request( + method='GET', + url='https://example.com/api/me', + headers={'Authorization': 'Bearer token-1', 'Accept-Language': 'en'}, + ) + request_2 = Request( + method='GET', + url='https://example.com/api/me', + headers={'Authorization': 'Bearer token-2', 'Accept-Language': 'en'}, + ) + key_1 = create_key(request_1, **kwargs) + key_2 = create_key(request_2, **kwargs) + assert (key_1 != key_2) == expect_different + + +@pytest.mark.parametrize( + 'ignored_parameters', + [ + pytest.param(None, id='hashed_only'), + pytest.param(['Authorization'], id='hashed_over_ignored'), + ], +) +def test_filter_sort_dict__hashed_parameters(ignored_parameters): + """filter_sort_dict should hash values for hashed_parameters, taking precedence + over ignored_parameters when both are specified.""" + from hashlib import sha256 + + data = {'Authorization': 'Bearer my-token', 'Accept': 'application/json'} + result = filter_sort_dict( + data, ignored_parameters=ignored_parameters, hashed_parameters=['Authorization'] + ) + assert result['Authorization'] == sha256(b'Bearer my-token').hexdigest() + assert result['Accept'] == 'application/json' + + +def test_redact_response__hashed_parameters(): + """Both hashed_parameters and ignored_parameters should be redacted in stored responses""" + url = 'https://example.com/api/me' + request = Request( + method='GET', + url=url, + headers={ + 'Authorization': 'Bearer secret-token', + 'X-API-KEY': 'my-api-key', + }, + ).prepare() + response = Response() + response.url = url + response.request = request + response.headers = {} + response.raw = HTTPResponse(request_url=url) + + redacted = redact_response( + response, + ignored_parameters=['X-API-KEY'], + hashed_parameters=['Authorization'], + ) + assert redacted.request.headers['Authorization'] == 'REDACTED' + assert redacted.request.headers['X-API-KEY'] == 'REDACTED'
diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 1824c9f..38e6e02 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py
@@ -657,6 +657,21 @@ assert response.request.headers['Authorization'] == 'REDACTED' +def test_hashed_parameters(mock_session): + """hashed_parameters should produce per-user cache keys (taking precedence over + Authorization being in default ignored_parameters) and redact values in stored responses""" + mock_session.settings.hashed_parameters = ['Authorization'] + + response_1 = mock_session.get(MOCKED_URL, headers={'Authorization': 'Bearer user-1-token'}) + response_2 = mock_session.get(MOCKED_URL, headers={'Authorization': 'Bearer user-2-token'}) + response_3 = mock_session.get(MOCKED_URL, headers={'Authorization': 'Bearer user-1-token'}) + + assert response_1.from_cache is False + assert response_2.from_cache is False # Different auth -> different cache entry + assert response_3.from_cache is True # Same auth as response_1 -> cache hit + assert response_3.request.headers['Authorization'] == 'REDACTED' + + @patch_normalize_url def test_filter_fn(mock_normalize_url, mock_session): mock_session.settings.filter_fn = lambda r: r.request.url != MOCKED_URL_JSON