blob: aeff1e1670f9c33c93ca8dcd99df40a51ce48274 [file] [edit]
"""Fixtures that will be automatically picked up by pytest
Short description:
* The ``mock_session`` fixture uses pre-configured mock requests, and should be used for unit tests.
* The ``tempfile_session`` fixture makes real HTTP requests, and should be used for integration tests.
Note: The protocol ``http(s)+mock://`` helps :py:class:`requests_mock.Adapter` play nicely with
:py:class:`requests.PreparedRequest`. More info here:
https://requests-mock.readthedocs.io/en/latest/adapter.html
"""
import os
import platform
import threading
import warnings
from contextlib import contextmanager, nullcontext
from datetime import datetime, timedelta, timezone
from functools import wraps
from importlib import import_module
from logging import basicConfig, getLogger
from pathlib import Path
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
import requests
from requests import Request
from requests_mock import ANY as ANY_METHOD
from requests_mock import Adapter
from rich.logging import RichHandler
from requests_cache import ALL_METHODS, CachedSession, install_cache, uninstall_cache, utcnow
# ignore missing time-travel library on PyPy
try:
from time_machine import travel as time_travel
except ImportError:
time_travel = nullcontext
# Configure logging to show log output when tests fail (or with pytest -s)
basicConfig(
level='INFO',
format='%(message)s',
datefmt='[%m-%d %H:%M:%S]',
handlers=[RichHandler(rich_tracebacks=True, markup=True)],
)
# getLogger('requests_cache').setLevel('DEBUG')
logger = getLogger(__name__)
# Allow running longer stress tests with an environment variable
STRESS_TEST_MULTIPLIER = int(os.getenv('STRESS_TEST_MULTIPLIER', '1'))
N_WORKERS = 5 * STRESS_TEST_MULTIPLIER
N_ITERATIONS = 4 * STRESS_TEST_MULTIPLIER
N_REQUESTS_PER_ITERATION = 10 + 10 * STRESS_TEST_MULTIPLIER
HTTPBIN_DEFAULT = 'http://localhost:8080'
HTTPBIN_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE']
HTTPBIN_FORMATS = [
'deflate',
'deny',
'encoding/utf8',
'gzip',
'html',
'image/jpeg',
'image/png',
'image/svg',
'image/webp',
'json',
'robots.txt',
'xml',
]
HTTPDATE_STR = 'Fri, 16 APR 2021 21:13:00 GMT'
HTTPDATE_DATETIME = datetime(2021, 4, 16, 21, 13, tzinfo=timezone.utc)
EXPIRED_DT = utcnow() - timedelta(1)
ETAG = '"644b5b0155e6404a9cc4bd9d8b1ae730"'
LAST_MODIFIED = 'Thu, 05 Jul 2012 15:31:30 GMT'
START_DT = utcnow()
YESTERDAY = utcnow() - timedelta(days=1)
MOCKED_URL = 'http+mock://requests-cache.com/text'
MOCKED_URL_ETAG = 'http+mock://requests-cache.com/etag'
MOCKED_URL_HTTPS = 'https+mock://requests-cache.com/text'
MOCKED_URL_JSON = 'http+mock://requests-cache.com/json'
MOCKED_URL_JSON_LIST = 'http+mock://requests-cache.com/json_list'
MOCKED_URL_REDIRECT = 'http+mock://requests-cache.com/redirect'
MOCKED_URL_REDIRECT_TARGET = 'http+mock://requests-cache.com/redirect_target'
MOCKED_URL_VARY = 'http+mock://requests-cache.com/vary'
MOCKED_URL_VARY_REDIRECT = 'http+mock://requests-cache.com/vary-redirect'
MOCKED_URL_VARY_REDIRECT_TARGET = 'http+mock://requests-cache.com/vary-redirect-target'
MOCKED_URL_404 = 'http+mock://requests-cache.com/nonexistent'
MOCKED_URL_500 = 'http+mock://requests-cache.com/answer?q=this-statement-is-false'
MOCKED_URL_200_404 = 'http+mock://requests-cache.com/200-404'
MOCKED_URL_ETAG_200_404 = 'http+mock://requests-cache.com/etag-200-404'
MOCK_PROTOCOLS = ['mock://', 'http+mock://', 'https+mock://']
CACHE_NAME = 'pytest_cache'
PROJECT_DIR = Path(__file__).parent.parent.absolute()
SAMPLE_DATA_DIR = PROJECT_DIR / 'tests' / 'sample_data'
SAMPLE_CACHE_FILES = list(SAMPLE_DATA_DIR.glob('sample.db.*'))
def httpbin(path):
"""Get the url for either a local or remote httpbin instance"""
base_url = os.getenv('HTTPBIN_URL', HTTPBIN_DEFAULT).rstrip('/')
return f'{base_url}/{path}'
USE_PYTEST_HTTPBIN = os.getenv('USE_PYTEST_HTTPBIN', '').lower() == 'true'
if USE_PYTEST_HTTPBIN:
import pytest_httpbin # noqa: F401
@pytest.fixture(scope='session', autouse=USE_PYTEST_HTTPBIN)
def httpbin_wrapper(httpbin):
"""Allow pytest-httpbin to be used instead of the httpbin Docker container. This fixture does
not need to be used manually. It will be autoused if both:
* pytest-httpbin is installed
* The environment variable USE_PYTEST_HTTPBIN is set to 'true'
"""
logger.info('Using pytest-httpin for integration tests')
os.environ['HTTPBIN_URL'] = httpbin.url
return httpbin
@pytest.fixture(scope='function')
def mock_session(tempfile_session) -> CachedSession:
"""Fixture for combining requests-cache with requests-mock. This will behave the same as a
CachedSession, except it will make mock requests for ``mock://`` URLs, if it hasn't been cached
already.
For example, ``mock_session.get(MOCKED_URL)`` will return a mock response on the first call,
and a cached mock response on the second call. Additional mock responses can be added via
``mock_session.mock_adapter.register_uri()``.
This uses a temporary SQLite db stored in ``/tmp``, which will be removed after the fixture has
exited.
"""
yield mount_mock_adapter(tempfile_session)
@pytest.fixture(scope='function')
def tempfile_session(tempfile_path) -> CachedSession:
"""Get a CachedSession using a temporary SQLite db"""
yield CachedSession(
cache_name=tempfile_path,
backend='sqlite',
allowable_methods=ALL_METHODS,
)
@pytest.fixture(scope='function')
def tempfile_path(tmpdir) -> str:
"""Get a unique tempfile path"""
yield str(tmpdir / f'{uuid4()}.db')
@pytest.fixture(scope='function')
def installed_session(tempfile_path) -> CachedSession:
"""Get a CachedSession using a temporary SQLite db, with global patching.
Installs cache before test and uninstalls after.
"""
install_cache(
cache_name=tempfile_path,
backend='sqlite',
allowable_methods=ALL_METHODS,
)
yield requests.Session()
uninstall_cache()
def mount_mock_adapter(session: CachedSession) -> CachedSession:
adapter = get_mock_adapter()
for protocol in MOCK_PROTOCOLS:
session.mount(protocol, adapter)
session.mock_adapter = adapter
return session
def get_mock_adapter() -> Adapter:
"""Get a requests-mock Adapter with some URLs mocked by default"""
adapter = Adapter()
adapter.register_uri(
ANY_METHOD,
MOCKED_URL,
headers={'Content-Type': 'text/plain'},
text='mock response',
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_ETAG,
headers={'ETag': ETAG},
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_HTTPS,
headers={'Content-Type': 'text/plain'},
text='mock https response',
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_JSON,
headers={'Content-Type': 'application/json'},
json={
'message': 'mock json response',
'int': 47,
'float': 47.1,
'bool': True,
'null': None,
},
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_JSON_LIST,
headers={'Content-Type': 'application/json'},
json=['item_1', 'item_2', {'message': 'mock json response'}],
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_REDIRECT,
headers={'Content-Type': 'text/plain', 'Location': MOCKED_URL_REDIRECT_TARGET},
text='mock redirect response (1/2)',
status_code=302,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_REDIRECT_TARGET,
headers={'Content-Type': 'text/plain'},
text='mock redirect response (2/2)',
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_VARY,
headers={'Content-Type': 'text/plain', 'Vary': 'Accept'},
text='mock response with Vary header',
status_code=200,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_VARY_REDIRECT,
headers={
'Content-Type': 'text/plain',
'Vary': 'Accept',
'Location': MOCKED_URL_VARY_REDIRECT_TARGET,
},
text='mock redirect response with Vary header (1/2)',
status_code=302,
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_VARY_REDIRECT_TARGET,
headers={'Content-Type': 'text/plain', 'Vary': 'Accept-Language'},
text='mock redirect response with Vary header (2/2)',
status_code=200,
)
adapter.register_uri(ANY_METHOD, MOCKED_URL_404, status_code=404)
adapter.register_uri(ANY_METHOD, MOCKED_URL_500, status_code=500)
adapter.register_uri(
ANY_METHOD, MOCKED_URL_200_404, [{'status_code': 200}, {'status_code': 404}]
)
adapter.register_uri(
ANY_METHOD,
MOCKED_URL_ETAG_200_404,
[{'status_code': 200, 'headers': {'ETag': ETAG}}, {'status_code': 404}],
)
return adapter
def get_mock_response(
method='GET',
url='https://img.site.com/base/img.jpg',
status_code=200,
headers=None,
request_headers=None,
):
return MagicMock(
url=url,
status_code=status_code,
headers=headers or {},
request=Request(method=method, url=url, headers=request_headers or {}),
)
def assert_delta_approx_equal(dt1: datetime, dt2: datetime, target_delta, threshold_seconds=2):
"""Assert that the given datetimes are approximately ``target_delta`` seconds apart"""
diff_in_seconds = (dt2 - dt1).total_seconds()
assert abs(diff_in_seconds - target_delta) <= threshold_seconds
def timeout(timeout_seconds: float):
"""Timeout decorator that uses threading instead of multiprocessing, for compatibility with
pytest-xdist on python 3.14+.
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
result = None
exception = None
def target() -> None:
nonlocal result, exception
try:
result = func(*args, **kwargs)
except Exception as e:
exception = e
thread = threading.Thread(target=target)
thread.daemon = True
thread.start()
thread.join(timeout=timeout_seconds)
if thread.is_alive():
raise TimeoutError(f'Function timed out after {timeout_seconds} seconds')
if exception is not None:
raise exception
return result
return wrapper
return decorator
def fail_if_no_connection(connect_timeout: float = 1.0) -> bool:
"""Decorator for testing a backend connection. This will intentionally cause a test failure if
the wrapped function doesn't have dependencies installed, doesn't connect after a short timeout,
or raises any exceptions.
This allows us to fail quickly for backends that aren't set up, rather than hanging for an
extended period of time.
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
timeout(connect_timeout)(func)(*args, **kwargs)
except Exception as e:
logger.error(e)
pytest.fail('Could not connect to backend')
return wrapper
return decorator
def is_installed(module_name: str) -> bool:
"""Check if a given dependency is installed"""
try:
import_module(module_name)
return True
except ImportError:
return False
def skip_missing_deps(module_name: str) -> pytest.Mark:
return pytest.mark.skipif(
not is_installed(module_name), reason=f'{module_name} is not installed'
)
@contextmanager
def ignore_deprecation():
"""Temporarily silence deprecation warnings"""
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning)
yield
# Some tests must disable url normalization to retain the custom `http+mock://` protocol
patch_normalize_url = patch('requests_cache.cache_keys.normalize_url', side_effect=lambda x, y: x)
# TODO: Debug OperationalErrors with pypy
skip_pypy = pytest.mark.skipif(
platform.python_implementation() == 'PyPy',
reason='pypy-specific database locking issue',
)