From 51d9afa5e42c1984332e5c476853ff6ff664dc26 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Fri, 13 Feb 2026 08:34:30 +0200 Subject: [PATCH 01/22] Improve test infrastructure for parallel execution and better isolation This commit extracts testing improvements from the feature branch, focusing on infrastructure enhancements without feature code: - Add pytest-xdist for parallel test execution - Add pytest-rerunfailures for handling flaky tests - Update test-local.sh script with -p (parallel) and -w (workers) flags - Add seriallocal marker for tests requiring sequential execution - Configure coverage for parallel execution - Add comprehensive fixtures in conftest.py: - `isolated_cache_directory`: Ensures pickle tests use worker-specific cache dirs - `inject_worker_schema_for_sql_tests`: Provides SQL schema isolation per worker - `cleanup_mongo_clients`: Proper MongoDB client cleanup to prevent ResourceWarning - `cleanup_test_schemas`: Cleanup SQL schemas after parallel test runs - Consolidate backend tests into dedicated files: - tests/test_mongo_core.py (MongoDB tests with cleanup improvements) - tests/test_pickle_core.py (Pickle tests with flaky handling) - tests/test_redis_core.py (Redis tests with isolation) - tests/test_sql_core.py (SQL tests with schema isolation) - Add comprehensive tests/README.md with: - Test suite overview and structure - Guidelines for writing isolated tests - Parallel testing documentation - Backend-specific testing instructions - Troubleshooting guide - Mark flaky tests with @pytest.mark.flaky for automatic retries - Add platform-specific timing adjustments (macOS vs Linux) - Improve MongoDB client tracking and cleanup - Add custom mongetter function for collection isolation Co-Authored-By: Claude Sonnet 4.5 --- pyproject.toml | 37 +- scripts/test-local.sh | 82 ++- tests/README.md | 472 ++++++++++++++ tests/conftest.py | 249 +++++++ tests/requirements.txt | 3 +- tests/test_mongo_core.py | 775 ++++++++++++++++++++++ tests/test_pickle_core.py | 1304 +++++++++++++++++++++++++++++++++++++ tests/test_redis_core.py | 1268 ++++++++++++++++++++++++++++++++++++ tests/test_sql_core.py | 537 +++++++++++++++ 9 files changed, 4710 insertions(+), 17 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/conftest.py create mode 100644 tests/test_mongo_core.py create mode 100644 tests/test_pickle_core.py create mode 100644 tests/test_redis_core.py create mode 100644 tests/test_sql_core.py diff --git a/pyproject.toml b/pyproject.toml index 3f49ba9a..bec7a337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", "Topic :: Other/Nonlisted Topic", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", @@ -90,6 +89,9 @@ namespaces = false # to disable scanning PEP 420 namespaces (true by default) # === Linting & Formatting === +[tool.black] +line-length = 79 + # --- ruff --- [tool.ruff] @@ -146,6 +148,7 @@ lint.per-file-ignores."tests/**" = [ "D401", "S101", "S105", + "S110", "S311", "S603", ] @@ -165,8 +168,8 @@ lint.mccabe.max-complexity = 10 [tool.docformatter] recursive = true # some docstring start with r""" -wrap-summaries = 120 -wrap-descriptions = 120 +wrap-summaries = 79 +wrap-descriptions = 79 blank = true # === Testing === @@ -188,6 +191,7 @@ addopts = [ "-v", "-s", "-W error", + # Note: parallel execution is opt-in via --parallel flag or -n option ] markers = [ "mongo: test the MongoDB core", @@ -197,22 +201,25 @@ markers = [ "sql: test the SQL core", "s3: test the S3 core", "maxage: test the max_age functionality", +<<<<<<< HEAD "asyncio: marks tests as async", "smoke: fast smoke tests with no external service dependencies", +======= + "seriallocal: local core tests that should run serially", +>>>>>>> 5116bb0 (Improve test infrastructure for parallel execution and better isolation) ] -[tool.coverage.report] -show_missing = true -# Regexes for lines to exclude from consideration -exclude_lines = [ - "pragma: no cover", # Have to re-enable the standard pragma - "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: - "if TYPE_CHECKING:", # Is only true when running mypy, not tests -] +# Parallel test execution configuration +# Use: pytest -n auto (for automatic worker detection) +# Or: pytest -n 4 (for specific number of workers) +# Memory tests are safe to run in parallel by default +# Pickle tests require isolation (handled by conftest.py fixture) + # --- coverage --- [tool.coverage.run] branch = true +parallel = true # dynamic_context = "test_function" omit = [ "tests/*", @@ -220,3 +227,11 @@ omit = [ "src/cachier/__init__.py", "**/scripts/**", ] +[tool.coverage.report] +show_missing = true +# Regexes for lines to exclude from consideration +exclude_lines = [ + "pragma: no cover", # Have to re-enable the standard pragma + "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: + "if TYPE_CHECKING:", # Is only true when running mypy, not tests +] diff --git a/scripts/test-local.sh b/scripts/test-local.sh index e7efc571..b39a3a51 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -26,6 +26,8 @@ KEEP_RUNNING=false SELECTED_CORES="" INCLUDE_LOCAL_CORES=false TEST_FILES="" +PARALLEL=false +PARALLEL_WORKERS="auto" # Function to print colored messages print_message() { @@ -57,6 +59,8 @@ OPTIONS: -k, --keep-running Keep containers running after tests -h, --html-coverage Generate HTML coverage report -f, --files Specify test files to run (can be used multiple times) + -p, --parallel Run tests in parallel using pytest-xdist + -w, --workers Number of parallel workers (default: auto) --help Show this help message EXAMPLES: @@ -66,6 +70,8 @@ EXAMPLES: $0 external -k # Run external backends, keep containers $0 mongo memory -v # Run MongoDB and memory tests verbosely $0 all -f tests/test_main.py -f tests/test_redis_core_coverage.py # Run specific test files + $0 memory pickle -p # Run local tests in parallel + $0 all -p -w 4 # Run all tests with 4 parallel workers ENVIRONMENT: You can also set cores via CACHIER_TEST_CORES environment variable: @@ -103,6 +109,20 @@ while [[ $# -gt 0 ]]; do usage exit 0 ;; + -p|--parallel) + PARALLEL=true + shift + ;; + -w|--workers) + shift + if [[ $# -eq 0 ]] || [[ "$1" == -* ]]; then + print_message $RED "Error: -w/--workers requires a number argument" + usage + exit 1 + fi + PARALLEL_WORKERS="$1" + shift + ;; -*) print_message $RED "Unknown option: $1" usage @@ -234,11 +254,22 @@ check_dependencies() { } fi + # Check for pytest-xdist if parallel testing is requested + if [ "$PARALLEL" = true ]; then + if ! python -c "import xdist" 2>/dev/null; then + print_message $YELLOW "Installing pytest-xdist for parallel testing..." + pip install pytest-xdist || { + print_message $RED "Failed to install pytest-xdist" + exit 1 + } + fi + fi + # Check MongoDB dependencies if testing MongoDB if echo "$SELECTED_CORES" | grep -qw "mongo"; then if ! python -c "import pymongo" 2>/dev/null; then print_message $YELLOW "Installing MongoDB test requirements..." - pip install -r tests/requirements_mongodb.txt || { + pip install -r tests/mongodb_requirements.txt || { print_message $RED "Failed to install MongoDB requirements" exit 1 } @@ -249,7 +280,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "redis"; then if ! python -c "import redis" 2>/dev/null; then print_message $YELLOW "Installing Redis test requirements..." - pip install -r tests/requirements_redis.txt || { + pip install -r tests/redis_requirements.txt || { print_message $RED "Failed to install Redis requirements" exit 1 } @@ -260,7 +291,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "sql"; then if ! python -c "import sqlalchemy" 2>/dev/null; then print_message $YELLOW "Installing SQL test requirements..." - pip install -r tests/requirements_postgres.txt || { + pip install -r tests/sql_requirements.txt || { print_message $RED "Failed to install SQL requirements" exit 1 } @@ -412,7 +443,7 @@ stop_postgres() { } test_sql() { - export SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" + export SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" } # Main execution @@ -423,14 +454,20 @@ main() { # Check and install dependencies check_dependencies - # Check if we need Docker + # Check if we need Docker, and if we should run serial pickle tests needs_docker=false + run_serial_local_tests=false for core in $SELECTED_CORES; do case $core in mongo|redis|sql) needs_docker=true ;; esac + case $core in + pickle|all) + run_serial_local_tests=true + ;; + esac done if [ "$needs_docker" = true ]; then @@ -497,15 +534,20 @@ main() { sql) test_sql ;; esac done + pytest_markers="$pytest_markers and not seriallocal" # Run pytest # Build pytest command PYTEST_CMD="pytest" + # and the specific pytest command for running serial pickle tests + SERIAL_PYTEST_CMD="pytest -m seriallocal -n0" # Add test files if specified if [ -n "$TEST_FILES" ]; then PYTEST_CMD="$PYTEST_CMD $TEST_FILES" print_message $BLUE "Test files specified: $TEST_FILES" + # and turn off serial local tests, so we run only selected files + run_serial_local_tests=false fi # Add markers if needed (only if no specific test files were given) @@ -517,6 +559,10 @@ main() { if [ "$selected_sorted" != "$all_sorted" ]; then PYTEST_CMD="$PYTEST_CMD -m \"$pytest_markers\"" + else + print_message $BLUE "Running all tests without markers since all cores are selected" + PYTEST_CMD="$PYTEST_CMD -m \"not seriallocal\"" + run_serial_local_tests=true fi else # When test files are specified, still apply markers if not running all cores @@ -532,15 +578,41 @@ main() { # Add verbose flag if needed if [ "$VERBOSE" = true ]; then PYTEST_CMD="$PYTEST_CMD -v" + SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD -v" + fi + + # Add parallel testing options if requested + if [ "$PARALLEL" = true ]; then + PYTEST_CMD="$PYTEST_CMD -n $PARALLEL_WORKERS" + + # Show parallel testing info + if [ "$PARALLEL_WORKERS" = "auto" ]; then + print_message $BLUE "Running tests in parallel with automatic worker detection" + else + print_message $BLUE "Running tests in parallel with $PARALLEL_WORKERS workers" + fi + + # Special note for pickle tests + if echo "$SELECTED_CORES" | grep -qw "pickle"; then + print_message $YELLOW "Note: Pickle tests will use isolated cache directories for parallel safety" + fi fi # Add coverage options PYTEST_CMD="$PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT" + SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT --cov-append" # Print and run the command print_message $BLUE "Running: $PYTEST_CMD" eval $PYTEST_CMD + if [ "$run_serial_local_tests" = true ]; then + print_message $BLUE "Running serial local tests (pickle, memory) with: $SERIAL_PYTEST_CMD" + eval $SERIAL_PYTEST_CMD + else + print_message $BLUE "Skipping serial local tests (pickle, memory) since not requested" + fi + TEST_EXIT_CODE=$? if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..07b54a88 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,472 @@ +# Cachier Test Suite Documentation + +This document provides comprehensive guidelines for writing and running tests for the Cachier package. + +## Table of Contents + +1. [Test Suite Overview](#test-suite-overview) +2. [Test Structure](#test-structure) +3. [Running Tests](#running-tests) +4. [Writing Tests](#writing-tests) +5. [Test Isolation](#test-isolation) +6. [Backend-Specific Testing](#backend-specific-testing) +7. [Parallel Testing](#parallel-testing) +8. [CI/CD Integration](#cicd-integration) +9. [Troubleshooting](#troubleshooting) + +## Test Suite Overview + +The Cachier test suite is designed to comprehensively test all caching backends while maintaining proper isolation between tests. The suite uses pytest with custom markers for backend-specific tests. + +### Supported Backends + +- **Memory**: In-memory caching (no external dependencies) +- **Pickle**: File-based caching using pickle (default backend) +- **MongoDB**: Database caching using MongoDB +- **Redis**: In-memory data store caching +- **SQL**: SQL database caching via SQLAlchemy (PostgreSQL, SQLite, MySQL) + +### Test Categories + +1. **Core Functionality**: Basic caching operations (get, set, clear) +2. **Stale Handling**: Testing `stale_after` parameter +3. **Concurrency**: Thread-safety and multi-process tests +4. **Error Handling**: Exception scenarios and recovery +5. **Performance**: Speed and efficiency tests +6. **Integration**: Cross-backend compatibility + +## Test Structure + +``` +tests/ +├── conftest.py # Shared fixtures and configuration +├── requirements.txt # Base test dependencies (includes pytest-rerunfailures) +├── mongodb_requirements.txt # MongoDB-specific dependencies +├── redis_requirements.txt # Redis-specific dependencies +├── sql_requirements.txt # SQL-specific dependencies +│ +├── test_*.py # Test modules +├── test_mongo_core.py # MongoDB-specific tests +├── test_redis_core.py # Redis-specific tests +├── test_sql_core.py # SQL-specific tests +├── test_memory_core.py # Memory backend tests +├── test_pickle_core.py # Pickle backend tests +├── test_general.py # Cross-backend tests +└── ... +``` + +### Test Markers + +Tests are marked with backend-specific markers: + +```python +@pytest.mark.mongo # MongoDB tests +@pytest.mark.redis # Redis tests +@pytest.mark.sql # SQL tests +@pytest.mark.memory # Memory backend tests +@pytest.mark.pickle # Pickle backend tests +@pytest.mark.maxage # Tests involving stale_after functionality +@pytest.mark.flaky # Flaky tests that should be retried (see Flaky Tests section) +``` + +## Running Tests + +### Quick Start + +```bash +# Run all tests +pytest + +# Run tests for specific backend +pytest -m mongo +pytest -m redis +pytest -m sql + +# Run tests for multiple backends +pytest -m "mongo or redis" + +# Exclude specific backends +pytest -m "not mongo" + +# Run with verbose output +pytest -v +``` + +### Using the Test Script + +The recommended way to run tests with proper backend setup: + +```bash +# Test single backend +./scripts/test-local.sh mongo + +# Test multiple backends +./scripts/test-local.sh mongo redis sql + +# Test all backends +./scripts/test-local.sh all + +# Run tests in parallel +./scripts/test-local.sh all -p + +# Keep containers running for debugging +./scripts/test-local.sh mongo redis -k +``` + +### Parallel Testing + +Tests can be run in parallel using pytest-xdist: + +```bash +# Run with automatic worker detection +./scripts/test-local.sh all -p + +# Specify number of workers +./scripts/test-local.sh all -p -w 4 + +# Or directly with pytest +pytest -n auto +pytest -n 4 +``` + +## Writing Tests + +### Basic Test Structure + +```python +import pytest +from cachier import cachier + + +def test_basic_caching(): + """Test basic caching functionality.""" + + # Define a cached function local to this test + @cachier() + def expensive_computation(x): + return x**2 + + # First call - should compute + result1 = expensive_computation(5) + assert result1 == 25 + + # Second call - should return from cache + result2 = expensive_computation(5) + assert result2 == 25 + + # Clear cache for cleanup + expensive_computation.clear_cache() +``` + +### Backend-Specific Tests + +```python +@pytest.mark.mongo +def test_mongo_specific_feature(): + """Test MongoDB-specific functionality.""" + from tests.test_mongo_core import _test_mongetter + + @cachier(mongetter=_test_mongetter) + def mongo_cached_func(x): + return x * 2 + + # Test implementation + assert mongo_cached_func(5) == 10 +``` + +## Test Isolation + +### Critical Rule: Function Isolation + +**Never share cachier-decorated functions between test functions.** Each test must have its own decorated function to ensure proper isolation. + +#### Why This Matters + +Cachier identifies cached functions by their full module path and function name. When tests share decorated functions: + +- Cache entries can conflict between tests +- Parallel test execution may fail unpredictably +- Test results become non-deterministic + +#### Good Practice + +```python +def test_feature_one(): + @cachier() + def compute_one(x): # Unique to this test + return x * 2 + + assert compute_one(5) == 10 + + +def test_feature_two(): + @cachier() + def compute_two(x): # Different function for different test + return x * 2 + + assert compute_two(5) == 10 +``` + +#### Bad Practice + +```python +# DON'T DO THIS! +@cachier() +def shared_compute(x): # Shared between tests + return x * 2 + + +def test_feature_one(): + assert shared_compute(5) == 10 # May conflict with test_feature_two + + +def test_feature_two(): + assert shared_compute(5) == 10 # May conflict with test_feature_one +``` + +### Isolation Mechanisms + +1. **Pickle Backend**: Uses `isolated_cache_directory` fixture that creates unique directories per pytest-xdist worker +2. **External Backends**: Rely on function namespacing (module + function name) +3. **Clear Cache**: Always clear cache at test end for cleanup + +### Best Practices for Isolation + +1. Define cached functions inside test functions +2. Use unique, descriptive function names +3. Clear cache after each test +4. Avoid module-level cached functions in tests +5. Use fixtures for common setup/teardown + +## Backend-Specific Testing + +### MongoDB Tests + +```python +@pytest.mark.mongo +def test_mongo_feature(): + """Test with MongoDB backend.""" + + @cachier(mongetter=_test_mongetter, wait_for_calc_timeout=2) + def mongo_func(x): + return x + + # MongoDB-specific assertions + assert mongo_func.get_cache_mongetter() is not None +``` + +### Redis Tests + +```python +@pytest.mark.redis +def test_redis_feature(): + """Test with Redis backend.""" + + @cachier(backend="redis", redis_client=_test_redis_client) + def redis_func(x): + return x + + # Redis-specific testing + assert redis_func(5) == 5 +``` + +### SQL Tests + +```python +@pytest.mark.sql +def test_sql_feature(): + """Test with SQL backend.""" + + @cachier(backend="sql", sql_engine=test_engine) + def sql_func(x): + return x + + # SQL-specific testing + assert sql_func(5) == 5 +``` + +### Memory Tests + +```python +@pytest.mark.memory +def test_memory_feature(): + """Test with memory backend.""" + + @cachier(backend="memory") + def memory_func(x): + return x + + # Memory-specific testing + assert memory_func(5) == 5 +``` + +## Parallel Testing + +### How It Works + +1. pytest-xdist creates multiple worker processes +2. Each worker gets a subset of tests +3. Cachier's function identification ensures natural isolation +4. Pickle backend uses worker-specific cache directories + +### Running Parallel Tests + +```bash +# Automatic worker detection +./scripts/test-local.sh all -p + +# Specify workers +./scripts/test-local.sh all -p -w 4 + +# Direct pytest command +pytest -n auto +``` + +### Parallel Testing Considerations + +1. **Resource Usage**: More workers = more CPU/memory usage +2. **External Services**: Ensure Docker has sufficient resources +3. **Test Output**: May be interleaved; use `-v` for clarity +4. **Debugging**: Harder with parallel execution; use `-n 1` for debugging + +## CI/CD Integration + +### GitHub Actions + +The CI pipeline tests all backends: + +```yaml +# Local backends run in parallel +pytest -m "memory or pickle" -n auto + +# External backends run sequentially for stability +pytest -m mongo +pytest -m redis +pytest -m sql +``` + +### Environment Variables + +- `CACHIER_TEST_VS_DOCKERIZED_MONGO`: Use real MongoDB in CI +- `CACHIER_TEST_REDIS_HOST`: Redis connection details +- `SQLALCHEMY_DATABASE_URL`: SQL database connection + +## Troubleshooting + +### Common Issues + +1. **Import Errors**: Install backend-specific requirements + + ```bash + pip install -r tests/redis_requirements.txt + ``` + +2. **Docker Not Running**: Start Docker Desktop or daemon + + ```bash + docker ps # Check if Docker is running + ``` + +3. **Port Conflicts**: Stop conflicting services + + ```bash + docker stop cachier-test-mongo cachier-test-redis cachier-test-postgres + ``` + +4. **Flaky Tests**: Usually due to timing issues + + - Increase timeouts + - Add proper waits + - Check for race conditions + +5. **Cache Conflicts**: Ensure function isolation + + - Don't share decorated functions + - Clear cache after tests + - Use unique function names + +### Handling Flaky Tests + +Some tests, particularly in the pickle core module, may occasionally fail due to race conditions in multi-threaded scenarios. To handle these, we use the `pytest-rerunfailures` plugin. + +#### Marking Flaky Tests + +```python +@pytest.mark.flaky(reruns=5, reruns_delay=0.1) +def test_that_may_fail_intermittently(): + """This test will retry up to 5 times with 0.1s delay between attempts.""" + # Test implementation +``` + +#### Current Flaky Tests + +- `test_bad_cache_file`: Tests handling of corrupted cache files with concurrent access +- `test_delete_cache_file`: Tests handling of missing cache files during concurrent operations + +These tests involve race conditions between threads that are difficult to reproduce consistently, so they're configured to retry multiple times before being marked as failed. + +### Debugging Tips + +1. **Run Single Test**: + + ```bash + pytest -k test_name -v + ``` + +2. **Disable Parallel**: + + ```bash + pytest -n 1 + ``` + +3. **Check Logs**: + + ```bash + docker logs cachier-test-mongo + ``` + +4. **Interactive Debugging**: + + ```python + import pdb + + pdb.set_trace() + ``` + +### Performance Considerations + +1. **Test Speed**: Memory/pickle tests are fastest +2. **External Backends**: Add overhead for Docker/network +3. **Parallel Execution**: Speeds up test suite significantly +4. **Cache Size**: Large caches slow down tests + +## Best Practices Summary + +1. **Always** define cached functions inside test functions +2. **Never** share cached functions between tests +3. **Clear** cache after each test +4. **Use** appropriate markers for backend-specific tests +5. **Run** full test suite before submitting PRs +6. **Test** with parallel execution to catch race conditions +7. **Document** any special test requirements +8. **Follow** existing test patterns in the codebase + +## Adding New Tests + +When adding new tests: + +1. Follow existing naming conventions +2. Add appropriate backend markers +3. Ensure function isolation +4. Include docstrings explaining test purpose +5. Test both success and failure cases +6. Consider edge cases and error conditions +7. Run with all backends if applicable +8. Update this documentation if needed + +## Questions or Issues? + +- Check existing tests for examples +- Review the main README.rst +- Open an issue on GitHub +- Contact maintainers listed in README.rst diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..49dc3cbd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,249 @@ +"""Pytest configuration and shared fixtures for cachier tests.""" + +import logging +import os +from urllib.parse import parse_qs, unquote, urlencode, urlparse, urlunparse + +import pytest + +logger = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def inject_worker_schema_for_sql_tests(monkeypatch, request): + """Automatically inject worker-specific schema into SQL connection string. + + This fixture enables parallel SQL test execution by giving each pytest- + xdist worker its own PostgreSQL schema, preventing table creation + conflicts. + + """ + # Only apply to SQL tests + if "sql" not in request.node.keywords: + yield + return + + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + + if worker_id == "master": + # Not running in parallel, no schema isolation needed + yield + return + + # Get the original SQL connection string + original_url = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" + ) + + if "postgresql" in original_url: + # Create worker-specific schema name + schema_name = f"test_worker_{worker_id.replace('gw', '')}" + + # Parse the URL + parsed = urlparse(original_url) + + # Get existing query parameters + query_params = parse_qs(parsed.query) + + # Add or update the options parameter to set search_path + if "options" in query_params: + # Append to existing options + current_options = unquote(query_params["options"][0]) + new_options = f"{current_options} -csearch_path={schema_name}" + else: + # Create new options + new_options = f"-csearch_path={schema_name}" + + query_params["options"] = [new_options] + + # Rebuild the URL with updated query parameters + new_query = urlencode(query_params, doseq=True) + new_url = urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + new_query, + parsed.fragment, + ) + ) + + # Override both the environment variable and the module constant + monkeypatch.setenv("SQLALCHEMY_DATABASE_URL", new_url) + + # Also patch the SQL_CONN_STR constant used in tests + import tests.test_sql_core + + monkeypatch.setattr(tests.test_sql_core, "SQL_CONN_STR", new_url) + + # Ensure schema creation by creating it before tests run + try: + from sqlalchemy import create_engine, text + + # Use original URL to create schema (without search_path) + engine = create_engine(original_url) + with engine.connect() as conn: + conn.execute( + text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + ) + conn.commit() + engine.dispose() + except Exception as e: + # If we can't create the schema, the test will fail anyway + logger.debug(f"Failed to create schema {schema_name}: {e}") + + yield + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_mongo_clients(): + """Clean up any MongoDB clients created during tests. + + This fixture runs automatically after all tests complete. + + """ + # Let tests run + yield + + # Cleanup after all tests + import contextlib + + try: + from tests.test_mongo_core import _mongo_clients, _test_mongetter + + # Close all tracked MongoDB clients + for client in _mongo_clients: + with contextlib.suppress(Exception): + client.close() + + # Clear the list for next test run + _mongo_clients.clear() + + # Also clean up _test_mongetter specifically + if hasattr(_test_mongetter, "client"): + # Remove the client attribute so future test runs start fresh + delattr(_test_mongetter, "client") + + # Clean up any _custom_mongetter functions that may have been created + import tests.test_mongo_core + + for attr_name in dir(tests.test_mongo_core): + attr = getattr(tests.test_mongo_core, attr_name) + if callable(attr) and hasattr(attr, "client"): + delattr(attr, "client") + + except (ImportError, AttributeError): + # If the module wasn't imported or client wasn't created, + # then there's nothing to clean up + pass + + +@pytest.fixture +def worker_id(request): + """Get the pytest-xdist worker ID.""" + return os.environ.get("PYTEST_XDIST_WORKER", "master") + + +@pytest.fixture(autouse=True) +def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): + """Ensure each test gets an isolated cache directory. + + This is especially important for pickle tests when running in parallel. + Each pytest-xdist worker gets its own cache directory to avoid conflicts. + + """ + if "pickle" in request.node.keywords: + # Create a unique cache directory for this test + if worker_id == "master": + # Not running in parallel mode + cache_dir = tmp_path / "cachier_cache" + else: + # Running with pytest-xdist - use worker-specific directory + cache_dir = tmp_path / f"cachier_cache_{worker_id}" + + cache_dir.mkdir(exist_ok=True, parents=True) + + # Monkeypatch the global cache directory for this test + import cachier.config + + monkeypatch.setattr( + cachier.config._global_params, "cache_dir", str(cache_dir) + ) + + # Also set environment variable as a backup + monkeypatch.setenv("CACHIER_TEST_CACHE_DIR", str(cache_dir)) + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_test_schemas(request): + """Clean up test schemas after all tests complete. + + This fixture ensures that worker-specific PostgreSQL schemas created during + parallel test execution are properly cleaned up. + + """ + yield # Let all tests run first + + # Cleanup after all tests + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") + + if worker_id != "master": + # Clean up the worker-specific schema + original_url = os.environ.get("SQLALCHEMY_DATABASE_URL", "") + + if "postgresql" in original_url: + schema_name = f"test_worker_{worker_id.replace('gw', '')}" + + try: + from sqlalchemy import create_engine, text + + # Parse URL to remove any schema options for cleanup + parsed = urlparse(original_url) + query_params = parse_qs(parsed.query) + + # Remove options parameter if it exists + query_params.pop("options", None) + + # Rebuild clean URL + clean_query = ( + urlencode(query_params, doseq=True) if query_params else "" + ) + clean_url = urlunparse( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + clean_query, + parsed.fragment, + ) + ) + + engine = create_engine(clean_url) + with engine.connect() as conn: + # Drop the schema and all its contents + conn.execute( + text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE") + ) + conn.commit() + engine.dispose() + except Exception as e: + # If cleanup fails, it's not critical + logger.debug(f"Failed to cleanup schema {schema_name}: {e}") + + +def pytest_addoption(parser): + """Add custom command line options for parallel testing.""" + parser.addoption( + "--parallel", + action="store_true", + default=False, + help="Run tests in parallel using pytest-xdist", + ) + parser.addoption( + "--parallel-workers", + action="store", + default="auto", + help="Number of parallel workers (default: auto)", + ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 78297278..c0fe4d4c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,10 @@ # todo: add some version range or pinning latest versions # tests and coverages pytest +pytest-xdist # for parallel test execution +pytest-rerunfailures # for retrying flaky tests coverage pytest-cov -pytest-asyncio birch # to be able to run `python setup.py checkdocs` collective.checkdocs diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py new file mode 100644 index 00000000..4dbd7119 --- /dev/null +++ b/tests/test_mongo_core.py @@ -0,0 +1,775 @@ +"""Testing the MongoDB core of cachier.""" + +# standard library imports +import datetime +import hashlib +import platform +import queue +import sys +import threading +from datetime import timedelta +from random import random +from time import sleep, time +from urllib.parse import quote_plus + +# third-party imports +import pytest +from birch import Birch # type: ignore[import-not-found] + +try: + import pandas as pd +except (ImportError, ModuleNotFoundError): + pd = None + print("pandas is not installed; tests requiring pandas will fail!") + +try: + import pymongo + from pymongo.errors import OperationFailure + from pymongo.mongo_client import MongoClient + + from cachier.cores.mongo import MissingMongetter +except (ImportError, ModuleNotFoundError): + print("pymongo is not installed; tests requiring pymongo will fail!") + pymongo = None + OperationFailure = None + MissingMongetter = None + + # define a mock MongoClient class that will raise an exception + # on init, warning that pymongo is not installed + class MongoClient: + """Mock MongoClient class raising ImportError on missing pymongo.""" + + def __init__(self, *args, **kwargs): + """Initialize the mock MongoClient.""" + raise ImportError("pymongo is not installed!") + + +try: + from pymongo_inmemory import MongoClient as InMemoryMongoClient +except (ImportError, ModuleNotFoundError): + + class InMemoryMongoClient: + """Mock InMemoryMongoClient class. + + Raises an ImportError on missing pymongo_inmemory. + + """ + + def __init__(self, *args, **kwargs): + """Initialize the mock InMemoryMongoClient.""" + raise ImportError("pymongo_inmemory is not installed!") + + print( + "pymongo_inmemory is not installed; in-memory MongoDB tests will fail!" + ) + +# local imports +from cachier import cachier +from cachier.config import CacheEntry +from cachier.cores.base import RecalculationNeeded +from cachier.cores.mongo import _MongoCore + +# === Enables testing vs a real MongoDB instance === + + +class CfgKey: + HOST = "TEST_HOST" + PORT = "TEST_PORT" + # UNAME = "TEST_USERNAME" + # PWD = "TEST_PASSWORD" + # DB = "TEST_DB" + TEST_VS_DOCKERIZED_MONGO = "TEST_VS_DOCKERIZED_MONGO" + + +CFG = Birch( + namespace="cachier", + defaults={CfgKey.TEST_VS_DOCKERIZED_MONGO: False}, +) + + +# URI_TEMPLATE = "mongodb://myUser:myPassword@localhost:27017/" +URI_TEMPLATE = "mongodb://{host}:{port}?retrywrites=true&w=majority" + + +def _get_cachier_db_mongo_client(): + host = quote_plus(CFG[CfgKey.HOST]) + port = quote_plus(CFG[CfgKey.PORT]) + # uname = quote_plus(CFG[CfgKey.UNAME]) + # pwd = quote_plus(CFG[CfgKey.PWD]) + # db = quote_plus(CFG[CfgKey.DB]) + uri = f"mongodb://{host}:{port}?retrywrites=true&w=majority" + return MongoClient(uri) + + +_COLLECTION_NAME = ( + f"cachier_test_{platform.system()}" + f"_{'.'.join(map(str, sys.version_info[:3]))}" +) + + +# Global registry to track all MongoDB clients created during tests +_mongo_clients = [] + + +def cleanup_all_mongo_clients(): + """Clean up all MongoDB clients to prevent ResourceWarning.""" + import contextlib + import sys + + global _mongo_clients + + # Close all tracked clients + for client in _mongo_clients: + with contextlib.suppress(Exception): + client.close() + + # Clear the list + _mongo_clients.clear() + + # Clean up any mongetter functions with clients + current_module = sys.modules[__name__] + for attr_name in dir(current_module): + attr = getattr(current_module, attr_name) + if callable(attr) and hasattr(attr, "client"): + with contextlib.suppress(Exception): + if hasattr(attr.client, "close"): + attr.client.close() + delattr(attr, "client") + + +def _test_mongetter(): + if not hasattr(_test_mongetter, "client"): + if str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_MONGO)).lower() == "true": + print("Using live MongoDB instance for testing.") + _test_mongetter.client = _get_cachier_db_mongo_client() + _mongo_clients.append(_test_mongetter.client) + else: + print("Using in-memory MongoDB instance for testing.") + _test_mongetter.client = InMemoryMongoClient() + _mongo_clients.append(_test_mongetter.client) + db_obj = _test_mongetter.client["cachier_test"] + if _COLLECTION_NAME not in db_obj.list_collection_names(): + db_obj.create_collection(_COLLECTION_NAME) + return db_obj[_COLLECTION_NAME] + + +def _get_mongetter_by_collection_name(collection_name=_COLLECTION_NAME): + """Returns a custom mongetter function using a specified collection name. + + This is important for preventing cache conflicts when running tests in + parallel. + + """ + + def _custom_mongetter(): + if not hasattr(_custom_mongetter, "client"): + if ( + str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_MONGO)).lower() + == "true" + ): + print("Using live MongoDB instance for testing.") + _custom_mongetter.client = _get_cachier_db_mongo_client() + _mongo_clients.append(_custom_mongetter.client) + else: + print("Using in-memory MongoDB instance for testing.") + _custom_mongetter.client = InMemoryMongoClient() + _mongo_clients.append(_custom_mongetter.client) + db_obj = _custom_mongetter.client["cachier_test"] + if _COLLECTION_NAME not in db_obj.list_collection_names(): + db_obj.create_collection(collection_name) + return db_obj[collection_name] + + # Store the mongetter function for cleanup + _custom_mongetter._collection_name = collection_name + return _custom_mongetter + + +@pytest.fixture(autouse=True) +def mongo_cleanup(): + """Ensure MongoDB clients are cleaned up after each test.""" + yield + # Clean up after test + cleanup_all_mongo_clients() + + +# === Mongo core tests === + + +@pytest.mark.mongo +def test_missing_mongetter(): + # Test that the appropriate exception is thrown + # when forgetting to specify the mongetter. + with pytest.raises(MissingMongetter): + + @cachier(backend="mongo", mongetter=None) + def dummy_func(): + pass + + +@pytest.mark.mongo +def test_information(): + print("\npymongo version: ", end="") + print(pymongo.__version__) + + +@pytest.mark.mongo +def test_mongo_index_creation(): + """Basic Mongo core functionality.""" + + @cachier(mongetter=_test_mongetter) + def _decorated(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + collection = _test_mongetter() + _decorated.clear_cache() + val1 = _decorated(1, 2) + val2 = _decorated(1, 2) + assert val1 == val2 + assert _MongoCore._INDEX_NAME in collection.index_information() + + +@pytest.mark.mongo +def test_mongo_core_basic(): + """Basic Mongo core functionality.""" + + @cachier(mongetter=_test_mongetter) + def _funci(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _funci.clear_cache() + val1 = _funci(1, 2) + val2 = _funci(1, 2) + assert val1 == val2 + val3 = _funci(1, 2, cachier__skip_cache=True) + assert val3 != val1 + val4 = _funci(1, 2) + assert val4 == val1 + val5 = _funci(1, 2, cachier__overwrite_cache=True) + assert val5 != val1 + val6 = _funci(1, 2) + assert val6 == val5 + + +@pytest.mark.mongo +def test_mongo_core_keywords(): + """Basic Mongo core functionality with keyword arguments.""" + + @cachier(mongetter=_test_mongetter) + def _func_keywords(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _func_keywords.clear_cache() + val1 = _func_keywords(1, arg_2=2) + val2 = _func_keywords(1, arg_2=2) + assert val1 == val2 + val3 = _func_keywords(1, arg_2=2, cachier__skip_cache=True) + assert val3 != val1 + val4 = _func_keywords(1, arg_2=2) + assert val4 == val1 + val5 = _func_keywords(1, arg_2=2, cachier__overwrite_cache=True) + assert val5 != val1 + val6 = _func_keywords(1, arg_2=2) + assert val6 == val5 + + +@pytest.mark.mongo +def test_mongo_stale_after(): + """Testing MongoDB core stale_after functionality.""" + + @cachier( + mongetter=_test_mongetter, + stale_after=datetime.timedelta(seconds=3), + next_time=False, + ) + def _stale_after_mongo(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _stale_after_mongo.clear_cache() + val1 = _stale_after_mongo(1, 2) + val2 = _stale_after_mongo(1, 2) + assert val1 == val2 + sleep(3) + val3 = _stale_after_mongo(1, 2) + assert val3 != val1 + + +def _calls_takes_time(res_queue): + @cachier(mongetter=_test_mongetter) + def _takes_time(arg_1, arg_2): + """Some function.""" + sleep(3) + return random() + arg_1 + arg_2 + + res = _takes_time(34, 82.3) + res_queue.put(res) + + +@pytest.mark.mongo +def test_mongo_being_calculated(): + """Testing MongoDB core handling of being calculated scenarios.""" + + @cachier(mongetter=_test_mongetter) + def _takes_time(arg_1, arg_2): + """Some function.""" + sleep(3) + return random() + arg_1 + arg_2 + + _takes_time.clear_cache() + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_takes_time, kwargs={"res_queue": res_queue}, daemon=True + ) + thread2 = threading.Thread( + target=_calls_takes_time, kwargs={"res_queue": res_queue}, daemon=True + ) + thread1.start() + sleep(1) + thread2.start() + thread1.join(timeout=4) + thread2.join(timeout=4) + assert res_queue.qsize() == 2 + res1 = res_queue.get() + res2 = res_queue.get() + assert res1 == res2 + + +class _BadMongoCollection: + def __init__(self, mongetter): + self.collection = mongetter() + self.index_information = self.collection.index_information + self.create_indexes = self.collection.create_indexes + self.find_one = self.collection.find_one + + def delete_many(self, *args, **kwargs): + pass + + def update_many(self, *args, **kwargs): + pass + + def update_one(self, *args, **kwargs): + raise OperationFailure(Exception()) + + +def _bad_mongetter(): + return _BadMongoCollection(_test_mongetter) + + +@pytest.mark.mongo +def test_mongo_write_failure(): + """Testing MongoDB core handling of writing failure scenarios.""" + + @cachier(mongetter=_bad_mongetter) + def _func_w_bad_mongo(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + with pytest.raises(OperationFailure): + _func_w_bad_mongo(1, 2) + with pytest.raises(OperationFailure): + _func_w_bad_mongo(1, 2) + # assert val1 == val2 + + +@pytest.mark.mongo +def test_mongo_clear_being_calculated(): + """Testing MongoDB core clear_being_calculated.""" + + @cachier(mongetter=_bad_mongetter) + def _func_w_bad_mongo(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _func_w_bad_mongo.clear_being_calculated() + + +@pytest.mark.mongo +def test_stalled_mongo_db_cache(): + @cachier(mongetter=_test_mongetter) + def _stalled_func(): + return 1 + + core = _MongoCore(None, _test_mongetter, 0) + core.set_func(_stalled_func) + core.clear_cache() + with pytest.raises(RecalculationNeeded): + core.wait_on_entry_calc(key=None) + + +@pytest.mark.mongo +def test_stalled_mong_db_core(monkeypatch): + def mock_get_entry(self, args, kwargs): + return "key", CacheEntry( + _processing=True, value=None, time=None, stale=None + ) + + def mock_get_entry_by_key(self, key): + return "key", None + + monkeypatch.setattr( + "cachier.cores.mongo._MongoCore.get_entry", mock_get_entry + ) + monkeypatch.setattr( + "cachier.cores.mongo._MongoCore.get_entry_by_key", + mock_get_entry_by_key, + ) + + @cachier(mongetter=_test_mongetter) + def _stalled_func(): + return 1 + + res = _stalled_func() + assert res == 1 + + def mock_get_entry_2(self, args, kwargs): + return "key", CacheEntry( + value=1, + time=datetime.datetime.now() - datetime.timedelta(seconds=10), + _processing=True, + stale=None, + ) + + monkeypatch.setattr( + "cachier.cores.mongo._MongoCore.get_entry", mock_get_entry_2 + ) + + stale_after = datetime.timedelta(seconds=1) + + @cachier(mongetter=_test_mongetter, stale_after=stale_after) + def _stalled_func_2(): + """Testing stalled function.""" + return 2 + + res = _stalled_func_2() + assert res == 2 + + @cachier( + mongetter=_test_mongetter, stale_after=stale_after, next_time=True + ) + def _stalled_func_3(): + """Testing stalled function.""" + return 3 + + res = _stalled_func_3() + assert res == 1 + + +@pytest.mark.mongo +def test_callable_hash_param(): + def _hash_func(args, kwargs): + def _hash(obj): + if isinstance(obj, pd.core.frame.DataFrame): + return hashlib.sha256( + pd.util.hash_pandas_object(obj).values.tobytes() + ).hexdigest() + return obj + + k_args = tuple(map(_hash, args)) + k_kwargs = tuple( + sorted({k: _hash(v) for k, v in kwargs.items()}.items()) + ) + return k_args + k_kwargs + + @cachier(mongetter=_test_mongetter, hash_func=_hash_func) + def _params_with_dataframe(*args, **kwargs): + """Some function.""" + return random() + + _params_with_dataframe.clear_cache() + + df_a = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) + df_b = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) + value_a = _params_with_dataframe(df_a, 1) + value_b = _params_with_dataframe(df_b, 1) + + assert value_a == value_b # same content --> same key + + value_a = _params_with_dataframe(1, df=df_a) + value_b = _params_with_dataframe(1, df=df_b) + + assert value_a == value_b # same content --> same key + + +# ==== Imported from test_general.py === + +MONGO_DELTA_LONG = datetime.timedelta(seconds=10) + + +@pytest.mark.mongo +@pytest.mark.parametrize("separate_files", [True, False]) +def test_wait_for_calc_timeout_ok(separate_files): + mongetter = _get_mongetter_by_collection_name( + "test_wait_for_calc_timeout_ok" + ) + + @cachier( + mongetter=mongetter, + stale_after=MONGO_DELTA_LONG, + separate_files=separate_files, + next_time=False, + wait_for_calc_timeout=2, + ) + def _wait_for_calc_timeout_fast(arg_1, arg_2): + """Some function.""" + sleep(1) + return random() + arg_1 + arg_2 + + def _calls_wait_for_calc_timeout_fast(res_queue): + res = _wait_for_calc_timeout_fast(1, 2) + res_queue.put(res) + + """ Testing calls that avoid timeouts store the values in cache. """ + _wait_for_calc_timeout_fast.clear_cache() + val1 = _wait_for_calc_timeout_fast(1, 2) + val2 = _wait_for_calc_timeout_fast(1, 2) + assert val1 == val2 + + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_wait_for_calc_timeout_fast, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_wait_for_calc_timeout_fast, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + + thread1.start() + thread2.start() + sleep(2) + thread1.join(timeout=2) + thread2.join(timeout=2) + assert res_queue.qsize() == 2 + res1 = res_queue.get() + res2 = res_queue.get() + assert res1 == res2 # Timeout did not kick in, a single call was done + + +@pytest.mark.mongo +@pytest.mark.parametrize("separate_files", [True, False]) +@pytest.mark.flaky(reruns=10, reruns_delay=0.5) +def test_wait_for_calc_timeout_slow(separate_files): + # Use unique test parameters to avoid cache conflicts in parallel execution + import os + import uuid + + test_id = os.getpid() + int( + uuid.uuid4().int >> 96 + ) # Unique but deterministic within test + arg1, arg2 = test_id, test_id + 1 + + # In parallel tests, add random delay to reduce thread contention + if os.environ.get("PYTEST_XDIST_WORKER"): + sleep(random() * 0.5) # 0-500ms random delay + + @cachier( + mongetter=_test_mongetter, + stale_after=MONGO_DELTA_LONG, + separate_files=separate_files, + next_time=False, + wait_for_calc_timeout=2, + ) + def _wait_for_calc_timeout_slow(arg_1, arg_2): + sleep(2) + return random() + arg_1 + arg_2 + + def _calls_wait_for_calc_timeout_slow(res_queue): + res = _wait_for_calc_timeout_slow(arg1, arg2) + res_queue.put(res) + + """Testing for calls timing out to be performed twice when needed.""" + _wait_for_calc_timeout_slow.clear_cache() + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_wait_for_calc_timeout_slow, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_wait_for_calc_timeout_slow, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + + thread1.start() + thread2.start() + sleep(1) + res3 = _wait_for_calc_timeout_slow(arg1, arg2) + sleep(3) # Increased from 4 to give more time for threads to complete + thread1.join(timeout=10) # Increased timeout for thread joins + thread2.join(timeout=10) + assert res_queue.qsize() == 2 + res1 = res_queue.get() + res2 = res_queue.get() + assert res1 != res2 # Timeout kicked in. Two calls were done + res4 = _wait_for_calc_timeout_slow(arg1, arg2) + # One of the cached values is returned + assert res1 == res4 or res2 == res4 or res3 == res4 + + +@pytest.mark.mongo +def test_precache_value(): + @cachier(mongetter=_test_mongetter) + def dummy_func(arg_1, arg_2): + """Some function.""" + return arg_1 + arg_2 + + assert dummy_func.precache_value(2, 2, value_to_cache=5) == 5 + assert dummy_func(2, 2) == 5 + dummy_func.clear_cache() + assert dummy_func(2, 2) == 4 + assert dummy_func.precache_value(2, arg_2=2, value_to_cache=5) == 5 + assert dummy_func(2, arg_2=2) == 5 + + +@pytest.mark.mongo +def test_ignore_self_in_methods(): + class DummyClass: + @cachier(mongetter=_test_mongetter) + def takes_2_seconds(self, arg_1, arg_2): + """Some function.""" + sleep(2) + return arg_1 + arg_2 + + test_object_1 = DummyClass() + test_object_2 = DummyClass() + test_object_1.takes_2_seconds.clear_cache() + test_object_2.takes_2_seconds.clear_cache() + assert test_object_1.takes_2_seconds(1, 2) == 3 + start = time() + assert test_object_2.takes_2_seconds(1, 2) == 3 + end = time() + assert end - start < 1 + + +# Test: MongoDB allow_none=False handling (line 99) +@pytest.mark.mongo +def test_mongo_allow_none_false(): + """Test MongoDB backend with allow_none=False and None return value.""" + + @cachier(mongetter=_test_mongetter, allow_none=False) + def returns_none(): + return None + + # First call should execute and return None + result1 = returns_none() + assert result1 is None + + # Second call should also execute (not cached) because None is not allowed + result2 = returns_none() + assert result2 is None + + # Clear cache + returns_none.clear_cache() + + +# test: mongodb none handling with allow_none=false +@pytest.mark.mongo +def test_mongo_allow_none_false_not_stored(): + """Test mongodb doesn't store none when allow_none=false.""" + call_count = 0 + + @cachier(mongetter=_test_mongetter, allow_none=False) + def returns_none(): + nonlocal call_count + call_count += 1 + return None + + returns_none.clear_cache() + + # first call + result1 = returns_none() + assert result1 is None + assert call_count == 1 + + # second call should also execute (not cached) + result2 = returns_none() + assert result2 is None + assert call_count == 2 + + returns_none.clear_cache() + + +# Test: MongoDB delete_stale_entries +@pytest.mark.mongo +def test_mongo_delete_stale_direct(): + """Test MongoDB stale entry deletion method directly.""" + + @cachier(mongetter=_test_mongetter, stale_after=timedelta(seconds=1)) + def test_func(x): + return x * 2 + + test_func.clear_cache() + + # Create entries + test_func(1) + test_func(2) + + # Wait for staleness + sleep(1.1) + + # Access the mongo core and call delete_stale_entries + # This is a bit hacky but needed to test the specific method + from cachier.cores.mongo import _MongoCore + + # Get the collection + _test_mongetter() # Ensure connection is available + + # Create a core instance just for deletion + core = _MongoCore( + mongetter=_test_mongetter, + hash_func=None, + wait_for_calc_timeout=0, + ) + + # Set the function to get the right cache key prefix + core.set_func(test_func) + + # Delete stale entries + core.delete_stale_entries(timedelta(seconds=1)) + + test_func.clear_cache() + + +@pytest.mark.mongo +def test_mongo_unsupported_replacement_policy(): + """Test that unsupported replacement policy raises ValueError.""" + from cachier.cores.mongo import _MongoCore + + # Clear before test + _test_mongetter().delete_many({}) + + @cachier( + mongetter=_test_mongetter, + cache_size_limit="100B", + replacement_policy="lru", # Start with valid policy + ) + def test_func(x): + return "a" * 50 + + # First, fill the cache to trigger eviction + test_func(1) + test_func(2) + + # Now create a core with an unsupported policy + core = _MongoCore( + hash_func=None, + mongetter=_test_mongetter, + wait_for_calc_timeout=0, + cache_size_limit=100, + replacement_policy="invalid_policy", # Invalid policy + ) + core.set_func(test_func) + + # This should raise ValueError when trying to evict + with pytest.raises( + ValueError, match="Unsupported replacement policy: invalid_policy" + ): + core.set_entry("new_key", "a" * 50) + + test_func.clear_cache() diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py new file mode 100644 index 00000000..aa0c5134 --- /dev/null +++ b/tests/test_pickle_core.py @@ -0,0 +1,1304 @@ +"""Test for the Cachier python package.""" + +# This file is part of Cachier. +# https://github.com/python-cachier/cachier + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/MIT-license +# Copyright (c) 2016, Shay Palachy + +# from os.path import ( +# realpath, +# dirname +# ) +import hashlib +import os +import pickle +import sys +import tempfile +import threading +import uuid +from datetime import datetime, timedelta +from random import random +from time import sleep, time +from unittest.mock import Mock, patch + +import pytest + +try: + import queue +except ImportError: # python 2 + import Queue as queue # type: ignore + + +import pandas as pd + +from cachier import cachier +from cachier.config import CacheEntry, _global_params +from cachier.cores.pickle import _PickleCore + + +def _get_decorated_func(func, **kwargs): + cachier_decorator = cachier(**kwargs) + decorated_func = cachier_decorator(func) + return decorated_func + + +# Pickle core tests + + +def _takes_2_seconds(arg_1, arg_2): + """Some function.""" + sleep(2) + return f"arg_1:{arg_1}, arg_2:{arg_2}" + + +@pytest.mark.pickle +@pytest.mark.parametrize("reload", [True, False]) +@pytest.mark.parametrize("separate_files", [True, False]) +def test_pickle_core(reload, separate_files): + """Basic Pickle core functionality.""" + _takes_2_seconds_decorated = _get_decorated_func( + _takes_2_seconds, + next_time=False, + pickle_reload=reload, + separate_files=separate_files, + ) + _takes_2_seconds_decorated.clear_cache() + _takes_2_seconds_decorated("a", "b") + start = time() + _takes_2_seconds_decorated("a", "b", cachier__verbose=True) + end = time() + assert end - start < 1 + _takes_2_seconds_decorated.clear_cache() + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_pickle_core_keywords(separate_files): + """Basic Pickle core functionality with keyword arguments.""" + _takes_2_seconds_decorated = _get_decorated_func( + _takes_2_seconds, next_time=False, separate_files=separate_files + ) + _takes_2_seconds_decorated.clear_cache() + _takes_2_seconds_decorated("a", arg_2="b") + start = time() + _takes_2_seconds_decorated("a", arg_2="b", cachier__verbose=True) + end = time() + assert end - start < 1 + _takes_2_seconds_decorated.clear_cache() + + +SECONDS_IN_DELTA = 3 +DELTA = timedelta(seconds=SECONDS_IN_DELTA) + + +def _stale_after_seconds(arg_1, arg_2): + """Some function.""" + return random() + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_stale_after(separate_files): + """Testing the stale_after functionality.""" + _stale_after_seconds_decorated = _get_decorated_func( + _stale_after_seconds, + stale_after=DELTA, + next_time=False, + separate_files=separate_files, + ) + _stale_after_seconds_decorated.clear_cache() + val1 = _stale_after_seconds_decorated(1, 2) + val2 = _stale_after_seconds_decorated(1, 2) + val3 = _stale_after_seconds_decorated(1, 3) + assert val1 == val2 + assert val1 != val3 + sleep(3) + val4 = _stale_after_seconds_decorated(1, 2) + assert val4 != val1 + _stale_after_seconds_decorated.clear_cache() + + +def _stale_after_next_time(arg_1, arg_2): + """Some function.""" + return random() + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_stale_after_next_time(separate_files): + """Testing the stale_after with next_time functionality.""" + _stale_after_next_time_decorated = _get_decorated_func( + _stale_after_next_time, + stale_after=DELTA, + next_time=True, + separate_files=separate_files, + ) + _stale_after_next_time_decorated.clear_cache() + val1 = _stale_after_next_time_decorated(1, 2) + val2 = _stale_after_next_time_decorated(1, 2) + val3 = _stale_after_next_time_decorated(1, 3) + assert val1 == val2 + assert val1 != val3 + sleep(SECONDS_IN_DELTA + 1) + val4 = _stale_after_next_time_decorated(1, 2) + assert val4 == val1 + sleep(0.5) + val5 = _stale_after_next_time_decorated(1, 2) + assert val5 != val1 + _stale_after_next_time_decorated.clear_cache() + + +def _random_num(): + return random() + + +def _random_num_with_arg(a): + # print(a) + return random() + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_overwrite_cache(separate_files): + """Tests that the overwrite feature works correctly.""" + _random_num_decorated = _get_decorated_func( + _random_num, separate_files=separate_files + ) + _random_num_with_arg_decorated = _get_decorated_func( + _random_num_with_arg, separate_files=separate_files + ) + _random_num_decorated.clear_cache() + int1 = _random_num_decorated() + int2 = _random_num_decorated() + assert int2 == int1 + int3 = _random_num_decorated(cachier__overwrite_cache=True) + assert int3 != int1 + int4 = _random_num_decorated() + assert int4 == int3 + _random_num_decorated.clear_cache() + + _random_num_with_arg_decorated.clear_cache() + int1 = _random_num_with_arg_decorated("a") + int2 = _random_num_with_arg_decorated("a") + assert int2 == int1 + int3 = _random_num_with_arg_decorated("a", cachier__overwrite_cache=True) + assert int3 != int1 + int4 = _random_num_with_arg_decorated("a") + assert int4 == int3 + _random_num_with_arg_decorated.clear_cache() + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_ignore_cache(separate_files): + """Tests that the ignore_cache feature works correctly.""" + _random_num_decorated = _get_decorated_func( + _random_num, separate_files=separate_files + ) + _random_num_with_arg_decorated = _get_decorated_func( + _random_num_with_arg, separate_files=separate_files + ) + _random_num_decorated.clear_cache() + int1 = _random_num_decorated() + int2 = _random_num_decorated() + assert int2 == int1 + int3 = _random_num_decorated(cachier__skip_cache=True) + assert int3 != int1 + int4 = _random_num_decorated() + assert int4 != int3 + assert int4 == int1 + _random_num_decorated.clear_cache() + + _random_num_with_arg_decorated.clear_cache() + int1 = _random_num_with_arg_decorated("a") + int2 = _random_num_with_arg_decorated("a") + assert int2 == int1 + int3 = _random_num_with_arg_decorated("a", cachier__skip_cache=True) + assert int3 != int1 + int4 = _random_num_with_arg_decorated("a") + assert int4 != int3 + assert int4 == int1 + _random_num_with_arg_decorated.clear_cache() + + +def _takes_time(arg_1, arg_2): + """Some function.""" + sleep(2) # this has to be enough time for check_calculation to run twice + return random() + arg_1 + arg_2 + + +def _calls_takes_time(takes_time_func, res_queue): + res = takes_time_func(0.13, 0.02) + res_queue.put(res) + + +@pytest.mark.pickle +@pytest.mark.flaky(reruns=5, reruns_delay=0.5) +@pytest.mark.parametrize("separate_files", [True, False]) +def test_pickle_being_calculated(separate_files): + """Testing pickle core handling of being calculated scenarios.""" + _takes_time_decorated = _get_decorated_func( + _takes_time, separate_files=separate_files + ) + _takes_time_decorated.clear_cache() + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_takes_time, + kwargs={ + "takes_time_func": _takes_time_decorated, + "res_queue": res_queue, + }, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_takes_time, + kwargs={ + "takes_time_func": _takes_time_decorated, + "res_queue": res_queue, + }, + daemon=True, + ) + thread1.start() + sleep(0.5) + thread2.start() + thread1.join(timeout=4) + thread2.join(timeout=4) + assert res_queue.qsize() == 2 + res1 = res_queue.get() + res2 = res_queue.get() + assert res1 == res2 + + +def _being_calc_next_time(arg_1, arg_2): + """Some function.""" + sleep(1) + return random() + arg_1 + arg_2 + + +def _calls_being_calc_next_time(being_calc_func, res_queue): + res = being_calc_func(0.13, 0.02) + res_queue.put(res) + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +@pytest.mark.flaky(reruns=5, reruns_delay=0.1) +def test_being_calc_next_time(separate_files): + """Testing pickle core handling of being calculated scenarios.""" + _being_calc_next_time_decorated = _get_decorated_func( + _being_calc_next_time, + stale_after=timedelta(seconds=1), + next_time=True, + separate_files=separate_files, + ) + _being_calc_next_time_decorated.clear_cache() + _being_calc_next_time(0.13, 0.02) + sleep(1.1) + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_being_calc_next_time, + kwargs={ + "being_calc_func": _being_calc_next_time_decorated, + "res_queue": res_queue, + }, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_being_calc_next_time, + kwargs={ + "being_calc_func": _being_calc_next_time_decorated, + "res_queue": res_queue, + }, + daemon=True, + ) + thread1.start() + sleep(0.5) + thread2.start() + thread1.join(timeout=2) + thread2.join(timeout=2) + assert res_queue.qsize() == 2 + res1 = res_queue.get() + res2 = res_queue.get() + assert res1 == res2 + + +def _bad_cache(arg_1, arg_2): + """Some function.""" + sleep(1) + return random() + arg_1 + arg_2 + + +# _BAD_CACHE_FNAME = '.__main__._bad_cache' +_BAD_CACHE_FNAME = ".tests.test_pickle_core._bad_cache" +_BAD_CACHE_FNAME_SEPARATE_FILES = ( + ".tests.test_pickle_core._bad_cache_" + f"{hashlib.sha256(pickle.dumps((0.13, 0.02))).hexdigest()}" +) +EXPANDED_CACHIER_DIR = os.path.expanduser(_global_params.cache_dir) +_BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME) +_BAD_CACHE_FPATH_SEPARATE_FILES = os.path.join( + EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME_SEPARATE_FILES +) +_BAD_CACHE_FPATHS = { + True: _BAD_CACHE_FPATH_SEPARATE_FILES, + False: _BAD_CACHE_FPATH, +} + + +def _calls_bad_cache( + bad_cache_func, res_queue, trash_cache, separate_files, cache_dir +): + try: + res = bad_cache_func(0.13, 0.02, cachier__verbose=True) + if trash_cache: + # Use the provided cache directory + if separate_files: + fname = _BAD_CACHE_FNAME_SEPARATE_FILES + else: + fname = _BAD_CACHE_FNAME + cache_fpath = os.path.join(cache_dir, fname) + with open(cache_fpath, "w") as cache_file: + cache_file.seek(0) + cache_file.truncate() + res_queue.put(res) + except Exception as exc: + res_queue.put(exc) + + +def _helper_bad_cache_file(sleep_time: float, separate_files: bool): + """Test pickle core handling of bad cache files.""" + # Use a unique cache directory for this test to avoid parallel conflicts + unique_cache_dir = os.path.join( + tempfile.gettempdir(), f"cachier_test_bad_{uuid.uuid4().hex[:8]}" + ) + os.makedirs(unique_cache_dir, exist_ok=True) + + _bad_cache_decorated = _get_decorated_func( + _bad_cache, separate_files=separate_files, cache_dir=unique_cache_dir + ) + _bad_cache_decorated.clear_cache() + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_bad_cache, + kwargs={ + "bad_cache_func": _bad_cache_decorated, + "res_queue": res_queue, + "trash_cache": True, + "separate_files": separate_files, + "cache_dir": unique_cache_dir, + }, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_bad_cache, + kwargs={ + "bad_cache_func": _bad_cache_decorated, + "res_queue": res_queue, + "trash_cache": False, + "separate_files": separate_files, + "cache_dir": unique_cache_dir, + }, + daemon=True, + ) + thread1.start() + sleep(sleep_time) + thread2.start() + thread1.join(timeout=2) + thread2.join(timeout=2) + if res_queue.qsize() != 2: + return False + res1 = res_queue.get() + if not isinstance(res1, float): + return False + res2 = res_queue.get() + return res2 is None + + +# we want this to succeed at least once +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +@pytest.mark.flaky(reruns=8, reruns_delay=0.1) +def test_bad_cache_file(separate_files): + """Test pickle core handling of bad cache files.""" + # On macOS, file system events and watchdog timing can be different + if sys.platform == "darwin": + sleep_times = [1.0, 1.5, 2.0, 2.5, 3.0] + else: + sleep_times = [0.6, 1, 1.5, 2, 2.5] + bad_file = False + for sleep_time in sleep_times * 2: + if _helper_bad_cache_file(sleep_time, separate_files): + bad_file = True + break + # it is expected that for separate_files=True files will not be bad + assert bad_file is not separate_files + + +def _delete_cache(arg_1, arg_2): + """Some function.""" + sleep(1) + return random() + arg_1 + arg_2 + + +# _DEL_CACHE_FNAME = '.__main__._delete_cache' +_DEL_CACHE_FNAME = ".tests.test_pickle_core._delete_cache" +_DEL_CACHE_FNAME_SEPARATE_FILES = ( + ".tests.test_pickle_core._delete_cache_" + f"{hashlib.sha256(pickle.dumps((0.13, 0.02))).hexdigest()}" +) +_DEL_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME) +_DEL_CACHE_FPATH_SEPARATE_FILES = os.path.join( + EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME_SEPARATE_FILES +) +_DEL_CACHE_FPATHS = { + True: _DEL_CACHE_FPATH_SEPARATE_FILES, + False: _DEL_CACHE_FPATH, +} + + +def _calls_delete_cache( + del_cache_func, + res_queue, + del_cache: bool, + separate_files: bool, + cache_dir: str, +): + try: + # print('in') + res = del_cache_func(0.13, 0.02) + # print('out with {}'.format(res)) + if del_cache: + # Use the provided cache directory + if separate_files: + fname = _DEL_CACHE_FNAME_SEPARATE_FILES + else: + fname = _DEL_CACHE_FNAME + cache_fpath = os.path.join(cache_dir, fname) + os.remove(cache_fpath) + # print(os.path.isfile(_DEL_CACHE_FPATH)) + res_queue.put(res) + except Exception as exc: + # print('found') + res_queue.put(exc) + + +def _helper_delete_cache_file(sleep_time: float, separate_files: bool): + """Test pickle core handling of missing cache files.""" + # Use a unique cache directory for this test to avoid parallel conflicts + unique_cache_dir = os.path.join( + tempfile.gettempdir(), f"cachier_test_del_{uuid.uuid4().hex[:8]}" + ) + os.makedirs(unique_cache_dir, exist_ok=True) + + _delete_cache_decorated = _get_decorated_func( + _delete_cache, + separate_files=separate_files, + cache_dir=unique_cache_dir, + ) + _delete_cache_decorated.clear_cache() + res_queue = queue.Queue() + thread1 = threading.Thread( + target=_calls_delete_cache, + kwargs={ + "del_cache_func": _delete_cache_decorated, + "res_queue": res_queue, + "del_cache": True, + "separate_files": separate_files, + "cache_dir": unique_cache_dir, + }, + daemon=True, + ) + thread2 = threading.Thread( + target=_calls_delete_cache, + kwargs={ + "del_cache_func": _delete_cache_decorated, + "res_queue": res_queue, + "del_cache": False, + "separate_files": separate_files, + "cache_dir": unique_cache_dir, + }, + daemon=True, + ) + thread1.start() + sleep(sleep_time) + thread2.start() + thread1.join(timeout=2) + thread2.join(timeout=2) + if res_queue.qsize() != 2: + return False + res1 = res_queue.get() + # print(res1) + if not isinstance(res1, float): + return False + res2 = res_queue.get() + return isinstance(res2, KeyError) or (res2 is None) + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [False, True]) +@pytest.mark.flaky(reruns=10, reruns_delay=0.1) +def test_delete_cache_file(separate_files): + """Test pickle core handling of missing cache files.""" + # On macOS, file system events and watchdog timing can be different + if sys.platform == "darwin": + sleep_times = [0.2, 0.4, 0.6, 0.8, 1.0, 1.5] + else: + sleep_times = [0.1, 0.2, 0.3, 0.5, 0.7, 1] + deleted = False + for sleep_time in sleep_times * 4: + if _helper_delete_cache_file(sleep_time, separate_files): + deleted = True + break + # it is expected that for separate_files=True files will not be deleted + assert deleted is not separate_files + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [False, True]) +def test_clear_being_calculated(separate_files): + """Test pickle core clear `being calculated` functionality.""" + _takes_time_decorated = _get_decorated_func( + _takes_time, separate_files=separate_files + ) + _takes_time_decorated.clear_being_calculated() + + +def _error_throwing_func(arg1): + if not hasattr(_error_throwing_func, "count"): + _error_throwing_func.count = 0 + _error_throwing_func.count += 1 + if _error_throwing_func.count > 1: + raise ValueError("Tiny Rick!") + return 7 + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_error_throwing_func(separate_files): + # with + _error_throwing_func.count = 0 + _error_throwing_func_decorated = _get_decorated_func( + _error_throwing_func, + stale_after=timedelta(seconds=1), + next_time=True, + separate_files=separate_files, + ) + _error_throwing_func_decorated.clear_cache() + res1 = _error_throwing_func_decorated(4) + sleep(1.5) + res2 = _error_throwing_func_decorated(4) + assert res1 == res2 + + +# test custom cache dir for pickle core + +CUSTOM_DIR = "~/.exparrot" +EXPANDED_CUSTOM_DIR = os.path.expanduser(CUSTOM_DIR) + + +def _takes_2_seconds_custom_dir(arg_1, arg_2): + """Some function.""" + sleep(2) + return f"arg_1:{arg_1}, arg_2:{arg_2}" + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_pickle_core_custom_cache_dir(separate_files): + """Basic Pickle core functionality.""" + _takes_2_seconds_custom_dir_decorated = _get_decorated_func( + _takes_2_seconds_custom_dir, + next_time=False, + cache_dir=CUSTOM_DIR, + separate_files=separate_files, + ) + _takes_2_seconds_custom_dir_decorated.clear_cache() + _takes_2_seconds_custom_dir_decorated("a", "b") + start = time() + _takes_2_seconds_custom_dir_decorated("a", "b", cachier__verbose=True) + end = time() + assert end - start < 1 + _takes_2_seconds_custom_dir_decorated.clear_cache() + path2test = _takes_2_seconds_custom_dir_decorated.cache_dpath() + assert path2test == EXPANDED_CUSTOM_DIR + + +@pytest.mark.pickle +@pytest.mark.parametrize("separate_files", [True, False]) +def test_callable_hash_param(separate_files): + def _hash_func(args, kwargs): + def _hash(obj): + if isinstance(obj, pd.core.frame.DataFrame): + return hashlib.sha256( + pd.util.hash_pandas_object(obj).values.tobytes() + ).hexdigest() + return obj + + k_args = tuple(map(_hash, args)) + k_kwargs = tuple( + sorted({k: _hash(v) for k, v in kwargs.items()}.items()) + ) + return k_args + k_kwargs + + @cachier(hash_func=_hash_func, separate_files=separate_files) + def _params_with_dataframe(*args, **kwargs): + """Some function.""" + return random() + + _params_with_dataframe.clear_cache() + + df_a = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) + df_b = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) + value_a = _params_with_dataframe(df_a, 1) + value_b = _params_with_dataframe(df_b, 1) + + assert value_a == value_b # same content --> same key + + value_a = _params_with_dataframe(1, df=df_a) + value_b = _params_with_dataframe(1, df=df_b) + + assert value_a == value_b # same content --> same key + + +@pytest.mark.pickle +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="inotify instance limit is only relevant on Linux", +) +def test_inotify_instance_limit_reached(): + """Reproduces the inotify instance exhaustion issue (see Issue #24). + + Rapidly creates many cache waits to exhaust inotify instances. + Reference: https://github.com/python-cachier/cachier/issues/24 + + """ + import queue + import subprocess + + # Try to get the current inotify limit + try: + result = subprocess.run( + ["/bin/cat", "/proc/sys/fs/inotify/max_user_instances"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + current_limit = int(result.stdout.strip()) + print(f"Current inotify max_user_instances limit: {current_limit}") + else: + current_limit = None + print("Could not determine inotify limit") + except Exception as e: + current_limit = None + print(f"Error getting inotify limit: {e}") + + @cachier(backend="pickle", wait_for_calc_timeout=0.1) + def slow_func(x): + sleep(0.5) # Make it slower to increase chance of hitting limit + return x + + # Start many threads to trigger wait_on_entry_calc + threads = [] + errors = [] + results = queue.Queue() + + # Be more aggressive - try to exhaust the limit + N = ( + min(current_limit * 4, 4096) if current_limit is not None else 4096 + ) # Try to exceed the limit more aggressively + print(f"Starting {N} threads to test inotify exhaustion") + + def call(): + try: + results.put(slow_func(1)) + except OSError as e: + errors.append(e) + except Exception as e: + # Capture any other exceptions for debugging + errors.append(e) + + for i in range(N): + t = threading.Thread(target=call) + threads.append(t) + t.start() + if i % 100 == 0: + print(f"Started {i} threads...") + + print("Waiting for all threads to complete...") + for t in threads: + t.join() + + print( + f"Test completed. Got {len(errors)} errors, {results.qsize()} results" + ) + + # If any OSError with "inotify instance limit reached" is raised, + # the test FAILS (expected failure due to the bug) + if any("inotify instance limit reached" in str(e) for e in errors): + print( + "FAILURE: Hit inotify instance limit - this indicates the bug " + "still exists" + ) + raise AssertionError( + "inotify instance limit reached error occurred. " + f"Got {len(errors)} errors with inotify limit issues." + ) + + # If no inotify errors but other errors, fail + if errors: + print(f"Unexpected errors occurred: {errors}") + raise AssertionError(f"Unexpected OSErrors: {errors}") + + # If no errors at all, the test PASSES (issue is fixed!) + print( + "SUCCESS: No inotify instance limit errors occurred - the issue " + "appears to be fixed!" + ) + # No need to return - test passes naturally + + +@pytest.mark.pickle +def test_convert_legacy_cache_entry_dict(): + """Test _convert_legacy_cache_entry with dict input.""" + # Test line 112-118: converting legacy dict format + legacy_entry = { + "value": "test_value", + "time": datetime.now(), + "stale": False, + "being_calculated": True, + "condition": None, + } + + result = _PickleCore._convert_legacy_cache_entry(legacy_entry) + + assert isinstance(result, CacheEntry) + assert result.value == "test_value" + assert result.stale is False + assert result._processing is True + + +@pytest.mark.pickle +def test_save_cache_with_invalid_separate_file_key(): + """Test _save_cache raises error with invalid separate_file_key.""" + # Test line 179-181: ValueError when separate_file_key used with dict + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=False, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Should raise ValueError when using separate_file_key with a dict + with pytest.raises( + ValueError, + match="`separate_file_key` should only be used with a CacheEntry", + ): + core._save_cache({"key": "value"}, separate_file_key="test_key") + + +@pytest.mark.pickle +def test_set_entry_should_not_store(): + """Test set_entry when value should not be stored.""" + # Test line 204: early return when _should_store returns False + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=False, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock _should_store to return False + core._should_store = Mock(return_value=False) + + result = core.set_entry("test_key", None) + assert result is False + + +@pytest.mark.pickle +def test_mark_entry_not_calculated_separate_files_no_entry(): + """Test _mark_entry_not_calculated_separate_files with no entry.""" + # Test line 236: early return when entry is None + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=True, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock get_entry_by_key to return None + core.get_entry_by_key = Mock(return_value=("test_key", None)) + + # Should return without error + core._mark_entry_not_calculated_separate_files("test_key") + + +@pytest.mark.pickle +def test_cleanup_observer_exception(): + """Test _cleanup_observer with exception during cleanup.""" + # Test lines 278-279: exception handling in observer cleanup + core = _PickleCore( + hash_func=None, + cache_dir=".", + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=False, + ) + + # Set a mock function + mock_func = Mock() + mock_func.__name__ = "test_func" + mock_func.__module__ = "test_module" + mock_func.__qualname__ = "test_func" + core.set_func(mock_func) + + # Mock observer that raises exception + mock_observer = Mock() + mock_observer.is_alive.return_value = True + mock_observer.stop.side_effect = Exception("Observer error") + + # Should not raise exception + core._cleanup_observer(mock_observer) + + +@pytest.mark.pickle +def test_wait_on_entry_calc_inotify_limit(): + """Test wait_on_entry_calc fallback when inotify limit is reached.""" + # Test lines 298-302: OSError handling for inotify limit + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=False, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Create a cache entry that's being calculated + cache_entry = CacheEntry( + value="test_value", + time=datetime.now(), + stale=False, + _processing=True, # Should be processing + ) + core._save_cache({"test_key": cache_entry}) + + # Mock _wait_with_inotify to raise OSError with inotify message + def mock_wait_inotify(key, filename): + raise OSError("inotify instance limit reached") + + core._wait_with_inotify = mock_wait_inotify + + # Mock _wait_with_polling to return a value + core._wait_with_polling = Mock(return_value="polling_result") + + result = core.wait_on_entry_calc("test_key") + assert result == "polling_result" + core._wait_with_polling.assert_called_once_with("test_key") + + +@pytest.mark.pickle +def test_wait_on_entry_calc_other_os_error(): + """Test wait_on_entry_calc re-raises non-inotify OSErrors.""" + # Test line 302: re-raise other OSErrors + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=False, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock _wait_with_inotify to raise different OSError + def mock_wait_inotify(key, filename): + raise OSError("Different error") + + core._wait_with_inotify = mock_wait_inotify + + with pytest.raises(OSError, match="Different error"): + core.wait_on_entry_calc("test_key") + + +@pytest.mark.pickle +def test_wait_with_polling_file_errors(): + """Test _wait_with_polling handles file errors gracefully.""" + # Test lines 352-354: FileNotFoundError/EOFError handling + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=2, # Short timeout + separate_files=False, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock methods to simulate file errors then success + call_count = 0 + + def mock_get_cache_dict(): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise FileNotFoundError("Cache file not found") + elif call_count == 2: + raise EOFError("Cache file corrupted") + else: + return { + "test_key": CacheEntry( + value="result", + time=datetime.now(), + stale=False, + _processing=False, + ) + } + + core.get_cache_dict = mock_get_cache_dict + core.separate_files = False + + result = core._wait_with_polling("test_key") + assert result == "result" + + +@pytest.mark.pickle +def test_wait_with_polling_separate_files(): + """Test _wait_with_polling with separate files mode.""" + # Test lines 342-343: separate files branch + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=True, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock _load_cache_by_key + entry = CacheEntry( + value="test_value", + time=datetime.now(), + stale=False, + _processing=False, + ) + core._load_cache_by_key = Mock(return_value=entry) + + result = core._wait_with_polling("test_key") + assert result == "test_value" + + +@pytest.mark.pickle +def test_delete_stale_entries_separate_files(): + """Test delete_stale_entries with separate files mode.""" + # Test lines 377-387: separate files deletion logic + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=True, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Create some cache files + base_path = core.cache_fpath + + # Create stale entry file + stale_entry = CacheEntry( + value="stale_value", + time=datetime.now() - timedelta(hours=2), + stale=False, + _processing=False, + ) + stale_file = f"{base_path}_stalekey" + with open(stale_file, "wb") as f: + pickle.dump(stale_entry, f) + + # Create fresh entry file + fresh_entry = CacheEntry( + value="fresh_value", + time=datetime.now(), + stale=False, + _processing=False, + ) + fresh_file = f"{base_path}_freshkey" + with open(fresh_file, "wb") as f: + pickle.dump(fresh_entry, f) + + # Create non-matching file (should be ignored) + other_file = os.path.join(temp_dir, "other_file.txt") + with open(other_file, "w") as f: + f.write("other content") + + # Before running delete, check that files exist + assert os.path.exists(stale_file) + assert os.path.exists(fresh_file) + + # Run delete_stale_entries + core.delete_stale_entries(timedelta(hours=1)) + + # Check that only stale file was deleted + assert not os.path.exists(stale_file) + assert os.path.exists(fresh_file) + assert os.path.exists(other_file) + + +@pytest.mark.pickle +def test_delete_stale_entries_file_not_found(): + """Test delete_stale_entries handles FileNotFoundError.""" + # Test lines 385-386: FileNotFoundError suppression + with tempfile.TemporaryDirectory() as temp_dir: + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=10, + separate_files=True, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Mock _load_cache_by_key to return a stale entry + stale_entry = CacheEntry( + value="stale", + time=datetime.now() - timedelta(hours=2), + stale=False, + _processing=False, + ) + core._load_cache_by_key = Mock(return_value=stale_entry) + + # Mock os.remove to raise FileNotFoundError + with patch("os.remove", side_effect=FileNotFoundError): + # Should not raise exception + core.delete_stale_entries(timedelta(hours=1)) + + +# Pickle clear being calculated with separate files +@pytest.mark.pickle +def test_pickle_clear_being_calculated_separate_files(): + """Test clearing processing flags in separate cache files.""" + with tempfile.TemporaryDirectory() as temp_dir: + + @cachier(backend="pickle", cache_dir=temp_dir, separate_files=True) + def test_func(x): + return x * 2 + + # Get the pickle core + from cachier.cores.pickle import _PickleCore + + # Create a temporary core to manipulate cache + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=0, + separate_files=True, + ) + core.set_func(test_func) + + # Create cache entries with processing flag + for i in range(3): + entry = CacheEntry( + value=i * 2, time=datetime.now(), stale=False, _processing=True + ) + # Create hash for key + key_hash = str(hash((i,))) + # For separate files, save the entry directly + core._save_cache(entry, separate_file_key=key_hash) + + # Clear being calculated + core._clear_being_calculated_all_cache_files() + + # Verify files exist but processing is cleared + cache_files = [f for f in os.listdir(temp_dir) if f.startswith(".")] + assert len(cache_files) >= 3 + + test_func.clear_cache() + + +# Pickle save with hash_str parameter +@pytest.mark.pickle +def test_pickle_save_with_hash_str(): + """Test _save_cache with hash_str creates correct filename.""" + with tempfile.TemporaryDirectory() as temp_dir: + from cachier.cores.pickle import _PickleCore + + core = _PickleCore( + hash_func=None, + cache_dir=temp_dir, + pickle_reload=False, + wait_for_calc_timeout=0, + separate_files=True, + ) + + # Mock function for filename + def test_func(): + pass + + core.set_func(test_func) + + # Save with hash_str + test_entry = CacheEntry( + value="test_value", + time=datetime.now(), + stale=False, + _processing=False, + _completed=True, + ) + test_data = {"test_key": test_entry} + hash_str = "testhash123" + core._save_cache(test_data, hash_str=hash_str) + + # Check file exists with hash in name + expected_pattern = f"test_func_{hash_str}" + files = os.listdir(temp_dir) + assert any( + expected_pattern in f and f.endswith(hash_str) for f in files + ), f"Expected file ending with {hash_str} not found. Files: {files}" + + +# Test Pickle timeout during wait (line 398) +@pytest.mark.pickle +def test_pickle_timeout_during_wait(): + """Test calculation timeout while waiting in pickle backend.""" + import queue + import threading + + @cachier( + backend="pickle", + wait_for_calc_timeout=0.5, # Short timeout + ) + def slow_func(x): + sleep(2) # Longer than timeout + return x * 2 + + slow_func.clear_cache() + + res_queue = queue.Queue() + + def call_slow_func(): + try: + res = slow_func(42) + res_queue.put(("success", res)) + except Exception as e: + res_queue.put(("error", e)) + + # Start first thread that will take long + thread1 = threading.Thread(target=call_slow_func) + thread1.start() + + # Give it time to start processing + sleep(0.1) + + # Start second thread that should timeout waiting + thread2 = threading.Thread(target=call_slow_func) + thread2.start() + + # Wait for threads + thread1.join(timeout=3) + thread2.join(timeout=3) + + # Check results - at least one should have succeeded + results = [] + while not res_queue.empty(): + results.append(res_queue.get()) + + assert len(results) >= 1 + + slow_func.clear_cache() + + +# Test Pickle wait timeout check +@pytest.mark.pickle +def test_pickle_wait_timeout_check(): + """Test pickle backend timeout check during wait.""" + import threading + + @cachier(backend="pickle", wait_for_calc_timeout=0.2) + def slow_func(x): + sleep(1) # Longer than timeout + return x * 2 + + slow_func.clear_cache() + + results = [] + + def worker1(): + results.append(("w1", slow_func(42))) + + def worker2(): + sleep(0.1) # Let first start + results.append(("w2", slow_func(42))) + + t1 = threading.Thread(target=worker1) + t2 = threading.Thread(target=worker2) + + t1.start() + t2.start() + + t1.join(timeout=2) + t2.join(timeout=2) + + # Both should have results (timeout should have triggered recalc) + assert len(results) >= 1 + + slow_func.clear_cache() diff --git a/tests/test_redis_core.py b/tests/test_redis_core.py new file mode 100644 index 00000000..1ac8157c --- /dev/null +++ b/tests/test_redis_core.py @@ -0,0 +1,1268 @@ +"""Testing the Redis core of cachier.""" + +import contextlib +import hashlib +import pickle +import queue +import sys +import threading +import time +import warnings +from datetime import datetime, timedelta +from random import random +from time import sleep +from unittest.mock import MagicMock, Mock, patch + +import pandas as pd +import pytest +from birch import Birch # type: ignore[import-not-found] + +from cachier import cachier +from cachier.cores.redis import MissingRedisClient, _RedisCore + +# === Enables testing vs a real Redis instance === + +try: + import redis + + REDIS_AVAILABLE = True +except ImportError: + REDIS_AVAILABLE = False + + +class CfgKey: + HOST = "TEST_REDIS_HOST" + PORT = "TEST_REDIS_PORT" + DB = "TEST_REDIS_DB" + TEST_VS_DOCKERIZED_REDIS = "TEST_VS_DOCKERIZED_REDIS" + + +CFG = Birch( + namespace="cachier", + defaults={CfgKey.TEST_VS_DOCKERIZED_REDIS: False}, +) + + +def _get_test_redis_client(): + """Get a Redis client for testing.""" + if not REDIS_AVAILABLE: + pytest.skip("Redis not available") + + if str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_REDIS)).lower() == "true": + print("Using live Redis instance for testing.") + host = CFG.get(CfgKey.HOST, "localhost") + port = int(CFG.get(CfgKey.PORT, 6379)) + db = int(CFG.get(CfgKey.DB, 0)) + try: + client = redis.Redis( + host=host, port=port, db=db, decode_responses=False + ) + # Test connection + client.ping() + return client + except redis.ConnectionError as e: + print(f"Failed to connect to Redis: {e}") + pytest.skip("Redis not available") + else: + print("Using mock Redis for testing.") + # For testing without Redis, we'll use a mock + return None + + +def _test_redis_getter(): + """Get Redis client for testing.""" + client = _get_test_redis_client() + if client is None: + # Create a mock Redis client for testing + # Use a singleton pattern to ensure the same instance is returned + if not hasattr(_test_redis_getter, "_mock_client"): + + class MockRedis: + def __init__(self): + self.data = {} + print("DEBUG: MockRedis initialized") + + def hgetall(self, key): + result = self.data.get(key, {}) + # Convert string values to bytes to match Redis behavior + bytes_result = {} + for k, v in result.items(): + if isinstance(v, str): + bytes_result[k.encode("utf-8")] = v.encode("utf-8") + else: + bytes_result[k.encode("utf-8")] = v + print( + f"DEBUG: hgetall({key}) = {result} -> {bytes_result}" + ) + return bytes_result + + def hset( + self, key, field=None, value=None, mapping=None, **kwargs + ): + if key not in self.data: + self.data[key] = {} + + # Handle different calling patterns + if mapping is not None: + # Called with mapping dict + self.data[key].update(mapping) + elif field is not None and value is not None: + # Called with field, value arguments + self.data[key][field] = value + elif kwargs: + # Called with keyword arguments + self.data[key].update(kwargs) + + print( + f"DEBUG: hset({key}, field={field}, value={value}, " + f"mapping={mapping}, kwargs={kwargs}) -> " + f"{self.data[key]}" + ) + + def keys(self, pattern): + import re + + pattern = pattern.replace("*", ".*") + # Fix: keys are strings, not bytes, so no need to decode + result = [k for k in self.data if re.match(pattern, k)] + print(f"DEBUG: keys({pattern}) = {result}") + return result + + def delete(self, *keys): + for key in keys: + self.data.pop(key, None) + print(f"DEBUG: delete({keys})") + + def pipeline(self): + return MockPipeline(self) + + def ping(self): + return True + + def set(self, key, value): + self.data[key] = value + print(f"DEBUG: set({key}, {value})") + + def get(self, key): + result = self.data.get(key) + if isinstance(result, str): + result = result.encode("utf-8") + print(f"DEBUG: get({key}) = {result}") + return result + + class MockPipeline: + def __init__(self, redis_client): + self.redis_client = redis_client + self.commands = [] + + def hset(self, key, field, value): + self.commands.append(("hset", key, field, value)) + return self + + def execute(self): + for cmd, key, field, value in self.commands: + if cmd == "hset": + self.redis_client.hset( + key, field=field, value=value + ) + + _test_redis_getter._mock_client = MockRedis() + + return _test_redis_getter._mock_client + return client + + +# === Redis core tests === + + +@pytest.mark.redis +def test_information(): + if REDIS_AVAILABLE: + print(f"\nredis version: {redis.__version__}") + else: + print("\nredis not available") + + +@pytest.mark.redis +def test_redis_connection(): + """Test Redis connection with environment variables.""" + client = _get_test_redis_client() + if client is None: + pytest.skip("Redis not available") + + try: + # Test basic Redis operations + client.set("test_key", "test_value") + value = client.get("test_key") + assert value == b"test_value" + client.delete("test_key") + print("✓ Redis connection and basic operations working") + except Exception as e: + pytest.fail(f"Redis connection test failed: {e}") + + +@pytest.mark.redis +def test_redis_core(): + """Basic Redis core functionality.""" + + @cachier(backend="redis", redis_client=_test_redis_getter) + def _test_redis_caching(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _test_redis_caching.clear_cache() + val1 = _test_redis_caching(1, 2) + val2 = _test_redis_caching(1, 2) + assert val1 == val2 + val3 = _test_redis_caching(1, 2, cachier__skip_cache=True) + assert val3 != val1 + val4 = _test_redis_caching(1, 2) + assert val4 == val1 + val5 = _test_redis_caching(1, 2, cachier__overwrite_cache=True) + assert val5 != val1 + val6 = _test_redis_caching(1, 2) + assert val6 == val5 + + +@pytest.mark.redis +def test_redis_core_keywords(): + """Basic Redis core functionality with keyword arguments.""" + + @cachier(backend="redis", redis_client=_test_redis_getter) + def _tfunc_for_keywords(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _tfunc_for_keywords.clear_cache() + val1 = _tfunc_for_keywords(1, arg_2=2) + val2 = _tfunc_for_keywords(1, arg_2=2) + assert val1 == val2 + val3 = _tfunc_for_keywords(1, arg_2=2, cachier__skip_cache=True) + assert val3 != val1 + val4 = _tfunc_for_keywords(1, arg_2=2) + assert val4 == val1 + val5 = _tfunc_for_keywords(1, arg_2=2, cachier__overwrite_cache=True) + assert val5 != val1 + val6 = _tfunc_for_keywords(1, arg_2=2) + assert val6 == val5 + + +@pytest.mark.redis +def test_redis_stale_after(): + """Testing Redis core stale_after functionality.""" + + @cachier( + backend="redis", + redis_client=_test_redis_getter, + stale_after=timedelta(seconds=3), + next_time=False, + ) + def _stale_after_redis(arg_1, arg_2): + """Some function.""" + return random() + arg_1 + arg_2 + + _stale_after_redis.clear_cache() + val1 = _stale_after_redis(1, 2) + val2 = _stale_after_redis(1, 2) + assert val1 == val2 + sleep(3) + val3 = _stale_after_redis(1, 2) + assert val3 != val1 + + +def _calls_takes_time_redis(res_queue): + print("DEBUG: _calls_takes_time_redis started") + + @cachier(backend="redis", redis_client=_test_redis_getter) + def _takes_time(arg_1, arg_2): + """Some function.""" + print( + f"DEBUG: _calls_takes_time_redis._takes_time({arg_1}, {arg_2})" + " called" + ) + sleep(3) + result = random() + arg_1 + arg_2 + print( + f"DEBUG: _calls_takes_time_redis._takes_time({arg_1}, {arg_2}) " + f"returning {result}" + ) + return result + + print("DEBUG: _calls_takes_time_redis calling _takes_time(34, 82.3)") + res = _takes_time(34, 82.3) + print(f"DEBUG: _calls_takes_time_redis got result {res}, putting in queue") + res_queue.put(res) + print("DEBUG: _calls_takes_time_redis completed") + + +@pytest.mark.redis +def test_redis_being_calculated(): + """Testing Redis core handling of being calculated scenarios.""" + print("DEBUG: test_redis_being_calculated started") + + @cachier(backend="redis", redis_client=_test_redis_getter) + def _takes_time(arg_1, arg_2): + """Some function.""" + print(f"DEBUG: _takes_time({arg_1}, {arg_2}) called") + sleep(3) + result = random() + arg_1 + arg_2 + print(f"DEBUG: _takes_time({arg_1}, {arg_2}) returning {result}") + return result + + print("DEBUG: Clearing cache") + _takes_time.clear_cache() + res_queue = queue.Queue() + print("DEBUG: Starting thread1") + thread1 = threading.Thread( + target=_calls_takes_time_redis, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + print("DEBUG: Starting thread2") + thread2 = threading.Thread( + target=_calls_takes_time_redis, + kwargs={"res_queue": res_queue}, + daemon=True, + ) + print("DEBUG: Starting thread1") + thread1.start() + print("DEBUG: Sleeping 1 second") + sleep(1) + print("DEBUG: Starting thread2") + thread2.start() + print("DEBUG: Waiting for thread1 to join") + thread1.join() + print("DEBUG: Waiting for thread2 to join") + thread2.join() + print("DEBUG: Getting results from queue") + res1 = res_queue.get() + res2 = res_queue.get() + print(f"DEBUG: Results: res1={res1}, res2={res2}") + assert res1 == res2 + print("DEBUG: test_redis_being_calculated completed successfully") + + +@pytest.mark.redis +def test_redis_callable_hash_param(): + """Testing Redis core with callable hash function.""" + + def _hash_func(args, kwargs): + def _hash(obj): + if isinstance(obj, pd.DataFrame): + return hashlib.sha256( + obj.to_string().encode("utf-8") + ).hexdigest() + return str(obj) + + key_parts = [] + for arg in args: + key_parts.append(_hash(arg)) + for key, value in sorted(kwargs.items()): + key_parts.append(f"{key}:{_hash(value)}") + return hashlib.sha256(":".join(key_parts).encode("utf-8")).hexdigest() + + @cachier( + backend="redis", redis_client=_test_redis_getter, hash_func=_hash_func + ) + def _params_with_dataframe(*args, **kwargs): + """Function that can handle DataFrames.""" + return sum(len(str(arg)) for arg in args) + sum( + len(str(val)) for val in kwargs.values() + ) + + df1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + df2 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + df3 = pd.DataFrame({"a": [7, 8, 9], "b": [10, 11, 12]}) + + _params_with_dataframe.clear_cache() + val1 = _params_with_dataframe(df1, x=1) + val2 = _params_with_dataframe(df2, x=1) + assert val1 == val2 + val3 = _params_with_dataframe(df3, x=1) + assert val3 != val1 + + +@pytest.mark.redis +def test_redis_missing_client(): + """Test that MissingRedisClient is raised when no client is provided.""" + with pytest.raises(MissingRedisClient): + + @cachier(backend="redis") + def _test_func(): + return "test" + + +@pytest.mark.redis +def test_redis_core_direct(): + """Test Redis core directly.""" + redis_client = _test_redis_getter() + core = _RedisCore( + hash_func=None, + redis_client=redis_client, + wait_for_calc_timeout=None, + ) + + def test_func(x, y): + return x + y + + core.set_func(test_func) + + # Test setting and getting entries + core.set_entry("test_key", "test_value") + key, entry = core.get_entry_by_key("test_key") + assert entry is not None + assert entry.value == "test_value" + + # Test marking as being calculated + core.mark_entry_being_calculated("calc_key") + key, entry = core.get_entry_by_key("calc_key") + assert entry is not None + assert entry._processing is True + + # Test marking as not being calculated + core.mark_entry_not_calculated("calc_key") + key, entry = core.get_entry_by_key("calc_key") + assert entry is not None + assert entry._processing is False + + # Test clearing cache + core.clear_cache() + key, entry = core.get_entry_by_key("test_key") + assert entry is None + + +@pytest.mark.redis +def test_redis_callable_client(): + """Test Redis core with callable client.""" + + def get_redis_client(): + return _test_redis_getter() + + @cachier(backend="redis", redis_client=get_redis_client) + def _test_callable_client(arg_1, arg_2): + """Test function with callable Redis client.""" + return random() + arg_1 + arg_2 + + _test_callable_client.clear_cache() + val1 = _test_callable_client(1, 2) + val2 = _test_callable_client(1, 2) + assert val1 == val2 + + +def test_redis_import_warning(): + """Test that import warning is raised when redis is not available.""" + ptc = patch("cachier.cores.redis.REDIS_AVAILABLE", False) + with ptc, pytest.warns(ImportWarning, match="`redis` was not found"): + _RedisCore( + hash_func=None, + redis_client=Mock(), + wait_for_calc_timeout=None, + ) + + +@pytest.mark.redis +def test_missing_redis_client(): + """Test MissingRedisClient exception when redis_client is None.""" + with pytest.raises( + MissingRedisClient, match="must specify ``redis_client``" + ): + _RedisCore( + hash_func=None, + redis_client=None, + wait_for_calc_timeout=None, + ) + + +@pytest.mark.redis +def test_redis_core_exceptions(): + """Test exception handling in Redis core methods.""" + # Create a mock Redis client that raises exceptions + mock_client = MagicMock() + + # Configure all methods to raise exceptions + mock_client.hgetall = MagicMock( + side_effect=Exception("Redis connection error") + ) + mock_client.hset = MagicMock(side_effect=Exception("Redis write error")) + mock_client.keys = MagicMock(side_effect=Exception("Redis keys error")) + mock_client.delete = MagicMock(side_effect=Exception("Redis delete error")) + + core = _RedisCore( + hash_func=None, + redis_client=mock_client, + wait_for_calc_timeout=10, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Test get_entry_by_key exception handling + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + key, entry = core.get_entry_by_key("test_key") + assert key == "test_key" + assert entry is None + assert len(w) == 1 + assert "Redis get_entry_by_key failed" in str(w[0].message) + + # Test set_entry exception handling + # Mock the client to ensure it's not callable + test_mock_client = MagicMock() + test_mock_client.hset = MagicMock( + side_effect=Exception("Redis write error") + ) + + # Create a new core with this specific mock + test_core = _RedisCore( + hash_func=None, + redis_client=test_mock_client, + wait_for_calc_timeout=10, + ) + test_core.set_func(mock_func) + + # Override _should_store to return True + test_core._should_store = lambda x: True + + # Also need to mock _resolve_redis_client and _get_redis_key + test_core._resolve_redis_client = lambda: test_mock_client + test_core._get_redis_key = lambda key: f"test:{key}" + + with warnings.catch_warnings(record=True) as w2: + warnings.simplefilter("always") + result = test_core.set_entry("test_key", "test_value") + assert result is False + assert len(w2) == 1 + assert "Redis set_entry failed" in str(w2[0].message) + + # Mock _resolve_redis_client and _get_redis_key for the core + core._resolve_redis_client = lambda: mock_client + core._get_redis_key = lambda key: f"test:{key}" + + # Test mark_entry_being_calculated exception handling + with warnings.catch_warnings(record=True) as w3: + warnings.simplefilter("always") + core.mark_entry_being_calculated("test_key") + assert len(w3) == 1 + assert "Redis mark_entry_being_calculated failed" in str(w3[0].message) + + # Test mark_entry_not_calculated exception handling + with warnings.catch_warnings(record=True) as w4: + warnings.simplefilter("always") + core.mark_entry_not_calculated("test_key") + assert len(w4) == 1 + assert "Redis mark_entry_not_calculated failed" in str(w4[0].message) + + # Test clear_cache exception handling + with warnings.catch_warnings(record=True) as w5: + warnings.simplefilter("always") + core.clear_cache() + assert len(w5) == 1 + assert "Redis clear_cache failed" in str(w5[0].message) + + # Test clear_being_calculated exception handling + with warnings.catch_warnings(record=True) as w6: + warnings.simplefilter("always") + core.clear_being_calculated() + assert len(w6) == 1 + assert "Redis clear_being_calculated failed" in str(w6[0].message) + + +@pytest.mark.redis +def test_redis_delete_stale_entries(): + """Test delete_stale_entries method with various scenarios.""" + mock_client = MagicMock() + + core = _RedisCore( + hash_func=None, + redis_client=mock_client, + wait_for_calc_timeout=10, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Test normal operation + # Create a new mock client for this test + delete_mock_client = MagicMock() + + # Set up keys method + delete_mock_client.keys = MagicMock( + return_value=[b"key1", b"key2", b"key3"] + ) + + now = datetime.now() + old_timestamp = (now - timedelta(hours=2)).isoformat() + recent_timestamp = (now - timedelta(minutes=30)).isoformat() + + # Set up hmget responses + delete_mock_client.hmget = MagicMock( + side_effect=[ + [old_timestamp.encode("utf-8"), b"100"], # key1 - stale + [recent_timestamp.encode("utf-8"), b"100"], # key2 - not stale + [None, None], # key3 - no timestamp + ] + ) + + # Set up delete mock + delete_mock_client.delete = MagicMock() + + # Create a new core for this test + delete_core = _RedisCore( + hash_func=None, + redis_client=delete_mock_client, + wait_for_calc_timeout=10, + ) + delete_core.set_func(mock_func) + + # Need to mock _resolve_redis_client to return our mock + delete_core._resolve_redis_client = lambda: delete_mock_client + + delete_core.delete_stale_entries(timedelta(hours=1)) + + # Should only delete key1 + assert delete_mock_client.delete.call_count == 1 + delete_mock_client.delete.assert_called_with(b"key1") + + # Test exception during timestamp parsing + mock_client.reset_mock() + mock_client.keys.return_value = [b"key4"] + mock_client.hmget.return_value = [b"invalid-timestamp", None] + + # Need to mock _resolve_redis_client for the original core as well + core._resolve_redis_client = lambda: mock_client + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + core.delete_stale_entries(timedelta(hours=1)) + assert len(w) == 1 + assert "Redis timestamp parse failed" in str(w[0].message) + + # Test exception during keys operation + mock_client.reset_mock() + mock_client.keys.side_effect = Exception("Redis keys error") + + with warnings.catch_warnings(record=True) as w2: + warnings.simplefilter("always") + core.delete_stale_entries(timedelta(hours=1)) + assert len(w2) == 1 + assert "Redis delete_stale_entries failed" in str(w2[0].message) + + +@pytest.mark.redis +def test_redis_wait_on_entry_calc_no_entry(): + """Test wait_on_entry_calc when entry is None.""" + from cachier.cores.base import RecalculationNeeded + + # Create a mock client + mock_client = MagicMock() + + # Mock get_entry_by_key to always return None entry + # This avoids the pickle.loads issue + _ = _RedisCore.get_entry_by_key + + def mock_get_entry_by_key(self, key): + return key, None + + core = _RedisCore( + hash_func=None, + redis_client=mock_client, + wait_for_calc_timeout=10, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Patch the method + core.get_entry_by_key = lambda key: mock_get_entry_by_key(core, key) + + # The test expects RecalculationNeeded to be raised when no entry exists + with pytest.raises(RecalculationNeeded): + core.wait_on_entry_calc("test_key") + + +@pytest.mark.redis +def test_redis_set_entry_should_not_store(): + """Test set_entry when value should not be stored (None not allowed).""" + mock_client = MagicMock() + + core = _RedisCore( + hash_func=None, + redis_client=mock_client, + wait_for_calc_timeout=10, + ) + + # Mock _should_store to return False + core._should_store = Mock(return_value=False) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + result = core.set_entry("test_key", None) + assert result is False + mock_client.hset.assert_not_called() + + +@pytest.mark.redis +def test_redis_clear_being_calculated_with_pipeline(): + """Test clear_being_calculated with multiple keys.""" + # Create fresh mocks for this test + pipeline_mock_client = MagicMock() + pipeline_mock = MagicMock() + + # Set up keys to return 3 keys + pipeline_mock_client.keys = MagicMock( + return_value=[b"key1", b"key2", b"key3"] + ) + + # Set up pipeline + pipeline_mock_client.pipeline = MagicMock(return_value=pipeline_mock) + pipeline_mock.hset = MagicMock() + pipeline_mock.execute = MagicMock() + + core = _RedisCore( + hash_func=None, + redis_client=pipeline_mock_client, + wait_for_calc_timeout=10, + ) + + # Set a mock function + def mock_func(): + pass + + core.set_func(mock_func) + + # Need to mock _resolve_redis_client to return our mock + core._resolve_redis_client = lambda: pipeline_mock_client + + core.clear_being_calculated() + + # Verify pipeline was used + assert pipeline_mock.hset.call_count == 3 + # Verify hset was called with correct parameters for each key + pipeline_mock.hset.assert_any_call(b"key1", "processing", "false") + pipeline_mock.hset.assert_any_call(b"key2", "processing", "false") + pipeline_mock.hset.assert_any_call(b"key3", "processing", "false") + pipeline_mock.execute.assert_called_once() + + +# Test Redis import error handling (lines 14-15) +def test_redis_import_error_handling(): + """Test Redis backend when redis package is not available.""" + # This test is already covered by test_redis_import_warning + # but let's ensure the specific lines are hit + with patch.dict(sys.modules, {"redis": None}): + # Force reload of redis core module + if "cachier.cores.redis" in sys.modules: + del sys.modules["cachier.cores.redis"] + + # Test import failure + try: + from cachier.cores.redis import _RedisCore # noqa: F401 + + pytest.skip("Redis is installed, cannot test import error") + except ImportError: + pass # Expected behavior + + +# Test Redis corrupted entry handling (lines 112-114) +@pytest.mark.redis +def test_redis_corrupted_entry_handling(): + """Test Redis backend with corrupted cache entries.""" + import redis + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + + try: + # Test connection + client.ping() + except redis.ConnectionError: + pytest.skip("Redis server not available") + + @cachier(backend="redis", redis_client=client) + def test_func(x): + return x * 2 + + # Clear cache + test_func.clear_cache() + + # Manually insert corrupted data + cache_key = "cachier:test_coverage_gaps:test_func:somehash" + client.hset(cache_key, "value", b"corrupted_pickle_data") + client.hset(cache_key, "time", str(time.time()).encode()) + client.hset(cache_key, "stale", b"0") + client.hset(cache_key, "being_calculated", b"0") + + # Try to access - should handle corrupted data gracefully + result = test_func(42) + assert result == 84 + + test_func.clear_cache() + + +# TestRedis deletion failure during eviction (lines 133-135) +@pytest.mark.redis +def test_redis_deletion_failure_during_eviction(): + """Test Redis LRU eviction with deletion failures.""" + import redis + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + + try: + client.ping() + except redis.ConnectionError: + pytest.skip("Redis server not available") + + @cachier( + backend="redis", + redis_client=client, + cache_size_limit="100B", # Very small limit to trigger eviction + ) + def test_func(x): + return "x" * 50 # Large result to fill cache quickly + + # Clear cache + test_func.clear_cache() + + # Fill cache to trigger eviction + test_func(1) + + # Mock delete to fail + original_delete = client.delete + delete_called = [] + + def mock_delete(*args): + delete_called.append(args) + # Fail on first delete attempt + if len(delete_called) == 1: + raise redis.RedisError("Mocked deletion failure") + return original_delete(*args) + + client.delete = mock_delete + + try: + # This should trigger eviction and handle the deletion failure + test_func(2) + # Verify delete was attempted + assert len(delete_called) > 0 + finally: + client.delete = original_delete + test_func.clear_cache() + + +# Test Redis non-bytes timestamp handling (line 364) +@pytest.mark.redis +def test_redis_non_bytes_timestamp(): + """Test Redis backend with non-bytes timestamp values.""" + import redis + + from cachier.cores.redis import _RedisCore + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + + try: + client.ping() + except redis.ConnectionError: + pytest.skip("Redis server not available") + + @cachier( + backend="redis", redis_client=client, stale_after=timedelta(seconds=10) + ) + def test_func(x): + return x * 2 + + # Clear cache + test_func.clear_cache() + + # Create an entry + test_func(1) + + # Manually modify timestamp to be a string instead of bytes + keys = list( + client.scan_iter(match="cachier:test_coverage_gaps:test_func:*") + ) + if keys: + # Force timestamp to be a string (non-bytes) + client.hset(keys[0], "time", "not_a_number") + + # Create a separate core instance to test stale deletion + core = _RedisCore( + hash_func=None, + redis_client=client, + wait_for_calc_timeout=0, + ) + core.set_func(test_func) + + # Try to delete stale entries - should handle non-bytes timestamp + # gracefully + with contextlib.suppress(Exception): + core.delete_stale_entries(timedelta(seconds=1)) + + test_func.clear_cache() + + +# Test Redis missing import +@pytest.mark.redis +def test_redis_import_error(): + """Test Redis client initialization warning.""" + # Test creating a Redis core without providing a client + import warnings + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + + with pytest.raises(Exception, match="redis_client"): + + @cachier(backend="redis", redis_client=None) + def test_func(): + return "test" + + +# Test Redis corrupted entry in LRU eviction +@pytest.mark.redis +def test_redis_lru_corrupted_entry(): + """Test Redis LRU eviction with corrupted entry.""" + import redis + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + try: + client.ping() + except redis.ConnectionError: + pytest.skip("Redis not available") + + @cachier( + backend="redis", + redis_client=client, + cache_size_limit="200B", # Small limit + ) + def test_func(x): + return f"result_{x}" * 10 # ~60 bytes per entry + + test_func.clear_cache() + + # Add valid entry + test_func(1) + + # Add corrupted entry manually + from cachier.cores.redis import _RedisCore + + core = _RedisCore( + hash_func=None, + redis_client=client, + wait_for_calc_timeout=0, + cache_size_limit="200B", + ) + core.set_func(test_func) + + # Create corrupted entry + bad_key = f"{core.key_prefix}:{core._func_str}:badkey" + client.hset(bad_key, "value", b"not_valid_pickle") + client.hset(bad_key, "time", str(time.time()).encode()) + client.hset(bad_key, "stale", b"0") + client.hset(bad_key, "being_calculated", b"0") + + # This should trigger eviction and handle the corrupted entry + test_func(2) + test_func(3) + + test_func.clear_cache() + + +# Test Redis deletion failure in eviction +@pytest.mark.redis +def test_redis_eviction_delete_failure(): + """Test Redis eviction handling delete failures.""" + import warnings + + import redis + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + try: + client.ping() + except redis.ConnectionError: + pytest.skip("Redis not available") + + # Create a unique function to avoid conflicts + @cachier(backend="redis", redis_client=client, cache_size_limit="150B") + def test_eviction_func(x): + return "x" * 50 # Large value + + test_eviction_func.clear_cache() + + # Fill cache to trigger eviction + test_eviction_func(100) + + # This should trigger eviction + with warnings.catch_warnings(record=True): + # Ignore warnings about eviction failures + warnings.simplefilter("always") + test_eviction_func(200) + + # Verify both values work (even if eviction had issues) + result1 = test_eviction_func(100) + result2 = test_eviction_func(200) + + assert result1 == "x" * 50 + assert result2 == "x" * 50 + + test_eviction_func.clear_cache() + + +# Test Redis stale deletion with size tracking +@pytest.mark.redis +def test_redis_stale_delete_size_tracking(): + """Test Redis stale deletion updates cache size.""" + import redis + + client = redis.Redis(host="localhost", port=6379, decode_responses=False) + try: + client.ping() + except redis.ConnectionError: + pytest.skip("Redis not available") + + @cachier( + backend="redis", + redis_client=client, + cache_size_limit="1KB", + stale_after=timedelta(seconds=0.1), + ) + def test_func(x): + return "data" * 20 + + test_func.clear_cache() + + # Create entries + test_func(1) + test_func(2) + + # Wait for staleness + sleep(0.2) + + # Get the core + from cachier.cores.redis import _RedisCore + + core = _RedisCore( + hash_func=None, + redis_client=client, + wait_for_calc_timeout=0, + cache_size_limit="1KB", + ) + core.set_func(test_func) + + # Delete stale entries - this should update cache size + core.delete_stale_entries(timedelta(seconds=0.1)) + + # Verify size tracking by adding new entry + test_func(3) + + test_func.clear_cache() + + +@pytest.mark.redis +def test_redis_lru_eviction_edge_cases(): + """Test Redis LRU eviction edge cases for coverage.""" + from cachier.cores.redis import _RedisCore + + redis_client = _test_redis_getter() + + # Test 1: Corrupted data during LRU eviction (lines 112-114) + core = _RedisCore( + hash_func=None, redis_client=redis_client, cache_size_limit=100 + ) + + def mock_func(x): + return x * 2 + + core.set_func(mock_func) + + # Add entries with corrupted metadata + for i in range(3): + key = core._get_redis_key(f"key{i}") + redis_client.hset(key, "value", pickle.dumps(i * 2)) + redis_client.hset( + key, "time", pickle.dumps(datetime.now().timestamp()) + ) + if i == 1: + # Corrupt metadata for one entry + redis_client.hset(key, "last_access", "invalid_json") + redis_client.hset(key, "size", "not_a_number") + else: + redis_client.hset(key, "last_access", str(time.time())) + redis_client.hset(key, "size", "20") + + # Set high cache size to trigger eviction + redis_client.set(core._cache_size_key, "1000") + + # Should handle corrupted entries gracefully + core._evict_lru_entries(redis_client, 1000) + + # Test 2: No eviction needed (line 138) + # Clear and set very low cache size + pattern = f"{core.key_prefix}:{core._func_str}:*" + for key in redis_client.scan_iter(match=pattern): + if b"__size__" not in key: + redis_client.delete(key) + + redis_client.set(core._cache_size_key, "10") + # Should not evict anything + core._evict_lru_entries(redis_client, 10) + + +@pytest.mark.redis +def test_redis_clear_and_delete_edge_cases(): + """Test Redis clear and delete operations edge cases.""" + from cachier.cores.redis import _RedisCore + + redis_client = _test_redis_getter() + + # Test 1: clear_being_calculated with no keys (line 325) + core = _RedisCore(hash_func=None, redis_client=redis_client) + + def mock_func(): + pass + + core.set_func(mock_func) + + # Ensure no keys exist + pattern = f"{core.key_prefix}:{core._func_str}:*" + for key in redis_client.scan_iter(match=pattern): + redis_client.delete(key) + + # Should handle empty key set gracefully + core.clear_being_calculated() + + # Test 2: delete_stale_entries with special keys (line 352) + core2 = _RedisCore(hash_func=None, redis_client=redis_client) + core2.stale_after = timedelta(seconds=1) + + def mock_func2(): + pass + + core2.set_func(mock_func2) + + # Add stale entries + for i in range(2): + key = core2._get_redis_key(f"entry{i}") + redis_client.hset(key, "value", pickle.dumps(f"value{i}")) + redis_client.hset( + key, + "timestamp", + (datetime.now() - timedelta(seconds=2)).isoformat(), + ) + + # Add special cache size key + redis_client.set(core2._cache_size_key, "100") + + # Delete stale - should skip special keys + core2.delete_stale_entries(timedelta(seconds=1)) + + # Special key should still exist + assert redis_client.exists(core2._cache_size_key) + + # Test 3: Non-bytes timestamp (line 364) + key = core2._get_redis_key("nonbytes") + redis_client.hset(key, "value", pickle.dumps("test")) + # String timestamp instead of bytes + redis_client.hset( + key, + "timestamp", + str((datetime.now() - timedelta(seconds=2)).isoformat()), + ) + + core2.delete_stale_entries(timedelta(seconds=1)) + # Should handle string timestamp + assert not redis_client.exists(key) + + +@pytest.mark.redis +def test_redis_delete_stale_size_handling(): + """Test Redis delete_stale_entries size handling.""" + from cachier.cores.redis import _RedisCore + + redis_client = _test_redis_getter() + + # Test 1: Corrupted size data (lines 374-375) + core = _RedisCore( + hash_func=None, redis_client=redis_client, cache_size_limit=1000 + ) + core.stale_after = timedelta(seconds=1) + + def mock_func(): + pass + + core.set_func(mock_func) + + # Add entries with one having corrupted size + for i in range(3): + key = core._get_redis_key(f"item{i}") + value = pickle.dumps(f"result{i}") + redis_client.hset(key, "value", value) + redis_client.hset( + key, + "time", + pickle.dumps((datetime.now() - timedelta(seconds=2)).timestamp()), + ) + if i == 1: + redis_client.hset(key, "size", "invalid_size") + else: + redis_client.hset(key, "size", str(len(value))) + + # Should handle corrupted size gracefully + core.delete_stale_entries(timedelta(seconds=1)) + + # Test 2: No cache_size_limit (line 380) + core2 = _RedisCore(hash_func=None, redis_client=redis_client) + core2.stale_after = timedelta(seconds=1) + core2.cache_size_limit = None + + def mock_func2(): + pass + + core2.set_func(mock_func2) + + # Add stale entries + for i in range(2): + key = core2._get_redis_key(f"old{i}") + redis_client.hset(key, "value", pickle.dumps(f"old{i}")) + redis_client.hset( + key, + "time", + pickle.dumps((datetime.now() - timedelta(seconds=2)).timestamp()), + ) + redis_client.hset(key, "size", "50") + + core2.delete_stale_entries(timedelta(seconds=1)) + + # Test 3: Nothing to delete (line 380) + core3 = _RedisCore( + hash_func=None, redis_client=redis_client, cache_size_limit=1000 + ) + core3.stale_after = timedelta(days=1) + + def mock_func3(): + pass + + core3.set_func(mock_func3) + + # Add fresh entries + for i in range(2): + key = core3._get_redis_key(f"fresh{i}") + redis_client.hset(key, "value", pickle.dumps(f"fresh{i}")) + redis_client.hset( + key, "time", pickle.dumps(datetime.now().timestamp()) + ) + redis_client.hset(key, "size", "30") + + # Nothing should be deleted + core3.delete_stale_entries(timedelta(days=1)) diff --git a/tests/test_sql_core.py b/tests/test_sql_core.py new file mode 100644 index 00000000..a7fad33d --- /dev/null +++ b/tests/test_sql_core.py @@ -0,0 +1,537 @@ +import os +import queue +import sys +import threading +from datetime import datetime, timedelta +from random import random +from time import sleep + +import pytest + +from cachier import cachier +from cachier.cores.base import RecalculationNeeded, _get_func_str +from cachier.cores.sql import _SQLCore + +SQL_CONN_STR = os.environ.get("SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:") + + +@pytest.mark.sql +def test_sql_core_basic(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, 2) + v2 = f(1, 2) + assert v1 == v2 + v3 = f(1, 2, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, 2) + assert v4 == v1 + v5 = f(1, 2, cachier__overwrite_cache=True) + assert v5 != v1 + v6 = f(1, 2) + assert v6 == v5 + + +@pytest.mark.sql +def test_sql_core_keywords(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, y=2) + v2 = f(1, y=2) + assert v1 == v2 + v3 = f(1, y=2, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, y=2) + assert v4 == v1 + v5 = f(1, y=2, cachier__overwrite_cache=True) + assert v5 != v1 + v6 = f(1, y=2) + assert v6 == v5 + + +@pytest.mark.sql +def test_sql_stale_after(): + @cachier( + backend="sql", + sql_engine=SQL_CONN_STR, + stale_after=timedelta(seconds=2), + next_time=False, + ) + def f(x, y): + return random() + x + y + + f.clear_cache() + v1 = f(1, 2) + v2 = f(1, 2) + assert v1 == v2 + sleep(2) + v3 = f(1, 2) + assert v3 != v1 + + +@pytest.mark.sql +def test_sql_overwrite_and_skip_cache(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return random() + x + + f.clear_cache() + v1 = f(1) + v2 = f(1) + assert v1 == v2 + v3 = f(1, cachier__skip_cache=True) + assert v3 != v1 + v4 = f(1, cachier__overwrite_cache=True) + assert v4 != v1 + v5 = f(1) + assert v5 == v4 + + +@pytest.mark.sql +def test_sql_concurrency(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def slow_func(x): + sleep(1) + return random() + x + + slow_func.clear_cache() + res_queue = queue.Queue() + + def call(): + res = slow_func(5) + res_queue.put(res) + + t1 = threading.Thread(target=call) + t2 = threading.Thread(target=call) + t1.start() + sleep(0.2) + t2.start() + t1.join(timeout=3) + t2.join(timeout=3) + assert res_queue.qsize() == 2 + r1 = res_queue.get() + r2 = res_queue.get() + assert r1 == r2 + + +@pytest.mark.sql +def test_sql_clear_being_calculated(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def slow_func(x): + sleep(1) + return random() + x + + slow_func.clear_cache() + slow_func(1) + slow_func.clear_being_calculated() + # Should not raise + slow_func(1) + + +@pytest.mark.sql +def test_sql_missing_entry(): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return x + + f.clear_cache() + # Should not raise + assert f(123) == 123 + + +class DummyWriteError(Exception): + pass + + +@pytest.mark.sql +def test_sql_failed_write(monkeypatch): + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def f(x): + return x + + f.clear_cache() + # Simulate DB failure by monkeypatching set_entry + orig = _SQLCore.set_entry + + def fail_set_entry(self, key, func_res): + raise DummyWriteError("fail") + + monkeypatch.setattr(_SQLCore, "set_entry", fail_set_entry) + with pytest.raises(DummyWriteError, match="fail"): + f(1) + monkeypatch.setattr(_SQLCore, "set_entry", orig) + + +@pytest.mark.sql +def test_import_cachier_without_sqlalchemy(monkeypatch): + """Test that importing cachier works when SQLAlchemy is missing. + + This should work unless SQL core is used. + + """ + # Simulate SQLAlchemy not installed + modules_backup = sys.modules.copy() + sys.modules["sqlalchemy"] = None + sys.modules["sqlalchemy.orm"] = None + sys.modules["sqlalchemy.engine"] = None + try: + import importlib # noqa: F401 + + import cachier # noqa: F401 + + # Should import fine + finally: + sys.modules.clear() + sys.modules.update(modules_backup) + + +@pytest.mark.sql +def test_sqlcore_importerror_without_sqlalchemy(monkeypatch): + """Test that using SQL core without SQLAlchemy raises an ImportError.""" + # Remove sql module from sys.modules to force reimport + if "cachier.cores.sql" in sys.modules: + del sys.modules["cachier.cores.sql"] + + # Mock the sqlalchemy import to raise ImportError + import builtins + + original_import = builtins.__import__ + + def mock_import(name, *args, **kwargs): + if name.startswith("sqlalchemy"): + raise ImportError(f"No module named '{name}'") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mock_import) + + try: + # Now import sql - it should set SQLALCHEMY_AVAILABLE = False + import importlib + + sql_mod = importlib.import_module("cachier.cores.sql") + + # Verify that SQLALCHEMY_AVAILABLE is False + assert not sql_mod.SQLALCHEMY_AVAILABLE + + # Now trying to create _SQLCore should raise ImportError + with pytest.raises(ImportError) as excinfo: + sql_mod._SQLCore(hash_func=None, sql_engine="sqlite:///:memory:") + assert "SQLAlchemy is required" in str(excinfo.value) + finally: + # Clean up - remove the module so next tests reimport it fresh + if "cachier.cores.sql" in sys.modules: + del sys.modules["cachier.cores.sql"] + + +@pytest.mark.sql +def test_sqlcore_invalid_sql_engine(): + with pytest.raises( + ValueError, match="sql_engine must be a SQLAlchemy Engine" + ): + _SQLCore(hash_func=None, sql_engine=12345) + + +@pytest.mark.sql +def test_sqlcore_get_entry_by_key_none_value(): + import pytest + + pytest.importorskip("sqlalchemy") + import cachier.cores.sql as sql_mod + from cachier.cores.sql import _SQLCore + + CacheTable = getattr(sql_mod, "CacheTable", None) + if CacheTable is None: + pytest.skip("CacheTable not available (SQLAlchemy missing)") + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Insert a row with value=None + with core._Session() as session: + session.add( + CacheTable( + id="testfunc:abc", + function_id=core._func_str, + key="abc", + value=None, + timestamp=datetime.now(), + stale=False, + processing=False, + completed=True, + ) + ) + session.commit() + key, entry = core.get_entry_by_key("abc") + assert entry is not None + assert entry.value is None + + +@pytest.mark.sql +def test_sqlcore_set_entry_fallback(monkeypatch): + from sqlalchemy.orm import Session + from sqlalchemy.sql.dml import Insert, Update + from sqlalchemy.sql.selectable import Select + + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Monkeypatch Session.execute to simulate fallback path + orig_execute = Session.execute + + def fake_execute(self, stmt, *args, **kwargs): + if isinstance(stmt, (Insert, Update)): + + class FakeInsert: + pass + + return FakeInsert() + elif isinstance(stmt, Select): + + class FakeSelectResult: + def scalar_one_or_none(self): + return None # Simulate no row found + + return FakeSelectResult() + + class Dummy: + pass + + return Dummy() + + monkeypatch.setattr(Session, "execute", fake_execute) + # Should not raise + core.set_entry("fallback", 123) + monkeypatch.setattr(Session, "execute", orig_execute) + + +@pytest.mark.sql +def test_sqlcore_wait_on_entry_calc_recalculation(): + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + with pytest.raises(RecalculationNeeded): + core.wait_on_entry_calc("missing_key") + + +@pytest.mark.sql +def test_sqlcore_clear_being_calculated_empty(): + core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) + core.set_func(lambda x: x) + # Should not raise even if nothing is being calculated + core.clear_being_calculated() + + +@pytest.mark.sql +def test_sqlcore_accepts_engine_instance(): + from sqlalchemy import create_engine + + engine = create_engine(SQL_CONN_STR) + core = _SQLCore(hash_func=None, sql_engine=engine) + core.set_func(lambda x: x) + core.set_entry("engine_test", 456) + key, entry = core.get_entry_by_key("engine_test") + assert entry.value == 456 + + +@pytest.mark.sql +def test_sqlcore_accepts_engine_callable(): + from sqlalchemy import create_engine + + def engine_factory(): + return create_engine(SQL_CONN_STR) + + core = _SQLCore(hash_func=None, sql_engine=engine_factory) + core.set_func(lambda x: x) + core.set_entry("callable_test", 789) + key, entry = core.get_entry_by_key("callable_test") + assert entry.value == 789 + + +# Test SQL allow_none=False +@pytest.mark.sql +def test_sql_allow_none_false_not_stored(): + """Test SQL doesn't store None when allow_none=False.""" + SQL_CONN_STR = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" + ) + call_count = 0 + + @cachier(backend="sql", sql_engine=SQL_CONN_STR, allow_none=False) + def returns_none(): + nonlocal call_count + call_count += 1 + return None + + returns_none.clear_cache() + + # First call + result1 = returns_none() + assert result1 is None + assert call_count == 1 + + # Second call should also execute + result2 = returns_none() + assert result2 is None + assert call_count == 2 + + returns_none.clear_cache() + + +# Test SQL delete_stale_entries direct call +@pytest.mark.sql +def test_sql_delete_stale_direct(): + """Test SQL stale entry deletion method.""" + from cachier.cores.sql import _SQLCore + + # Get the engine from environment or use default + SQL_CONN_STR = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" + ) + + @cachier( + backend="sql", + sql_engine=SQL_CONN_STR, + stale_after=timedelta(seconds=0.5), + ) + def test_func(x): + return x * 2 + + test_func.clear_cache() + + # Create entries + test_func(1) + test_func(2) + + # Wait for staleness + sleep(0.6) + + # Create core instance for direct testing + core = _SQLCore( + hash_func=None, + sql_engine=SQL_CONN_STR, + wait_for_calc_timeout=0, + ) + core.set_func(test_func) + + # Delete stale entries + core.delete_stale_entries(timedelta(seconds=0.5)) + + test_func.clear_cache() + + +# Test Non-standard SQL database fallback +@pytest.mark.sql +def test_sql_non_standard_db(): + """Test SQL backend code coverage for set_entry method.""" + # This test improves coverage for the SQL set_entry method + SQL_CONN_STR = os.environ.get( + "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" + ) + + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def test_func(x): + return x * 3 + + test_func.clear_cache() + + # Test basic set/get functionality + result1 = test_func(10) + assert result1 == 30 + + # Test overwriting existing entry + result2 = test_func(10, cachier__overwrite_cache=True) + assert result2 == 30 + + # Test with None value when allow_none is True (default) + @cachier(backend="sql", sql_engine=SQL_CONN_STR, allow_none=True) + def returns_none_allowed(): + return None + + returns_none_allowed.clear_cache() + result3 = returns_none_allowed() + assert result3 is None + + # Second call should use cache + result4 = returns_none_allowed() + assert result4 is None + + test_func.clear_cache() + returns_none_allowed.clear_cache() + + +@pytest.mark.sql +def test_sql_should_store_false(): + """Test SQL set_entry when _should_store returns False (line 128).""" + from cachier.cores.sql import _SQLCore + + # Create core with entry size limit + core = _SQLCore( + sql_engine=SQL_CONN_STR, hash_func=None, entry_size_limit=10 + ) + + def mock_func(x): + return x + + core.set_func(mock_func) + + # Create a large object that exceeds the size limit + large_object = "x" * 1000 # Much larger than 10 bytes + + # set_entry with large object should return False + result = core.set_entry("test_key", large_object) + assert result is False + + +@pytest.mark.sql +def test_sql_on_conflict_do_update(): + """Test SQL on_conflict_do_update path (line 158).""" + # When running with PostgreSQL, this will test the + # on_conflict_do_update path + # With SQLite in memory, it will also support on_conflict_do_update + + @cachier(backend="sql", sql_engine=SQL_CONN_STR) + def test_func(x): + return x * 2 + + test_func.clear_cache() + + # First call + result1 = test_func(5) + assert result1 == 10 + + # Force an update scenario by marking stale + if "postgresql" in SQL_CONN_STR or "sqlite" in SQL_CONN_STR: + # Direct table manipulation to force update path + from sqlalchemy import create_engine, update + from sqlalchemy.orm import sessionmaker + + from cachier.cores.sql import CacheTable + + engine = create_engine(SQL_CONN_STR) + Session = sessionmaker(bind=engine) + session = Session() + + func_str = _get_func_str(test_func) + + # Mark as stale to force update + stmt = ( + update(CacheTable) + .where(CacheTable.function_id == func_str) + .values(stale=True) + ) + + try: + session.execute(stmt) + session.commit() + except Exception: + # If table doesn't exist or other issue, skip + # This is expected in some test configurations + pass + finally: + session.close() + + # Second call - will use on_conflict_do_update + result2 = test_func(5) + assert result2 == 10 From 12bd8580d912a1990551d458d8204aa31ff941f8 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Sat, 28 Feb 2026 23:03:38 +0200 Subject: [PATCH 02/22] clean up --- pyproject.toml | 9 +- tests/test_mongo_core.py | 775 ---------------------- tests/test_pickle_core.py | 1304 ------------------------------------- tests/test_redis_core.py | 1268 ------------------------------------ tests/test_sql_core.py | 537 --------------- 5 files changed, 3 insertions(+), 3890 deletions(-) delete mode 100644 tests/test_mongo_core.py delete mode 100644 tests/test_pickle_core.py delete mode 100644 tests/test_redis_core.py delete mode 100644 tests/test_sql_core.py diff --git a/pyproject.toml b/pyproject.toml index bec7a337..2e68cd1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,7 +90,7 @@ namespaces = false # to disable scanning PEP 420 namespaces (true by default) # === Linting & Formatting === [tool.black] -line-length = 79 +line-length = 120 # --- ruff --- @@ -168,8 +168,8 @@ lint.mccabe.max-complexity = 10 [tool.docformatter] recursive = true # some docstring start with r""" -wrap-summaries = 79 -wrap-descriptions = 79 +wrap-summaries = 120 +wrap-descriptions = 120 blank = true # === Testing === @@ -201,12 +201,9 @@ markers = [ "sql: test the SQL core", "s3: test the S3 core", "maxage: test the max_age functionality", -<<<<<<< HEAD "asyncio: marks tests as async", "smoke: fast smoke tests with no external service dependencies", -======= "seriallocal: local core tests that should run serially", ->>>>>>> 5116bb0 (Improve test infrastructure for parallel execution and better isolation) ] # Parallel test execution configuration diff --git a/tests/test_mongo_core.py b/tests/test_mongo_core.py deleted file mode 100644 index 4dbd7119..00000000 --- a/tests/test_mongo_core.py +++ /dev/null @@ -1,775 +0,0 @@ -"""Testing the MongoDB core of cachier.""" - -# standard library imports -import datetime -import hashlib -import platform -import queue -import sys -import threading -from datetime import timedelta -from random import random -from time import sleep, time -from urllib.parse import quote_plus - -# third-party imports -import pytest -from birch import Birch # type: ignore[import-not-found] - -try: - import pandas as pd -except (ImportError, ModuleNotFoundError): - pd = None - print("pandas is not installed; tests requiring pandas will fail!") - -try: - import pymongo - from pymongo.errors import OperationFailure - from pymongo.mongo_client import MongoClient - - from cachier.cores.mongo import MissingMongetter -except (ImportError, ModuleNotFoundError): - print("pymongo is not installed; tests requiring pymongo will fail!") - pymongo = None - OperationFailure = None - MissingMongetter = None - - # define a mock MongoClient class that will raise an exception - # on init, warning that pymongo is not installed - class MongoClient: - """Mock MongoClient class raising ImportError on missing pymongo.""" - - def __init__(self, *args, **kwargs): - """Initialize the mock MongoClient.""" - raise ImportError("pymongo is not installed!") - - -try: - from pymongo_inmemory import MongoClient as InMemoryMongoClient -except (ImportError, ModuleNotFoundError): - - class InMemoryMongoClient: - """Mock InMemoryMongoClient class. - - Raises an ImportError on missing pymongo_inmemory. - - """ - - def __init__(self, *args, **kwargs): - """Initialize the mock InMemoryMongoClient.""" - raise ImportError("pymongo_inmemory is not installed!") - - print( - "pymongo_inmemory is not installed; in-memory MongoDB tests will fail!" - ) - -# local imports -from cachier import cachier -from cachier.config import CacheEntry -from cachier.cores.base import RecalculationNeeded -from cachier.cores.mongo import _MongoCore - -# === Enables testing vs a real MongoDB instance === - - -class CfgKey: - HOST = "TEST_HOST" - PORT = "TEST_PORT" - # UNAME = "TEST_USERNAME" - # PWD = "TEST_PASSWORD" - # DB = "TEST_DB" - TEST_VS_DOCKERIZED_MONGO = "TEST_VS_DOCKERIZED_MONGO" - - -CFG = Birch( - namespace="cachier", - defaults={CfgKey.TEST_VS_DOCKERIZED_MONGO: False}, -) - - -# URI_TEMPLATE = "mongodb://myUser:myPassword@localhost:27017/" -URI_TEMPLATE = "mongodb://{host}:{port}?retrywrites=true&w=majority" - - -def _get_cachier_db_mongo_client(): - host = quote_plus(CFG[CfgKey.HOST]) - port = quote_plus(CFG[CfgKey.PORT]) - # uname = quote_plus(CFG[CfgKey.UNAME]) - # pwd = quote_plus(CFG[CfgKey.PWD]) - # db = quote_plus(CFG[CfgKey.DB]) - uri = f"mongodb://{host}:{port}?retrywrites=true&w=majority" - return MongoClient(uri) - - -_COLLECTION_NAME = ( - f"cachier_test_{platform.system()}" - f"_{'.'.join(map(str, sys.version_info[:3]))}" -) - - -# Global registry to track all MongoDB clients created during tests -_mongo_clients = [] - - -def cleanup_all_mongo_clients(): - """Clean up all MongoDB clients to prevent ResourceWarning.""" - import contextlib - import sys - - global _mongo_clients - - # Close all tracked clients - for client in _mongo_clients: - with contextlib.suppress(Exception): - client.close() - - # Clear the list - _mongo_clients.clear() - - # Clean up any mongetter functions with clients - current_module = sys.modules[__name__] - for attr_name in dir(current_module): - attr = getattr(current_module, attr_name) - if callable(attr) and hasattr(attr, "client"): - with contextlib.suppress(Exception): - if hasattr(attr.client, "close"): - attr.client.close() - delattr(attr, "client") - - -def _test_mongetter(): - if not hasattr(_test_mongetter, "client"): - if str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_MONGO)).lower() == "true": - print("Using live MongoDB instance for testing.") - _test_mongetter.client = _get_cachier_db_mongo_client() - _mongo_clients.append(_test_mongetter.client) - else: - print("Using in-memory MongoDB instance for testing.") - _test_mongetter.client = InMemoryMongoClient() - _mongo_clients.append(_test_mongetter.client) - db_obj = _test_mongetter.client["cachier_test"] - if _COLLECTION_NAME not in db_obj.list_collection_names(): - db_obj.create_collection(_COLLECTION_NAME) - return db_obj[_COLLECTION_NAME] - - -def _get_mongetter_by_collection_name(collection_name=_COLLECTION_NAME): - """Returns a custom mongetter function using a specified collection name. - - This is important for preventing cache conflicts when running tests in - parallel. - - """ - - def _custom_mongetter(): - if not hasattr(_custom_mongetter, "client"): - if ( - str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_MONGO)).lower() - == "true" - ): - print("Using live MongoDB instance for testing.") - _custom_mongetter.client = _get_cachier_db_mongo_client() - _mongo_clients.append(_custom_mongetter.client) - else: - print("Using in-memory MongoDB instance for testing.") - _custom_mongetter.client = InMemoryMongoClient() - _mongo_clients.append(_custom_mongetter.client) - db_obj = _custom_mongetter.client["cachier_test"] - if _COLLECTION_NAME not in db_obj.list_collection_names(): - db_obj.create_collection(collection_name) - return db_obj[collection_name] - - # Store the mongetter function for cleanup - _custom_mongetter._collection_name = collection_name - return _custom_mongetter - - -@pytest.fixture(autouse=True) -def mongo_cleanup(): - """Ensure MongoDB clients are cleaned up after each test.""" - yield - # Clean up after test - cleanup_all_mongo_clients() - - -# === Mongo core tests === - - -@pytest.mark.mongo -def test_missing_mongetter(): - # Test that the appropriate exception is thrown - # when forgetting to specify the mongetter. - with pytest.raises(MissingMongetter): - - @cachier(backend="mongo", mongetter=None) - def dummy_func(): - pass - - -@pytest.mark.mongo -def test_information(): - print("\npymongo version: ", end="") - print(pymongo.__version__) - - -@pytest.mark.mongo -def test_mongo_index_creation(): - """Basic Mongo core functionality.""" - - @cachier(mongetter=_test_mongetter) - def _decorated(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - collection = _test_mongetter() - _decorated.clear_cache() - val1 = _decorated(1, 2) - val2 = _decorated(1, 2) - assert val1 == val2 - assert _MongoCore._INDEX_NAME in collection.index_information() - - -@pytest.mark.mongo -def test_mongo_core_basic(): - """Basic Mongo core functionality.""" - - @cachier(mongetter=_test_mongetter) - def _funci(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _funci.clear_cache() - val1 = _funci(1, 2) - val2 = _funci(1, 2) - assert val1 == val2 - val3 = _funci(1, 2, cachier__skip_cache=True) - assert val3 != val1 - val4 = _funci(1, 2) - assert val4 == val1 - val5 = _funci(1, 2, cachier__overwrite_cache=True) - assert val5 != val1 - val6 = _funci(1, 2) - assert val6 == val5 - - -@pytest.mark.mongo -def test_mongo_core_keywords(): - """Basic Mongo core functionality with keyword arguments.""" - - @cachier(mongetter=_test_mongetter) - def _func_keywords(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _func_keywords.clear_cache() - val1 = _func_keywords(1, arg_2=2) - val2 = _func_keywords(1, arg_2=2) - assert val1 == val2 - val3 = _func_keywords(1, arg_2=2, cachier__skip_cache=True) - assert val3 != val1 - val4 = _func_keywords(1, arg_2=2) - assert val4 == val1 - val5 = _func_keywords(1, arg_2=2, cachier__overwrite_cache=True) - assert val5 != val1 - val6 = _func_keywords(1, arg_2=2) - assert val6 == val5 - - -@pytest.mark.mongo -def test_mongo_stale_after(): - """Testing MongoDB core stale_after functionality.""" - - @cachier( - mongetter=_test_mongetter, - stale_after=datetime.timedelta(seconds=3), - next_time=False, - ) - def _stale_after_mongo(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _stale_after_mongo.clear_cache() - val1 = _stale_after_mongo(1, 2) - val2 = _stale_after_mongo(1, 2) - assert val1 == val2 - sleep(3) - val3 = _stale_after_mongo(1, 2) - assert val3 != val1 - - -def _calls_takes_time(res_queue): - @cachier(mongetter=_test_mongetter) - def _takes_time(arg_1, arg_2): - """Some function.""" - sleep(3) - return random() + arg_1 + arg_2 - - res = _takes_time(34, 82.3) - res_queue.put(res) - - -@pytest.mark.mongo -def test_mongo_being_calculated(): - """Testing MongoDB core handling of being calculated scenarios.""" - - @cachier(mongetter=_test_mongetter) - def _takes_time(arg_1, arg_2): - """Some function.""" - sleep(3) - return random() + arg_1 + arg_2 - - _takes_time.clear_cache() - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_takes_time, kwargs={"res_queue": res_queue}, daemon=True - ) - thread2 = threading.Thread( - target=_calls_takes_time, kwargs={"res_queue": res_queue}, daemon=True - ) - thread1.start() - sleep(1) - thread2.start() - thread1.join(timeout=4) - thread2.join(timeout=4) - assert res_queue.qsize() == 2 - res1 = res_queue.get() - res2 = res_queue.get() - assert res1 == res2 - - -class _BadMongoCollection: - def __init__(self, mongetter): - self.collection = mongetter() - self.index_information = self.collection.index_information - self.create_indexes = self.collection.create_indexes - self.find_one = self.collection.find_one - - def delete_many(self, *args, **kwargs): - pass - - def update_many(self, *args, **kwargs): - pass - - def update_one(self, *args, **kwargs): - raise OperationFailure(Exception()) - - -def _bad_mongetter(): - return _BadMongoCollection(_test_mongetter) - - -@pytest.mark.mongo -def test_mongo_write_failure(): - """Testing MongoDB core handling of writing failure scenarios.""" - - @cachier(mongetter=_bad_mongetter) - def _func_w_bad_mongo(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - with pytest.raises(OperationFailure): - _func_w_bad_mongo(1, 2) - with pytest.raises(OperationFailure): - _func_w_bad_mongo(1, 2) - # assert val1 == val2 - - -@pytest.mark.mongo -def test_mongo_clear_being_calculated(): - """Testing MongoDB core clear_being_calculated.""" - - @cachier(mongetter=_bad_mongetter) - def _func_w_bad_mongo(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _func_w_bad_mongo.clear_being_calculated() - - -@pytest.mark.mongo -def test_stalled_mongo_db_cache(): - @cachier(mongetter=_test_mongetter) - def _stalled_func(): - return 1 - - core = _MongoCore(None, _test_mongetter, 0) - core.set_func(_stalled_func) - core.clear_cache() - with pytest.raises(RecalculationNeeded): - core.wait_on_entry_calc(key=None) - - -@pytest.mark.mongo -def test_stalled_mong_db_core(monkeypatch): - def mock_get_entry(self, args, kwargs): - return "key", CacheEntry( - _processing=True, value=None, time=None, stale=None - ) - - def mock_get_entry_by_key(self, key): - return "key", None - - monkeypatch.setattr( - "cachier.cores.mongo._MongoCore.get_entry", mock_get_entry - ) - monkeypatch.setattr( - "cachier.cores.mongo._MongoCore.get_entry_by_key", - mock_get_entry_by_key, - ) - - @cachier(mongetter=_test_mongetter) - def _stalled_func(): - return 1 - - res = _stalled_func() - assert res == 1 - - def mock_get_entry_2(self, args, kwargs): - return "key", CacheEntry( - value=1, - time=datetime.datetime.now() - datetime.timedelta(seconds=10), - _processing=True, - stale=None, - ) - - monkeypatch.setattr( - "cachier.cores.mongo._MongoCore.get_entry", mock_get_entry_2 - ) - - stale_after = datetime.timedelta(seconds=1) - - @cachier(mongetter=_test_mongetter, stale_after=stale_after) - def _stalled_func_2(): - """Testing stalled function.""" - return 2 - - res = _stalled_func_2() - assert res == 2 - - @cachier( - mongetter=_test_mongetter, stale_after=stale_after, next_time=True - ) - def _stalled_func_3(): - """Testing stalled function.""" - return 3 - - res = _stalled_func_3() - assert res == 1 - - -@pytest.mark.mongo -def test_callable_hash_param(): - def _hash_func(args, kwargs): - def _hash(obj): - if isinstance(obj, pd.core.frame.DataFrame): - return hashlib.sha256( - pd.util.hash_pandas_object(obj).values.tobytes() - ).hexdigest() - return obj - - k_args = tuple(map(_hash, args)) - k_kwargs = tuple( - sorted({k: _hash(v) for k, v in kwargs.items()}.items()) - ) - return k_args + k_kwargs - - @cachier(mongetter=_test_mongetter, hash_func=_hash_func) - def _params_with_dataframe(*args, **kwargs): - """Some function.""" - return random() - - _params_with_dataframe.clear_cache() - - df_a = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) - df_b = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) - value_a = _params_with_dataframe(df_a, 1) - value_b = _params_with_dataframe(df_b, 1) - - assert value_a == value_b # same content --> same key - - value_a = _params_with_dataframe(1, df=df_a) - value_b = _params_with_dataframe(1, df=df_b) - - assert value_a == value_b # same content --> same key - - -# ==== Imported from test_general.py === - -MONGO_DELTA_LONG = datetime.timedelta(seconds=10) - - -@pytest.mark.mongo -@pytest.mark.parametrize("separate_files", [True, False]) -def test_wait_for_calc_timeout_ok(separate_files): - mongetter = _get_mongetter_by_collection_name( - "test_wait_for_calc_timeout_ok" - ) - - @cachier( - mongetter=mongetter, - stale_after=MONGO_DELTA_LONG, - separate_files=separate_files, - next_time=False, - wait_for_calc_timeout=2, - ) - def _wait_for_calc_timeout_fast(arg_1, arg_2): - """Some function.""" - sleep(1) - return random() + arg_1 + arg_2 - - def _calls_wait_for_calc_timeout_fast(res_queue): - res = _wait_for_calc_timeout_fast(1, 2) - res_queue.put(res) - - """ Testing calls that avoid timeouts store the values in cache. """ - _wait_for_calc_timeout_fast.clear_cache() - val1 = _wait_for_calc_timeout_fast(1, 2) - val2 = _wait_for_calc_timeout_fast(1, 2) - assert val1 == val2 - - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_wait_for_calc_timeout_fast, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_wait_for_calc_timeout_fast, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - - thread1.start() - thread2.start() - sleep(2) - thread1.join(timeout=2) - thread2.join(timeout=2) - assert res_queue.qsize() == 2 - res1 = res_queue.get() - res2 = res_queue.get() - assert res1 == res2 # Timeout did not kick in, a single call was done - - -@pytest.mark.mongo -@pytest.mark.parametrize("separate_files", [True, False]) -@pytest.mark.flaky(reruns=10, reruns_delay=0.5) -def test_wait_for_calc_timeout_slow(separate_files): - # Use unique test parameters to avoid cache conflicts in parallel execution - import os - import uuid - - test_id = os.getpid() + int( - uuid.uuid4().int >> 96 - ) # Unique but deterministic within test - arg1, arg2 = test_id, test_id + 1 - - # In parallel tests, add random delay to reduce thread contention - if os.environ.get("PYTEST_XDIST_WORKER"): - sleep(random() * 0.5) # 0-500ms random delay - - @cachier( - mongetter=_test_mongetter, - stale_after=MONGO_DELTA_LONG, - separate_files=separate_files, - next_time=False, - wait_for_calc_timeout=2, - ) - def _wait_for_calc_timeout_slow(arg_1, arg_2): - sleep(2) - return random() + arg_1 + arg_2 - - def _calls_wait_for_calc_timeout_slow(res_queue): - res = _wait_for_calc_timeout_slow(arg1, arg2) - res_queue.put(res) - - """Testing for calls timing out to be performed twice when needed.""" - _wait_for_calc_timeout_slow.clear_cache() - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_wait_for_calc_timeout_slow, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_wait_for_calc_timeout_slow, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - - thread1.start() - thread2.start() - sleep(1) - res3 = _wait_for_calc_timeout_slow(arg1, arg2) - sleep(3) # Increased from 4 to give more time for threads to complete - thread1.join(timeout=10) # Increased timeout for thread joins - thread2.join(timeout=10) - assert res_queue.qsize() == 2 - res1 = res_queue.get() - res2 = res_queue.get() - assert res1 != res2 # Timeout kicked in. Two calls were done - res4 = _wait_for_calc_timeout_slow(arg1, arg2) - # One of the cached values is returned - assert res1 == res4 or res2 == res4 or res3 == res4 - - -@pytest.mark.mongo -def test_precache_value(): - @cachier(mongetter=_test_mongetter) - def dummy_func(arg_1, arg_2): - """Some function.""" - return arg_1 + arg_2 - - assert dummy_func.precache_value(2, 2, value_to_cache=5) == 5 - assert dummy_func(2, 2) == 5 - dummy_func.clear_cache() - assert dummy_func(2, 2) == 4 - assert dummy_func.precache_value(2, arg_2=2, value_to_cache=5) == 5 - assert dummy_func(2, arg_2=2) == 5 - - -@pytest.mark.mongo -def test_ignore_self_in_methods(): - class DummyClass: - @cachier(mongetter=_test_mongetter) - def takes_2_seconds(self, arg_1, arg_2): - """Some function.""" - sleep(2) - return arg_1 + arg_2 - - test_object_1 = DummyClass() - test_object_2 = DummyClass() - test_object_1.takes_2_seconds.clear_cache() - test_object_2.takes_2_seconds.clear_cache() - assert test_object_1.takes_2_seconds(1, 2) == 3 - start = time() - assert test_object_2.takes_2_seconds(1, 2) == 3 - end = time() - assert end - start < 1 - - -# Test: MongoDB allow_none=False handling (line 99) -@pytest.mark.mongo -def test_mongo_allow_none_false(): - """Test MongoDB backend with allow_none=False and None return value.""" - - @cachier(mongetter=_test_mongetter, allow_none=False) - def returns_none(): - return None - - # First call should execute and return None - result1 = returns_none() - assert result1 is None - - # Second call should also execute (not cached) because None is not allowed - result2 = returns_none() - assert result2 is None - - # Clear cache - returns_none.clear_cache() - - -# test: mongodb none handling with allow_none=false -@pytest.mark.mongo -def test_mongo_allow_none_false_not_stored(): - """Test mongodb doesn't store none when allow_none=false.""" - call_count = 0 - - @cachier(mongetter=_test_mongetter, allow_none=False) - def returns_none(): - nonlocal call_count - call_count += 1 - return None - - returns_none.clear_cache() - - # first call - result1 = returns_none() - assert result1 is None - assert call_count == 1 - - # second call should also execute (not cached) - result2 = returns_none() - assert result2 is None - assert call_count == 2 - - returns_none.clear_cache() - - -# Test: MongoDB delete_stale_entries -@pytest.mark.mongo -def test_mongo_delete_stale_direct(): - """Test MongoDB stale entry deletion method directly.""" - - @cachier(mongetter=_test_mongetter, stale_after=timedelta(seconds=1)) - def test_func(x): - return x * 2 - - test_func.clear_cache() - - # Create entries - test_func(1) - test_func(2) - - # Wait for staleness - sleep(1.1) - - # Access the mongo core and call delete_stale_entries - # This is a bit hacky but needed to test the specific method - from cachier.cores.mongo import _MongoCore - - # Get the collection - _test_mongetter() # Ensure connection is available - - # Create a core instance just for deletion - core = _MongoCore( - mongetter=_test_mongetter, - hash_func=None, - wait_for_calc_timeout=0, - ) - - # Set the function to get the right cache key prefix - core.set_func(test_func) - - # Delete stale entries - core.delete_stale_entries(timedelta(seconds=1)) - - test_func.clear_cache() - - -@pytest.mark.mongo -def test_mongo_unsupported_replacement_policy(): - """Test that unsupported replacement policy raises ValueError.""" - from cachier.cores.mongo import _MongoCore - - # Clear before test - _test_mongetter().delete_many({}) - - @cachier( - mongetter=_test_mongetter, - cache_size_limit="100B", - replacement_policy="lru", # Start with valid policy - ) - def test_func(x): - return "a" * 50 - - # First, fill the cache to trigger eviction - test_func(1) - test_func(2) - - # Now create a core with an unsupported policy - core = _MongoCore( - hash_func=None, - mongetter=_test_mongetter, - wait_for_calc_timeout=0, - cache_size_limit=100, - replacement_policy="invalid_policy", # Invalid policy - ) - core.set_func(test_func) - - # This should raise ValueError when trying to evict - with pytest.raises( - ValueError, match="Unsupported replacement policy: invalid_policy" - ): - core.set_entry("new_key", "a" * 50) - - test_func.clear_cache() diff --git a/tests/test_pickle_core.py b/tests/test_pickle_core.py deleted file mode 100644 index aa0c5134..00000000 --- a/tests/test_pickle_core.py +++ /dev/null @@ -1,1304 +0,0 @@ -"""Test for the Cachier python package.""" - -# This file is part of Cachier. -# https://github.com/python-cachier/cachier - -# Licensed under the MIT license: -# http://www.opensource.org/licenses/MIT-license -# Copyright (c) 2016, Shay Palachy - -# from os.path import ( -# realpath, -# dirname -# ) -import hashlib -import os -import pickle -import sys -import tempfile -import threading -import uuid -from datetime import datetime, timedelta -from random import random -from time import sleep, time -from unittest.mock import Mock, patch - -import pytest - -try: - import queue -except ImportError: # python 2 - import Queue as queue # type: ignore - - -import pandas as pd - -from cachier import cachier -from cachier.config import CacheEntry, _global_params -from cachier.cores.pickle import _PickleCore - - -def _get_decorated_func(func, **kwargs): - cachier_decorator = cachier(**kwargs) - decorated_func = cachier_decorator(func) - return decorated_func - - -# Pickle core tests - - -def _takes_2_seconds(arg_1, arg_2): - """Some function.""" - sleep(2) - return f"arg_1:{arg_1}, arg_2:{arg_2}" - - -@pytest.mark.pickle -@pytest.mark.parametrize("reload", [True, False]) -@pytest.mark.parametrize("separate_files", [True, False]) -def test_pickle_core(reload, separate_files): - """Basic Pickle core functionality.""" - _takes_2_seconds_decorated = _get_decorated_func( - _takes_2_seconds, - next_time=False, - pickle_reload=reload, - separate_files=separate_files, - ) - _takes_2_seconds_decorated.clear_cache() - _takes_2_seconds_decorated("a", "b") - start = time() - _takes_2_seconds_decorated("a", "b", cachier__verbose=True) - end = time() - assert end - start < 1 - _takes_2_seconds_decorated.clear_cache() - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_pickle_core_keywords(separate_files): - """Basic Pickle core functionality with keyword arguments.""" - _takes_2_seconds_decorated = _get_decorated_func( - _takes_2_seconds, next_time=False, separate_files=separate_files - ) - _takes_2_seconds_decorated.clear_cache() - _takes_2_seconds_decorated("a", arg_2="b") - start = time() - _takes_2_seconds_decorated("a", arg_2="b", cachier__verbose=True) - end = time() - assert end - start < 1 - _takes_2_seconds_decorated.clear_cache() - - -SECONDS_IN_DELTA = 3 -DELTA = timedelta(seconds=SECONDS_IN_DELTA) - - -def _stale_after_seconds(arg_1, arg_2): - """Some function.""" - return random() - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_stale_after(separate_files): - """Testing the stale_after functionality.""" - _stale_after_seconds_decorated = _get_decorated_func( - _stale_after_seconds, - stale_after=DELTA, - next_time=False, - separate_files=separate_files, - ) - _stale_after_seconds_decorated.clear_cache() - val1 = _stale_after_seconds_decorated(1, 2) - val2 = _stale_after_seconds_decorated(1, 2) - val3 = _stale_after_seconds_decorated(1, 3) - assert val1 == val2 - assert val1 != val3 - sleep(3) - val4 = _stale_after_seconds_decorated(1, 2) - assert val4 != val1 - _stale_after_seconds_decorated.clear_cache() - - -def _stale_after_next_time(arg_1, arg_2): - """Some function.""" - return random() - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_stale_after_next_time(separate_files): - """Testing the stale_after with next_time functionality.""" - _stale_after_next_time_decorated = _get_decorated_func( - _stale_after_next_time, - stale_after=DELTA, - next_time=True, - separate_files=separate_files, - ) - _stale_after_next_time_decorated.clear_cache() - val1 = _stale_after_next_time_decorated(1, 2) - val2 = _stale_after_next_time_decorated(1, 2) - val3 = _stale_after_next_time_decorated(1, 3) - assert val1 == val2 - assert val1 != val3 - sleep(SECONDS_IN_DELTA + 1) - val4 = _stale_after_next_time_decorated(1, 2) - assert val4 == val1 - sleep(0.5) - val5 = _stale_after_next_time_decorated(1, 2) - assert val5 != val1 - _stale_after_next_time_decorated.clear_cache() - - -def _random_num(): - return random() - - -def _random_num_with_arg(a): - # print(a) - return random() - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_overwrite_cache(separate_files): - """Tests that the overwrite feature works correctly.""" - _random_num_decorated = _get_decorated_func( - _random_num, separate_files=separate_files - ) - _random_num_with_arg_decorated = _get_decorated_func( - _random_num_with_arg, separate_files=separate_files - ) - _random_num_decorated.clear_cache() - int1 = _random_num_decorated() - int2 = _random_num_decorated() - assert int2 == int1 - int3 = _random_num_decorated(cachier__overwrite_cache=True) - assert int3 != int1 - int4 = _random_num_decorated() - assert int4 == int3 - _random_num_decorated.clear_cache() - - _random_num_with_arg_decorated.clear_cache() - int1 = _random_num_with_arg_decorated("a") - int2 = _random_num_with_arg_decorated("a") - assert int2 == int1 - int3 = _random_num_with_arg_decorated("a", cachier__overwrite_cache=True) - assert int3 != int1 - int4 = _random_num_with_arg_decorated("a") - assert int4 == int3 - _random_num_with_arg_decorated.clear_cache() - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_ignore_cache(separate_files): - """Tests that the ignore_cache feature works correctly.""" - _random_num_decorated = _get_decorated_func( - _random_num, separate_files=separate_files - ) - _random_num_with_arg_decorated = _get_decorated_func( - _random_num_with_arg, separate_files=separate_files - ) - _random_num_decorated.clear_cache() - int1 = _random_num_decorated() - int2 = _random_num_decorated() - assert int2 == int1 - int3 = _random_num_decorated(cachier__skip_cache=True) - assert int3 != int1 - int4 = _random_num_decorated() - assert int4 != int3 - assert int4 == int1 - _random_num_decorated.clear_cache() - - _random_num_with_arg_decorated.clear_cache() - int1 = _random_num_with_arg_decorated("a") - int2 = _random_num_with_arg_decorated("a") - assert int2 == int1 - int3 = _random_num_with_arg_decorated("a", cachier__skip_cache=True) - assert int3 != int1 - int4 = _random_num_with_arg_decorated("a") - assert int4 != int3 - assert int4 == int1 - _random_num_with_arg_decorated.clear_cache() - - -def _takes_time(arg_1, arg_2): - """Some function.""" - sleep(2) # this has to be enough time for check_calculation to run twice - return random() + arg_1 + arg_2 - - -def _calls_takes_time(takes_time_func, res_queue): - res = takes_time_func(0.13, 0.02) - res_queue.put(res) - - -@pytest.mark.pickle -@pytest.mark.flaky(reruns=5, reruns_delay=0.5) -@pytest.mark.parametrize("separate_files", [True, False]) -def test_pickle_being_calculated(separate_files): - """Testing pickle core handling of being calculated scenarios.""" - _takes_time_decorated = _get_decorated_func( - _takes_time, separate_files=separate_files - ) - _takes_time_decorated.clear_cache() - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_takes_time, - kwargs={ - "takes_time_func": _takes_time_decorated, - "res_queue": res_queue, - }, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_takes_time, - kwargs={ - "takes_time_func": _takes_time_decorated, - "res_queue": res_queue, - }, - daemon=True, - ) - thread1.start() - sleep(0.5) - thread2.start() - thread1.join(timeout=4) - thread2.join(timeout=4) - assert res_queue.qsize() == 2 - res1 = res_queue.get() - res2 = res_queue.get() - assert res1 == res2 - - -def _being_calc_next_time(arg_1, arg_2): - """Some function.""" - sleep(1) - return random() + arg_1 + arg_2 - - -def _calls_being_calc_next_time(being_calc_func, res_queue): - res = being_calc_func(0.13, 0.02) - res_queue.put(res) - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -@pytest.mark.flaky(reruns=5, reruns_delay=0.1) -def test_being_calc_next_time(separate_files): - """Testing pickle core handling of being calculated scenarios.""" - _being_calc_next_time_decorated = _get_decorated_func( - _being_calc_next_time, - stale_after=timedelta(seconds=1), - next_time=True, - separate_files=separate_files, - ) - _being_calc_next_time_decorated.clear_cache() - _being_calc_next_time(0.13, 0.02) - sleep(1.1) - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_being_calc_next_time, - kwargs={ - "being_calc_func": _being_calc_next_time_decorated, - "res_queue": res_queue, - }, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_being_calc_next_time, - kwargs={ - "being_calc_func": _being_calc_next_time_decorated, - "res_queue": res_queue, - }, - daemon=True, - ) - thread1.start() - sleep(0.5) - thread2.start() - thread1.join(timeout=2) - thread2.join(timeout=2) - assert res_queue.qsize() == 2 - res1 = res_queue.get() - res2 = res_queue.get() - assert res1 == res2 - - -def _bad_cache(arg_1, arg_2): - """Some function.""" - sleep(1) - return random() + arg_1 + arg_2 - - -# _BAD_CACHE_FNAME = '.__main__._bad_cache' -_BAD_CACHE_FNAME = ".tests.test_pickle_core._bad_cache" -_BAD_CACHE_FNAME_SEPARATE_FILES = ( - ".tests.test_pickle_core._bad_cache_" - f"{hashlib.sha256(pickle.dumps((0.13, 0.02))).hexdigest()}" -) -EXPANDED_CACHIER_DIR = os.path.expanduser(_global_params.cache_dir) -_BAD_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME) -_BAD_CACHE_FPATH_SEPARATE_FILES = os.path.join( - EXPANDED_CACHIER_DIR, _BAD_CACHE_FNAME_SEPARATE_FILES -) -_BAD_CACHE_FPATHS = { - True: _BAD_CACHE_FPATH_SEPARATE_FILES, - False: _BAD_CACHE_FPATH, -} - - -def _calls_bad_cache( - bad_cache_func, res_queue, trash_cache, separate_files, cache_dir -): - try: - res = bad_cache_func(0.13, 0.02, cachier__verbose=True) - if trash_cache: - # Use the provided cache directory - if separate_files: - fname = _BAD_CACHE_FNAME_SEPARATE_FILES - else: - fname = _BAD_CACHE_FNAME - cache_fpath = os.path.join(cache_dir, fname) - with open(cache_fpath, "w") as cache_file: - cache_file.seek(0) - cache_file.truncate() - res_queue.put(res) - except Exception as exc: - res_queue.put(exc) - - -def _helper_bad_cache_file(sleep_time: float, separate_files: bool): - """Test pickle core handling of bad cache files.""" - # Use a unique cache directory for this test to avoid parallel conflicts - unique_cache_dir = os.path.join( - tempfile.gettempdir(), f"cachier_test_bad_{uuid.uuid4().hex[:8]}" - ) - os.makedirs(unique_cache_dir, exist_ok=True) - - _bad_cache_decorated = _get_decorated_func( - _bad_cache, separate_files=separate_files, cache_dir=unique_cache_dir - ) - _bad_cache_decorated.clear_cache() - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_bad_cache, - kwargs={ - "bad_cache_func": _bad_cache_decorated, - "res_queue": res_queue, - "trash_cache": True, - "separate_files": separate_files, - "cache_dir": unique_cache_dir, - }, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_bad_cache, - kwargs={ - "bad_cache_func": _bad_cache_decorated, - "res_queue": res_queue, - "trash_cache": False, - "separate_files": separate_files, - "cache_dir": unique_cache_dir, - }, - daemon=True, - ) - thread1.start() - sleep(sleep_time) - thread2.start() - thread1.join(timeout=2) - thread2.join(timeout=2) - if res_queue.qsize() != 2: - return False - res1 = res_queue.get() - if not isinstance(res1, float): - return False - res2 = res_queue.get() - return res2 is None - - -# we want this to succeed at least once -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -@pytest.mark.flaky(reruns=8, reruns_delay=0.1) -def test_bad_cache_file(separate_files): - """Test pickle core handling of bad cache files.""" - # On macOS, file system events and watchdog timing can be different - if sys.platform == "darwin": - sleep_times = [1.0, 1.5, 2.0, 2.5, 3.0] - else: - sleep_times = [0.6, 1, 1.5, 2, 2.5] - bad_file = False - for sleep_time in sleep_times * 2: - if _helper_bad_cache_file(sleep_time, separate_files): - bad_file = True - break - # it is expected that for separate_files=True files will not be bad - assert bad_file is not separate_files - - -def _delete_cache(arg_1, arg_2): - """Some function.""" - sleep(1) - return random() + arg_1 + arg_2 - - -# _DEL_CACHE_FNAME = '.__main__._delete_cache' -_DEL_CACHE_FNAME = ".tests.test_pickle_core._delete_cache" -_DEL_CACHE_FNAME_SEPARATE_FILES = ( - ".tests.test_pickle_core._delete_cache_" - f"{hashlib.sha256(pickle.dumps((0.13, 0.02))).hexdigest()}" -) -_DEL_CACHE_FPATH = os.path.join(EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME) -_DEL_CACHE_FPATH_SEPARATE_FILES = os.path.join( - EXPANDED_CACHIER_DIR, _DEL_CACHE_FNAME_SEPARATE_FILES -) -_DEL_CACHE_FPATHS = { - True: _DEL_CACHE_FPATH_SEPARATE_FILES, - False: _DEL_CACHE_FPATH, -} - - -def _calls_delete_cache( - del_cache_func, - res_queue, - del_cache: bool, - separate_files: bool, - cache_dir: str, -): - try: - # print('in') - res = del_cache_func(0.13, 0.02) - # print('out with {}'.format(res)) - if del_cache: - # Use the provided cache directory - if separate_files: - fname = _DEL_CACHE_FNAME_SEPARATE_FILES - else: - fname = _DEL_CACHE_FNAME - cache_fpath = os.path.join(cache_dir, fname) - os.remove(cache_fpath) - # print(os.path.isfile(_DEL_CACHE_FPATH)) - res_queue.put(res) - except Exception as exc: - # print('found') - res_queue.put(exc) - - -def _helper_delete_cache_file(sleep_time: float, separate_files: bool): - """Test pickle core handling of missing cache files.""" - # Use a unique cache directory for this test to avoid parallel conflicts - unique_cache_dir = os.path.join( - tempfile.gettempdir(), f"cachier_test_del_{uuid.uuid4().hex[:8]}" - ) - os.makedirs(unique_cache_dir, exist_ok=True) - - _delete_cache_decorated = _get_decorated_func( - _delete_cache, - separate_files=separate_files, - cache_dir=unique_cache_dir, - ) - _delete_cache_decorated.clear_cache() - res_queue = queue.Queue() - thread1 = threading.Thread( - target=_calls_delete_cache, - kwargs={ - "del_cache_func": _delete_cache_decorated, - "res_queue": res_queue, - "del_cache": True, - "separate_files": separate_files, - "cache_dir": unique_cache_dir, - }, - daemon=True, - ) - thread2 = threading.Thread( - target=_calls_delete_cache, - kwargs={ - "del_cache_func": _delete_cache_decorated, - "res_queue": res_queue, - "del_cache": False, - "separate_files": separate_files, - "cache_dir": unique_cache_dir, - }, - daemon=True, - ) - thread1.start() - sleep(sleep_time) - thread2.start() - thread1.join(timeout=2) - thread2.join(timeout=2) - if res_queue.qsize() != 2: - return False - res1 = res_queue.get() - # print(res1) - if not isinstance(res1, float): - return False - res2 = res_queue.get() - return isinstance(res2, KeyError) or (res2 is None) - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [False, True]) -@pytest.mark.flaky(reruns=10, reruns_delay=0.1) -def test_delete_cache_file(separate_files): - """Test pickle core handling of missing cache files.""" - # On macOS, file system events and watchdog timing can be different - if sys.platform == "darwin": - sleep_times = [0.2, 0.4, 0.6, 0.8, 1.0, 1.5] - else: - sleep_times = [0.1, 0.2, 0.3, 0.5, 0.7, 1] - deleted = False - for sleep_time in sleep_times * 4: - if _helper_delete_cache_file(sleep_time, separate_files): - deleted = True - break - # it is expected that for separate_files=True files will not be deleted - assert deleted is not separate_files - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [False, True]) -def test_clear_being_calculated(separate_files): - """Test pickle core clear `being calculated` functionality.""" - _takes_time_decorated = _get_decorated_func( - _takes_time, separate_files=separate_files - ) - _takes_time_decorated.clear_being_calculated() - - -def _error_throwing_func(arg1): - if not hasattr(_error_throwing_func, "count"): - _error_throwing_func.count = 0 - _error_throwing_func.count += 1 - if _error_throwing_func.count > 1: - raise ValueError("Tiny Rick!") - return 7 - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_error_throwing_func(separate_files): - # with - _error_throwing_func.count = 0 - _error_throwing_func_decorated = _get_decorated_func( - _error_throwing_func, - stale_after=timedelta(seconds=1), - next_time=True, - separate_files=separate_files, - ) - _error_throwing_func_decorated.clear_cache() - res1 = _error_throwing_func_decorated(4) - sleep(1.5) - res2 = _error_throwing_func_decorated(4) - assert res1 == res2 - - -# test custom cache dir for pickle core - -CUSTOM_DIR = "~/.exparrot" -EXPANDED_CUSTOM_DIR = os.path.expanduser(CUSTOM_DIR) - - -def _takes_2_seconds_custom_dir(arg_1, arg_2): - """Some function.""" - sleep(2) - return f"arg_1:{arg_1}, arg_2:{arg_2}" - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_pickle_core_custom_cache_dir(separate_files): - """Basic Pickle core functionality.""" - _takes_2_seconds_custom_dir_decorated = _get_decorated_func( - _takes_2_seconds_custom_dir, - next_time=False, - cache_dir=CUSTOM_DIR, - separate_files=separate_files, - ) - _takes_2_seconds_custom_dir_decorated.clear_cache() - _takes_2_seconds_custom_dir_decorated("a", "b") - start = time() - _takes_2_seconds_custom_dir_decorated("a", "b", cachier__verbose=True) - end = time() - assert end - start < 1 - _takes_2_seconds_custom_dir_decorated.clear_cache() - path2test = _takes_2_seconds_custom_dir_decorated.cache_dpath() - assert path2test == EXPANDED_CUSTOM_DIR - - -@pytest.mark.pickle -@pytest.mark.parametrize("separate_files", [True, False]) -def test_callable_hash_param(separate_files): - def _hash_func(args, kwargs): - def _hash(obj): - if isinstance(obj, pd.core.frame.DataFrame): - return hashlib.sha256( - pd.util.hash_pandas_object(obj).values.tobytes() - ).hexdigest() - return obj - - k_args = tuple(map(_hash, args)) - k_kwargs = tuple( - sorted({k: _hash(v) for k, v in kwargs.items()}.items()) - ) - return k_args + k_kwargs - - @cachier(hash_func=_hash_func, separate_files=separate_files) - def _params_with_dataframe(*args, **kwargs): - """Some function.""" - return random() - - _params_with_dataframe.clear_cache() - - df_a = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) - df_b = pd.DataFrame.from_dict({"a": [0], "b": [2], "c": [3]}) - value_a = _params_with_dataframe(df_a, 1) - value_b = _params_with_dataframe(df_b, 1) - - assert value_a == value_b # same content --> same key - - value_a = _params_with_dataframe(1, df=df_a) - value_b = _params_with_dataframe(1, df=df_b) - - assert value_a == value_b # same content --> same key - - -@pytest.mark.pickle -@pytest.mark.skipif( - not sys.platform.startswith("linux"), - reason="inotify instance limit is only relevant on Linux", -) -def test_inotify_instance_limit_reached(): - """Reproduces the inotify instance exhaustion issue (see Issue #24). - - Rapidly creates many cache waits to exhaust inotify instances. - Reference: https://github.com/python-cachier/cachier/issues/24 - - """ - import queue - import subprocess - - # Try to get the current inotify limit - try: - result = subprocess.run( - ["/bin/cat", "/proc/sys/fs/inotify/max_user_instances"], - capture_output=True, - text=True, - timeout=5, - ) - if result.returncode == 0: - current_limit = int(result.stdout.strip()) - print(f"Current inotify max_user_instances limit: {current_limit}") - else: - current_limit = None - print("Could not determine inotify limit") - except Exception as e: - current_limit = None - print(f"Error getting inotify limit: {e}") - - @cachier(backend="pickle", wait_for_calc_timeout=0.1) - def slow_func(x): - sleep(0.5) # Make it slower to increase chance of hitting limit - return x - - # Start many threads to trigger wait_on_entry_calc - threads = [] - errors = [] - results = queue.Queue() - - # Be more aggressive - try to exhaust the limit - N = ( - min(current_limit * 4, 4096) if current_limit is not None else 4096 - ) # Try to exceed the limit more aggressively - print(f"Starting {N} threads to test inotify exhaustion") - - def call(): - try: - results.put(slow_func(1)) - except OSError as e: - errors.append(e) - except Exception as e: - # Capture any other exceptions for debugging - errors.append(e) - - for i in range(N): - t = threading.Thread(target=call) - threads.append(t) - t.start() - if i % 100 == 0: - print(f"Started {i} threads...") - - print("Waiting for all threads to complete...") - for t in threads: - t.join() - - print( - f"Test completed. Got {len(errors)} errors, {results.qsize()} results" - ) - - # If any OSError with "inotify instance limit reached" is raised, - # the test FAILS (expected failure due to the bug) - if any("inotify instance limit reached" in str(e) for e in errors): - print( - "FAILURE: Hit inotify instance limit - this indicates the bug " - "still exists" - ) - raise AssertionError( - "inotify instance limit reached error occurred. " - f"Got {len(errors)} errors with inotify limit issues." - ) - - # If no inotify errors but other errors, fail - if errors: - print(f"Unexpected errors occurred: {errors}") - raise AssertionError(f"Unexpected OSErrors: {errors}") - - # If no errors at all, the test PASSES (issue is fixed!) - print( - "SUCCESS: No inotify instance limit errors occurred - the issue " - "appears to be fixed!" - ) - # No need to return - test passes naturally - - -@pytest.mark.pickle -def test_convert_legacy_cache_entry_dict(): - """Test _convert_legacy_cache_entry with dict input.""" - # Test line 112-118: converting legacy dict format - legacy_entry = { - "value": "test_value", - "time": datetime.now(), - "stale": False, - "being_calculated": True, - "condition": None, - } - - result = _PickleCore._convert_legacy_cache_entry(legacy_entry) - - assert isinstance(result, CacheEntry) - assert result.value == "test_value" - assert result.stale is False - assert result._processing is True - - -@pytest.mark.pickle -def test_save_cache_with_invalid_separate_file_key(): - """Test _save_cache raises error with invalid separate_file_key.""" - # Test line 179-181: ValueError when separate_file_key used with dict - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=False, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Should raise ValueError when using separate_file_key with a dict - with pytest.raises( - ValueError, - match="`separate_file_key` should only be used with a CacheEntry", - ): - core._save_cache({"key": "value"}, separate_file_key="test_key") - - -@pytest.mark.pickle -def test_set_entry_should_not_store(): - """Test set_entry when value should not be stored.""" - # Test line 204: early return when _should_store returns False - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=False, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock _should_store to return False - core._should_store = Mock(return_value=False) - - result = core.set_entry("test_key", None) - assert result is False - - -@pytest.mark.pickle -def test_mark_entry_not_calculated_separate_files_no_entry(): - """Test _mark_entry_not_calculated_separate_files with no entry.""" - # Test line 236: early return when entry is None - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=True, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock get_entry_by_key to return None - core.get_entry_by_key = Mock(return_value=("test_key", None)) - - # Should return without error - core._mark_entry_not_calculated_separate_files("test_key") - - -@pytest.mark.pickle -def test_cleanup_observer_exception(): - """Test _cleanup_observer with exception during cleanup.""" - # Test lines 278-279: exception handling in observer cleanup - core = _PickleCore( - hash_func=None, - cache_dir=".", - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=False, - ) - - # Set a mock function - mock_func = Mock() - mock_func.__name__ = "test_func" - mock_func.__module__ = "test_module" - mock_func.__qualname__ = "test_func" - core.set_func(mock_func) - - # Mock observer that raises exception - mock_observer = Mock() - mock_observer.is_alive.return_value = True - mock_observer.stop.side_effect = Exception("Observer error") - - # Should not raise exception - core._cleanup_observer(mock_observer) - - -@pytest.mark.pickle -def test_wait_on_entry_calc_inotify_limit(): - """Test wait_on_entry_calc fallback when inotify limit is reached.""" - # Test lines 298-302: OSError handling for inotify limit - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=False, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Create a cache entry that's being calculated - cache_entry = CacheEntry( - value="test_value", - time=datetime.now(), - stale=False, - _processing=True, # Should be processing - ) - core._save_cache({"test_key": cache_entry}) - - # Mock _wait_with_inotify to raise OSError with inotify message - def mock_wait_inotify(key, filename): - raise OSError("inotify instance limit reached") - - core._wait_with_inotify = mock_wait_inotify - - # Mock _wait_with_polling to return a value - core._wait_with_polling = Mock(return_value="polling_result") - - result = core.wait_on_entry_calc("test_key") - assert result == "polling_result" - core._wait_with_polling.assert_called_once_with("test_key") - - -@pytest.mark.pickle -def test_wait_on_entry_calc_other_os_error(): - """Test wait_on_entry_calc re-raises non-inotify OSErrors.""" - # Test line 302: re-raise other OSErrors - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=False, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock _wait_with_inotify to raise different OSError - def mock_wait_inotify(key, filename): - raise OSError("Different error") - - core._wait_with_inotify = mock_wait_inotify - - with pytest.raises(OSError, match="Different error"): - core.wait_on_entry_calc("test_key") - - -@pytest.mark.pickle -def test_wait_with_polling_file_errors(): - """Test _wait_with_polling handles file errors gracefully.""" - # Test lines 352-354: FileNotFoundError/EOFError handling - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=2, # Short timeout - separate_files=False, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock methods to simulate file errors then success - call_count = 0 - - def mock_get_cache_dict(): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise FileNotFoundError("Cache file not found") - elif call_count == 2: - raise EOFError("Cache file corrupted") - else: - return { - "test_key": CacheEntry( - value="result", - time=datetime.now(), - stale=False, - _processing=False, - ) - } - - core.get_cache_dict = mock_get_cache_dict - core.separate_files = False - - result = core._wait_with_polling("test_key") - assert result == "result" - - -@pytest.mark.pickle -def test_wait_with_polling_separate_files(): - """Test _wait_with_polling with separate files mode.""" - # Test lines 342-343: separate files branch - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=True, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock _load_cache_by_key - entry = CacheEntry( - value="test_value", - time=datetime.now(), - stale=False, - _processing=False, - ) - core._load_cache_by_key = Mock(return_value=entry) - - result = core._wait_with_polling("test_key") - assert result == "test_value" - - -@pytest.mark.pickle -def test_delete_stale_entries_separate_files(): - """Test delete_stale_entries with separate files mode.""" - # Test lines 377-387: separate files deletion logic - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=True, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Create some cache files - base_path = core.cache_fpath - - # Create stale entry file - stale_entry = CacheEntry( - value="stale_value", - time=datetime.now() - timedelta(hours=2), - stale=False, - _processing=False, - ) - stale_file = f"{base_path}_stalekey" - with open(stale_file, "wb") as f: - pickle.dump(stale_entry, f) - - # Create fresh entry file - fresh_entry = CacheEntry( - value="fresh_value", - time=datetime.now(), - stale=False, - _processing=False, - ) - fresh_file = f"{base_path}_freshkey" - with open(fresh_file, "wb") as f: - pickle.dump(fresh_entry, f) - - # Create non-matching file (should be ignored) - other_file = os.path.join(temp_dir, "other_file.txt") - with open(other_file, "w") as f: - f.write("other content") - - # Before running delete, check that files exist - assert os.path.exists(stale_file) - assert os.path.exists(fresh_file) - - # Run delete_stale_entries - core.delete_stale_entries(timedelta(hours=1)) - - # Check that only stale file was deleted - assert not os.path.exists(stale_file) - assert os.path.exists(fresh_file) - assert os.path.exists(other_file) - - -@pytest.mark.pickle -def test_delete_stale_entries_file_not_found(): - """Test delete_stale_entries handles FileNotFoundError.""" - # Test lines 385-386: FileNotFoundError suppression - with tempfile.TemporaryDirectory() as temp_dir: - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=10, - separate_files=True, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Mock _load_cache_by_key to return a stale entry - stale_entry = CacheEntry( - value="stale", - time=datetime.now() - timedelta(hours=2), - stale=False, - _processing=False, - ) - core._load_cache_by_key = Mock(return_value=stale_entry) - - # Mock os.remove to raise FileNotFoundError - with patch("os.remove", side_effect=FileNotFoundError): - # Should not raise exception - core.delete_stale_entries(timedelta(hours=1)) - - -# Pickle clear being calculated with separate files -@pytest.mark.pickle -def test_pickle_clear_being_calculated_separate_files(): - """Test clearing processing flags in separate cache files.""" - with tempfile.TemporaryDirectory() as temp_dir: - - @cachier(backend="pickle", cache_dir=temp_dir, separate_files=True) - def test_func(x): - return x * 2 - - # Get the pickle core - from cachier.cores.pickle import _PickleCore - - # Create a temporary core to manipulate cache - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=0, - separate_files=True, - ) - core.set_func(test_func) - - # Create cache entries with processing flag - for i in range(3): - entry = CacheEntry( - value=i * 2, time=datetime.now(), stale=False, _processing=True - ) - # Create hash for key - key_hash = str(hash((i,))) - # For separate files, save the entry directly - core._save_cache(entry, separate_file_key=key_hash) - - # Clear being calculated - core._clear_being_calculated_all_cache_files() - - # Verify files exist but processing is cleared - cache_files = [f for f in os.listdir(temp_dir) if f.startswith(".")] - assert len(cache_files) >= 3 - - test_func.clear_cache() - - -# Pickle save with hash_str parameter -@pytest.mark.pickle -def test_pickle_save_with_hash_str(): - """Test _save_cache with hash_str creates correct filename.""" - with tempfile.TemporaryDirectory() as temp_dir: - from cachier.cores.pickle import _PickleCore - - core = _PickleCore( - hash_func=None, - cache_dir=temp_dir, - pickle_reload=False, - wait_for_calc_timeout=0, - separate_files=True, - ) - - # Mock function for filename - def test_func(): - pass - - core.set_func(test_func) - - # Save with hash_str - test_entry = CacheEntry( - value="test_value", - time=datetime.now(), - stale=False, - _processing=False, - _completed=True, - ) - test_data = {"test_key": test_entry} - hash_str = "testhash123" - core._save_cache(test_data, hash_str=hash_str) - - # Check file exists with hash in name - expected_pattern = f"test_func_{hash_str}" - files = os.listdir(temp_dir) - assert any( - expected_pattern in f and f.endswith(hash_str) for f in files - ), f"Expected file ending with {hash_str} not found. Files: {files}" - - -# Test Pickle timeout during wait (line 398) -@pytest.mark.pickle -def test_pickle_timeout_during_wait(): - """Test calculation timeout while waiting in pickle backend.""" - import queue - import threading - - @cachier( - backend="pickle", - wait_for_calc_timeout=0.5, # Short timeout - ) - def slow_func(x): - sleep(2) # Longer than timeout - return x * 2 - - slow_func.clear_cache() - - res_queue = queue.Queue() - - def call_slow_func(): - try: - res = slow_func(42) - res_queue.put(("success", res)) - except Exception as e: - res_queue.put(("error", e)) - - # Start first thread that will take long - thread1 = threading.Thread(target=call_slow_func) - thread1.start() - - # Give it time to start processing - sleep(0.1) - - # Start second thread that should timeout waiting - thread2 = threading.Thread(target=call_slow_func) - thread2.start() - - # Wait for threads - thread1.join(timeout=3) - thread2.join(timeout=3) - - # Check results - at least one should have succeeded - results = [] - while not res_queue.empty(): - results.append(res_queue.get()) - - assert len(results) >= 1 - - slow_func.clear_cache() - - -# Test Pickle wait timeout check -@pytest.mark.pickle -def test_pickle_wait_timeout_check(): - """Test pickle backend timeout check during wait.""" - import threading - - @cachier(backend="pickle", wait_for_calc_timeout=0.2) - def slow_func(x): - sleep(1) # Longer than timeout - return x * 2 - - slow_func.clear_cache() - - results = [] - - def worker1(): - results.append(("w1", slow_func(42))) - - def worker2(): - sleep(0.1) # Let first start - results.append(("w2", slow_func(42))) - - t1 = threading.Thread(target=worker1) - t2 = threading.Thread(target=worker2) - - t1.start() - t2.start() - - t1.join(timeout=2) - t2.join(timeout=2) - - # Both should have results (timeout should have triggered recalc) - assert len(results) >= 1 - - slow_func.clear_cache() diff --git a/tests/test_redis_core.py b/tests/test_redis_core.py deleted file mode 100644 index 1ac8157c..00000000 --- a/tests/test_redis_core.py +++ /dev/null @@ -1,1268 +0,0 @@ -"""Testing the Redis core of cachier.""" - -import contextlib -import hashlib -import pickle -import queue -import sys -import threading -import time -import warnings -from datetime import datetime, timedelta -from random import random -from time import sleep -from unittest.mock import MagicMock, Mock, patch - -import pandas as pd -import pytest -from birch import Birch # type: ignore[import-not-found] - -from cachier import cachier -from cachier.cores.redis import MissingRedisClient, _RedisCore - -# === Enables testing vs a real Redis instance === - -try: - import redis - - REDIS_AVAILABLE = True -except ImportError: - REDIS_AVAILABLE = False - - -class CfgKey: - HOST = "TEST_REDIS_HOST" - PORT = "TEST_REDIS_PORT" - DB = "TEST_REDIS_DB" - TEST_VS_DOCKERIZED_REDIS = "TEST_VS_DOCKERIZED_REDIS" - - -CFG = Birch( - namespace="cachier", - defaults={CfgKey.TEST_VS_DOCKERIZED_REDIS: False}, -) - - -def _get_test_redis_client(): - """Get a Redis client for testing.""" - if not REDIS_AVAILABLE: - pytest.skip("Redis not available") - - if str(CFG.mget(CfgKey.TEST_VS_DOCKERIZED_REDIS)).lower() == "true": - print("Using live Redis instance for testing.") - host = CFG.get(CfgKey.HOST, "localhost") - port = int(CFG.get(CfgKey.PORT, 6379)) - db = int(CFG.get(CfgKey.DB, 0)) - try: - client = redis.Redis( - host=host, port=port, db=db, decode_responses=False - ) - # Test connection - client.ping() - return client - except redis.ConnectionError as e: - print(f"Failed to connect to Redis: {e}") - pytest.skip("Redis not available") - else: - print("Using mock Redis for testing.") - # For testing without Redis, we'll use a mock - return None - - -def _test_redis_getter(): - """Get Redis client for testing.""" - client = _get_test_redis_client() - if client is None: - # Create a mock Redis client for testing - # Use a singleton pattern to ensure the same instance is returned - if not hasattr(_test_redis_getter, "_mock_client"): - - class MockRedis: - def __init__(self): - self.data = {} - print("DEBUG: MockRedis initialized") - - def hgetall(self, key): - result = self.data.get(key, {}) - # Convert string values to bytes to match Redis behavior - bytes_result = {} - for k, v in result.items(): - if isinstance(v, str): - bytes_result[k.encode("utf-8")] = v.encode("utf-8") - else: - bytes_result[k.encode("utf-8")] = v - print( - f"DEBUG: hgetall({key}) = {result} -> {bytes_result}" - ) - return bytes_result - - def hset( - self, key, field=None, value=None, mapping=None, **kwargs - ): - if key not in self.data: - self.data[key] = {} - - # Handle different calling patterns - if mapping is not None: - # Called with mapping dict - self.data[key].update(mapping) - elif field is not None and value is not None: - # Called with field, value arguments - self.data[key][field] = value - elif kwargs: - # Called with keyword arguments - self.data[key].update(kwargs) - - print( - f"DEBUG: hset({key}, field={field}, value={value}, " - f"mapping={mapping}, kwargs={kwargs}) -> " - f"{self.data[key]}" - ) - - def keys(self, pattern): - import re - - pattern = pattern.replace("*", ".*") - # Fix: keys are strings, not bytes, so no need to decode - result = [k for k in self.data if re.match(pattern, k)] - print(f"DEBUG: keys({pattern}) = {result}") - return result - - def delete(self, *keys): - for key in keys: - self.data.pop(key, None) - print(f"DEBUG: delete({keys})") - - def pipeline(self): - return MockPipeline(self) - - def ping(self): - return True - - def set(self, key, value): - self.data[key] = value - print(f"DEBUG: set({key}, {value})") - - def get(self, key): - result = self.data.get(key) - if isinstance(result, str): - result = result.encode("utf-8") - print(f"DEBUG: get({key}) = {result}") - return result - - class MockPipeline: - def __init__(self, redis_client): - self.redis_client = redis_client - self.commands = [] - - def hset(self, key, field, value): - self.commands.append(("hset", key, field, value)) - return self - - def execute(self): - for cmd, key, field, value in self.commands: - if cmd == "hset": - self.redis_client.hset( - key, field=field, value=value - ) - - _test_redis_getter._mock_client = MockRedis() - - return _test_redis_getter._mock_client - return client - - -# === Redis core tests === - - -@pytest.mark.redis -def test_information(): - if REDIS_AVAILABLE: - print(f"\nredis version: {redis.__version__}") - else: - print("\nredis not available") - - -@pytest.mark.redis -def test_redis_connection(): - """Test Redis connection with environment variables.""" - client = _get_test_redis_client() - if client is None: - pytest.skip("Redis not available") - - try: - # Test basic Redis operations - client.set("test_key", "test_value") - value = client.get("test_key") - assert value == b"test_value" - client.delete("test_key") - print("✓ Redis connection and basic operations working") - except Exception as e: - pytest.fail(f"Redis connection test failed: {e}") - - -@pytest.mark.redis -def test_redis_core(): - """Basic Redis core functionality.""" - - @cachier(backend="redis", redis_client=_test_redis_getter) - def _test_redis_caching(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _test_redis_caching.clear_cache() - val1 = _test_redis_caching(1, 2) - val2 = _test_redis_caching(1, 2) - assert val1 == val2 - val3 = _test_redis_caching(1, 2, cachier__skip_cache=True) - assert val3 != val1 - val4 = _test_redis_caching(1, 2) - assert val4 == val1 - val5 = _test_redis_caching(1, 2, cachier__overwrite_cache=True) - assert val5 != val1 - val6 = _test_redis_caching(1, 2) - assert val6 == val5 - - -@pytest.mark.redis -def test_redis_core_keywords(): - """Basic Redis core functionality with keyword arguments.""" - - @cachier(backend="redis", redis_client=_test_redis_getter) - def _tfunc_for_keywords(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _tfunc_for_keywords.clear_cache() - val1 = _tfunc_for_keywords(1, arg_2=2) - val2 = _tfunc_for_keywords(1, arg_2=2) - assert val1 == val2 - val3 = _tfunc_for_keywords(1, arg_2=2, cachier__skip_cache=True) - assert val3 != val1 - val4 = _tfunc_for_keywords(1, arg_2=2) - assert val4 == val1 - val5 = _tfunc_for_keywords(1, arg_2=2, cachier__overwrite_cache=True) - assert val5 != val1 - val6 = _tfunc_for_keywords(1, arg_2=2) - assert val6 == val5 - - -@pytest.mark.redis -def test_redis_stale_after(): - """Testing Redis core stale_after functionality.""" - - @cachier( - backend="redis", - redis_client=_test_redis_getter, - stale_after=timedelta(seconds=3), - next_time=False, - ) - def _stale_after_redis(arg_1, arg_2): - """Some function.""" - return random() + arg_1 + arg_2 - - _stale_after_redis.clear_cache() - val1 = _stale_after_redis(1, 2) - val2 = _stale_after_redis(1, 2) - assert val1 == val2 - sleep(3) - val3 = _stale_after_redis(1, 2) - assert val3 != val1 - - -def _calls_takes_time_redis(res_queue): - print("DEBUG: _calls_takes_time_redis started") - - @cachier(backend="redis", redis_client=_test_redis_getter) - def _takes_time(arg_1, arg_2): - """Some function.""" - print( - f"DEBUG: _calls_takes_time_redis._takes_time({arg_1}, {arg_2})" - " called" - ) - sleep(3) - result = random() + arg_1 + arg_2 - print( - f"DEBUG: _calls_takes_time_redis._takes_time({arg_1}, {arg_2}) " - f"returning {result}" - ) - return result - - print("DEBUG: _calls_takes_time_redis calling _takes_time(34, 82.3)") - res = _takes_time(34, 82.3) - print(f"DEBUG: _calls_takes_time_redis got result {res}, putting in queue") - res_queue.put(res) - print("DEBUG: _calls_takes_time_redis completed") - - -@pytest.mark.redis -def test_redis_being_calculated(): - """Testing Redis core handling of being calculated scenarios.""" - print("DEBUG: test_redis_being_calculated started") - - @cachier(backend="redis", redis_client=_test_redis_getter) - def _takes_time(arg_1, arg_2): - """Some function.""" - print(f"DEBUG: _takes_time({arg_1}, {arg_2}) called") - sleep(3) - result = random() + arg_1 + arg_2 - print(f"DEBUG: _takes_time({arg_1}, {arg_2}) returning {result}") - return result - - print("DEBUG: Clearing cache") - _takes_time.clear_cache() - res_queue = queue.Queue() - print("DEBUG: Starting thread1") - thread1 = threading.Thread( - target=_calls_takes_time_redis, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - print("DEBUG: Starting thread2") - thread2 = threading.Thread( - target=_calls_takes_time_redis, - kwargs={"res_queue": res_queue}, - daemon=True, - ) - print("DEBUG: Starting thread1") - thread1.start() - print("DEBUG: Sleeping 1 second") - sleep(1) - print("DEBUG: Starting thread2") - thread2.start() - print("DEBUG: Waiting for thread1 to join") - thread1.join() - print("DEBUG: Waiting for thread2 to join") - thread2.join() - print("DEBUG: Getting results from queue") - res1 = res_queue.get() - res2 = res_queue.get() - print(f"DEBUG: Results: res1={res1}, res2={res2}") - assert res1 == res2 - print("DEBUG: test_redis_being_calculated completed successfully") - - -@pytest.mark.redis -def test_redis_callable_hash_param(): - """Testing Redis core with callable hash function.""" - - def _hash_func(args, kwargs): - def _hash(obj): - if isinstance(obj, pd.DataFrame): - return hashlib.sha256( - obj.to_string().encode("utf-8") - ).hexdigest() - return str(obj) - - key_parts = [] - for arg in args: - key_parts.append(_hash(arg)) - for key, value in sorted(kwargs.items()): - key_parts.append(f"{key}:{_hash(value)}") - return hashlib.sha256(":".join(key_parts).encode("utf-8")).hexdigest() - - @cachier( - backend="redis", redis_client=_test_redis_getter, hash_func=_hash_func - ) - def _params_with_dataframe(*args, **kwargs): - """Function that can handle DataFrames.""" - return sum(len(str(arg)) for arg in args) + sum( - len(str(val)) for val in kwargs.values() - ) - - df1 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - df2 = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - df3 = pd.DataFrame({"a": [7, 8, 9], "b": [10, 11, 12]}) - - _params_with_dataframe.clear_cache() - val1 = _params_with_dataframe(df1, x=1) - val2 = _params_with_dataframe(df2, x=1) - assert val1 == val2 - val3 = _params_with_dataframe(df3, x=1) - assert val3 != val1 - - -@pytest.mark.redis -def test_redis_missing_client(): - """Test that MissingRedisClient is raised when no client is provided.""" - with pytest.raises(MissingRedisClient): - - @cachier(backend="redis") - def _test_func(): - return "test" - - -@pytest.mark.redis -def test_redis_core_direct(): - """Test Redis core directly.""" - redis_client = _test_redis_getter() - core = _RedisCore( - hash_func=None, - redis_client=redis_client, - wait_for_calc_timeout=None, - ) - - def test_func(x, y): - return x + y - - core.set_func(test_func) - - # Test setting and getting entries - core.set_entry("test_key", "test_value") - key, entry = core.get_entry_by_key("test_key") - assert entry is not None - assert entry.value == "test_value" - - # Test marking as being calculated - core.mark_entry_being_calculated("calc_key") - key, entry = core.get_entry_by_key("calc_key") - assert entry is not None - assert entry._processing is True - - # Test marking as not being calculated - core.mark_entry_not_calculated("calc_key") - key, entry = core.get_entry_by_key("calc_key") - assert entry is not None - assert entry._processing is False - - # Test clearing cache - core.clear_cache() - key, entry = core.get_entry_by_key("test_key") - assert entry is None - - -@pytest.mark.redis -def test_redis_callable_client(): - """Test Redis core with callable client.""" - - def get_redis_client(): - return _test_redis_getter() - - @cachier(backend="redis", redis_client=get_redis_client) - def _test_callable_client(arg_1, arg_2): - """Test function with callable Redis client.""" - return random() + arg_1 + arg_2 - - _test_callable_client.clear_cache() - val1 = _test_callable_client(1, 2) - val2 = _test_callable_client(1, 2) - assert val1 == val2 - - -def test_redis_import_warning(): - """Test that import warning is raised when redis is not available.""" - ptc = patch("cachier.cores.redis.REDIS_AVAILABLE", False) - with ptc, pytest.warns(ImportWarning, match="`redis` was not found"): - _RedisCore( - hash_func=None, - redis_client=Mock(), - wait_for_calc_timeout=None, - ) - - -@pytest.mark.redis -def test_missing_redis_client(): - """Test MissingRedisClient exception when redis_client is None.""" - with pytest.raises( - MissingRedisClient, match="must specify ``redis_client``" - ): - _RedisCore( - hash_func=None, - redis_client=None, - wait_for_calc_timeout=None, - ) - - -@pytest.mark.redis -def test_redis_core_exceptions(): - """Test exception handling in Redis core methods.""" - # Create a mock Redis client that raises exceptions - mock_client = MagicMock() - - # Configure all methods to raise exceptions - mock_client.hgetall = MagicMock( - side_effect=Exception("Redis connection error") - ) - mock_client.hset = MagicMock(side_effect=Exception("Redis write error")) - mock_client.keys = MagicMock(side_effect=Exception("Redis keys error")) - mock_client.delete = MagicMock(side_effect=Exception("Redis delete error")) - - core = _RedisCore( - hash_func=None, - redis_client=mock_client, - wait_for_calc_timeout=10, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Test get_entry_by_key exception handling - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - key, entry = core.get_entry_by_key("test_key") - assert key == "test_key" - assert entry is None - assert len(w) == 1 - assert "Redis get_entry_by_key failed" in str(w[0].message) - - # Test set_entry exception handling - # Mock the client to ensure it's not callable - test_mock_client = MagicMock() - test_mock_client.hset = MagicMock( - side_effect=Exception("Redis write error") - ) - - # Create a new core with this specific mock - test_core = _RedisCore( - hash_func=None, - redis_client=test_mock_client, - wait_for_calc_timeout=10, - ) - test_core.set_func(mock_func) - - # Override _should_store to return True - test_core._should_store = lambda x: True - - # Also need to mock _resolve_redis_client and _get_redis_key - test_core._resolve_redis_client = lambda: test_mock_client - test_core._get_redis_key = lambda key: f"test:{key}" - - with warnings.catch_warnings(record=True) as w2: - warnings.simplefilter("always") - result = test_core.set_entry("test_key", "test_value") - assert result is False - assert len(w2) == 1 - assert "Redis set_entry failed" in str(w2[0].message) - - # Mock _resolve_redis_client and _get_redis_key for the core - core._resolve_redis_client = lambda: mock_client - core._get_redis_key = lambda key: f"test:{key}" - - # Test mark_entry_being_calculated exception handling - with warnings.catch_warnings(record=True) as w3: - warnings.simplefilter("always") - core.mark_entry_being_calculated("test_key") - assert len(w3) == 1 - assert "Redis mark_entry_being_calculated failed" in str(w3[0].message) - - # Test mark_entry_not_calculated exception handling - with warnings.catch_warnings(record=True) as w4: - warnings.simplefilter("always") - core.mark_entry_not_calculated("test_key") - assert len(w4) == 1 - assert "Redis mark_entry_not_calculated failed" in str(w4[0].message) - - # Test clear_cache exception handling - with warnings.catch_warnings(record=True) as w5: - warnings.simplefilter("always") - core.clear_cache() - assert len(w5) == 1 - assert "Redis clear_cache failed" in str(w5[0].message) - - # Test clear_being_calculated exception handling - with warnings.catch_warnings(record=True) as w6: - warnings.simplefilter("always") - core.clear_being_calculated() - assert len(w6) == 1 - assert "Redis clear_being_calculated failed" in str(w6[0].message) - - -@pytest.mark.redis -def test_redis_delete_stale_entries(): - """Test delete_stale_entries method with various scenarios.""" - mock_client = MagicMock() - - core = _RedisCore( - hash_func=None, - redis_client=mock_client, - wait_for_calc_timeout=10, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Test normal operation - # Create a new mock client for this test - delete_mock_client = MagicMock() - - # Set up keys method - delete_mock_client.keys = MagicMock( - return_value=[b"key1", b"key2", b"key3"] - ) - - now = datetime.now() - old_timestamp = (now - timedelta(hours=2)).isoformat() - recent_timestamp = (now - timedelta(minutes=30)).isoformat() - - # Set up hmget responses - delete_mock_client.hmget = MagicMock( - side_effect=[ - [old_timestamp.encode("utf-8"), b"100"], # key1 - stale - [recent_timestamp.encode("utf-8"), b"100"], # key2 - not stale - [None, None], # key3 - no timestamp - ] - ) - - # Set up delete mock - delete_mock_client.delete = MagicMock() - - # Create a new core for this test - delete_core = _RedisCore( - hash_func=None, - redis_client=delete_mock_client, - wait_for_calc_timeout=10, - ) - delete_core.set_func(mock_func) - - # Need to mock _resolve_redis_client to return our mock - delete_core._resolve_redis_client = lambda: delete_mock_client - - delete_core.delete_stale_entries(timedelta(hours=1)) - - # Should only delete key1 - assert delete_mock_client.delete.call_count == 1 - delete_mock_client.delete.assert_called_with(b"key1") - - # Test exception during timestamp parsing - mock_client.reset_mock() - mock_client.keys.return_value = [b"key4"] - mock_client.hmget.return_value = [b"invalid-timestamp", None] - - # Need to mock _resolve_redis_client for the original core as well - core._resolve_redis_client = lambda: mock_client - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - core.delete_stale_entries(timedelta(hours=1)) - assert len(w) == 1 - assert "Redis timestamp parse failed" in str(w[0].message) - - # Test exception during keys operation - mock_client.reset_mock() - mock_client.keys.side_effect = Exception("Redis keys error") - - with warnings.catch_warnings(record=True) as w2: - warnings.simplefilter("always") - core.delete_stale_entries(timedelta(hours=1)) - assert len(w2) == 1 - assert "Redis delete_stale_entries failed" in str(w2[0].message) - - -@pytest.mark.redis -def test_redis_wait_on_entry_calc_no_entry(): - """Test wait_on_entry_calc when entry is None.""" - from cachier.cores.base import RecalculationNeeded - - # Create a mock client - mock_client = MagicMock() - - # Mock get_entry_by_key to always return None entry - # This avoids the pickle.loads issue - _ = _RedisCore.get_entry_by_key - - def mock_get_entry_by_key(self, key): - return key, None - - core = _RedisCore( - hash_func=None, - redis_client=mock_client, - wait_for_calc_timeout=10, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Patch the method - core.get_entry_by_key = lambda key: mock_get_entry_by_key(core, key) - - # The test expects RecalculationNeeded to be raised when no entry exists - with pytest.raises(RecalculationNeeded): - core.wait_on_entry_calc("test_key") - - -@pytest.mark.redis -def test_redis_set_entry_should_not_store(): - """Test set_entry when value should not be stored (None not allowed).""" - mock_client = MagicMock() - - core = _RedisCore( - hash_func=None, - redis_client=mock_client, - wait_for_calc_timeout=10, - ) - - # Mock _should_store to return False - core._should_store = Mock(return_value=False) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - result = core.set_entry("test_key", None) - assert result is False - mock_client.hset.assert_not_called() - - -@pytest.mark.redis -def test_redis_clear_being_calculated_with_pipeline(): - """Test clear_being_calculated with multiple keys.""" - # Create fresh mocks for this test - pipeline_mock_client = MagicMock() - pipeline_mock = MagicMock() - - # Set up keys to return 3 keys - pipeline_mock_client.keys = MagicMock( - return_value=[b"key1", b"key2", b"key3"] - ) - - # Set up pipeline - pipeline_mock_client.pipeline = MagicMock(return_value=pipeline_mock) - pipeline_mock.hset = MagicMock() - pipeline_mock.execute = MagicMock() - - core = _RedisCore( - hash_func=None, - redis_client=pipeline_mock_client, - wait_for_calc_timeout=10, - ) - - # Set a mock function - def mock_func(): - pass - - core.set_func(mock_func) - - # Need to mock _resolve_redis_client to return our mock - core._resolve_redis_client = lambda: pipeline_mock_client - - core.clear_being_calculated() - - # Verify pipeline was used - assert pipeline_mock.hset.call_count == 3 - # Verify hset was called with correct parameters for each key - pipeline_mock.hset.assert_any_call(b"key1", "processing", "false") - pipeline_mock.hset.assert_any_call(b"key2", "processing", "false") - pipeline_mock.hset.assert_any_call(b"key3", "processing", "false") - pipeline_mock.execute.assert_called_once() - - -# Test Redis import error handling (lines 14-15) -def test_redis_import_error_handling(): - """Test Redis backend when redis package is not available.""" - # This test is already covered by test_redis_import_warning - # but let's ensure the specific lines are hit - with patch.dict(sys.modules, {"redis": None}): - # Force reload of redis core module - if "cachier.cores.redis" in sys.modules: - del sys.modules["cachier.cores.redis"] - - # Test import failure - try: - from cachier.cores.redis import _RedisCore # noqa: F401 - - pytest.skip("Redis is installed, cannot test import error") - except ImportError: - pass # Expected behavior - - -# Test Redis corrupted entry handling (lines 112-114) -@pytest.mark.redis -def test_redis_corrupted_entry_handling(): - """Test Redis backend with corrupted cache entries.""" - import redis - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - - try: - # Test connection - client.ping() - except redis.ConnectionError: - pytest.skip("Redis server not available") - - @cachier(backend="redis", redis_client=client) - def test_func(x): - return x * 2 - - # Clear cache - test_func.clear_cache() - - # Manually insert corrupted data - cache_key = "cachier:test_coverage_gaps:test_func:somehash" - client.hset(cache_key, "value", b"corrupted_pickle_data") - client.hset(cache_key, "time", str(time.time()).encode()) - client.hset(cache_key, "stale", b"0") - client.hset(cache_key, "being_calculated", b"0") - - # Try to access - should handle corrupted data gracefully - result = test_func(42) - assert result == 84 - - test_func.clear_cache() - - -# TestRedis deletion failure during eviction (lines 133-135) -@pytest.mark.redis -def test_redis_deletion_failure_during_eviction(): - """Test Redis LRU eviction with deletion failures.""" - import redis - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - - try: - client.ping() - except redis.ConnectionError: - pytest.skip("Redis server not available") - - @cachier( - backend="redis", - redis_client=client, - cache_size_limit="100B", # Very small limit to trigger eviction - ) - def test_func(x): - return "x" * 50 # Large result to fill cache quickly - - # Clear cache - test_func.clear_cache() - - # Fill cache to trigger eviction - test_func(1) - - # Mock delete to fail - original_delete = client.delete - delete_called = [] - - def mock_delete(*args): - delete_called.append(args) - # Fail on first delete attempt - if len(delete_called) == 1: - raise redis.RedisError("Mocked deletion failure") - return original_delete(*args) - - client.delete = mock_delete - - try: - # This should trigger eviction and handle the deletion failure - test_func(2) - # Verify delete was attempted - assert len(delete_called) > 0 - finally: - client.delete = original_delete - test_func.clear_cache() - - -# Test Redis non-bytes timestamp handling (line 364) -@pytest.mark.redis -def test_redis_non_bytes_timestamp(): - """Test Redis backend with non-bytes timestamp values.""" - import redis - - from cachier.cores.redis import _RedisCore - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - - try: - client.ping() - except redis.ConnectionError: - pytest.skip("Redis server not available") - - @cachier( - backend="redis", redis_client=client, stale_after=timedelta(seconds=10) - ) - def test_func(x): - return x * 2 - - # Clear cache - test_func.clear_cache() - - # Create an entry - test_func(1) - - # Manually modify timestamp to be a string instead of bytes - keys = list( - client.scan_iter(match="cachier:test_coverage_gaps:test_func:*") - ) - if keys: - # Force timestamp to be a string (non-bytes) - client.hset(keys[0], "time", "not_a_number") - - # Create a separate core instance to test stale deletion - core = _RedisCore( - hash_func=None, - redis_client=client, - wait_for_calc_timeout=0, - ) - core.set_func(test_func) - - # Try to delete stale entries - should handle non-bytes timestamp - # gracefully - with contextlib.suppress(Exception): - core.delete_stale_entries(timedelta(seconds=1)) - - test_func.clear_cache() - - -# Test Redis missing import -@pytest.mark.redis -def test_redis_import_error(): - """Test Redis client initialization warning.""" - # Test creating a Redis core without providing a client - import warnings - - with warnings.catch_warnings(record=True): - warnings.simplefilter("always") - - with pytest.raises(Exception, match="redis_client"): - - @cachier(backend="redis", redis_client=None) - def test_func(): - return "test" - - -# Test Redis corrupted entry in LRU eviction -@pytest.mark.redis -def test_redis_lru_corrupted_entry(): - """Test Redis LRU eviction with corrupted entry.""" - import redis - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - try: - client.ping() - except redis.ConnectionError: - pytest.skip("Redis not available") - - @cachier( - backend="redis", - redis_client=client, - cache_size_limit="200B", # Small limit - ) - def test_func(x): - return f"result_{x}" * 10 # ~60 bytes per entry - - test_func.clear_cache() - - # Add valid entry - test_func(1) - - # Add corrupted entry manually - from cachier.cores.redis import _RedisCore - - core = _RedisCore( - hash_func=None, - redis_client=client, - wait_for_calc_timeout=0, - cache_size_limit="200B", - ) - core.set_func(test_func) - - # Create corrupted entry - bad_key = f"{core.key_prefix}:{core._func_str}:badkey" - client.hset(bad_key, "value", b"not_valid_pickle") - client.hset(bad_key, "time", str(time.time()).encode()) - client.hset(bad_key, "stale", b"0") - client.hset(bad_key, "being_calculated", b"0") - - # This should trigger eviction and handle the corrupted entry - test_func(2) - test_func(3) - - test_func.clear_cache() - - -# Test Redis deletion failure in eviction -@pytest.mark.redis -def test_redis_eviction_delete_failure(): - """Test Redis eviction handling delete failures.""" - import warnings - - import redis - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - try: - client.ping() - except redis.ConnectionError: - pytest.skip("Redis not available") - - # Create a unique function to avoid conflicts - @cachier(backend="redis", redis_client=client, cache_size_limit="150B") - def test_eviction_func(x): - return "x" * 50 # Large value - - test_eviction_func.clear_cache() - - # Fill cache to trigger eviction - test_eviction_func(100) - - # This should trigger eviction - with warnings.catch_warnings(record=True): - # Ignore warnings about eviction failures - warnings.simplefilter("always") - test_eviction_func(200) - - # Verify both values work (even if eviction had issues) - result1 = test_eviction_func(100) - result2 = test_eviction_func(200) - - assert result1 == "x" * 50 - assert result2 == "x" * 50 - - test_eviction_func.clear_cache() - - -# Test Redis stale deletion with size tracking -@pytest.mark.redis -def test_redis_stale_delete_size_tracking(): - """Test Redis stale deletion updates cache size.""" - import redis - - client = redis.Redis(host="localhost", port=6379, decode_responses=False) - try: - client.ping() - except redis.ConnectionError: - pytest.skip("Redis not available") - - @cachier( - backend="redis", - redis_client=client, - cache_size_limit="1KB", - stale_after=timedelta(seconds=0.1), - ) - def test_func(x): - return "data" * 20 - - test_func.clear_cache() - - # Create entries - test_func(1) - test_func(2) - - # Wait for staleness - sleep(0.2) - - # Get the core - from cachier.cores.redis import _RedisCore - - core = _RedisCore( - hash_func=None, - redis_client=client, - wait_for_calc_timeout=0, - cache_size_limit="1KB", - ) - core.set_func(test_func) - - # Delete stale entries - this should update cache size - core.delete_stale_entries(timedelta(seconds=0.1)) - - # Verify size tracking by adding new entry - test_func(3) - - test_func.clear_cache() - - -@pytest.mark.redis -def test_redis_lru_eviction_edge_cases(): - """Test Redis LRU eviction edge cases for coverage.""" - from cachier.cores.redis import _RedisCore - - redis_client = _test_redis_getter() - - # Test 1: Corrupted data during LRU eviction (lines 112-114) - core = _RedisCore( - hash_func=None, redis_client=redis_client, cache_size_limit=100 - ) - - def mock_func(x): - return x * 2 - - core.set_func(mock_func) - - # Add entries with corrupted metadata - for i in range(3): - key = core._get_redis_key(f"key{i}") - redis_client.hset(key, "value", pickle.dumps(i * 2)) - redis_client.hset( - key, "time", pickle.dumps(datetime.now().timestamp()) - ) - if i == 1: - # Corrupt metadata for one entry - redis_client.hset(key, "last_access", "invalid_json") - redis_client.hset(key, "size", "not_a_number") - else: - redis_client.hset(key, "last_access", str(time.time())) - redis_client.hset(key, "size", "20") - - # Set high cache size to trigger eviction - redis_client.set(core._cache_size_key, "1000") - - # Should handle corrupted entries gracefully - core._evict_lru_entries(redis_client, 1000) - - # Test 2: No eviction needed (line 138) - # Clear and set very low cache size - pattern = f"{core.key_prefix}:{core._func_str}:*" - for key in redis_client.scan_iter(match=pattern): - if b"__size__" not in key: - redis_client.delete(key) - - redis_client.set(core._cache_size_key, "10") - # Should not evict anything - core._evict_lru_entries(redis_client, 10) - - -@pytest.mark.redis -def test_redis_clear_and_delete_edge_cases(): - """Test Redis clear and delete operations edge cases.""" - from cachier.cores.redis import _RedisCore - - redis_client = _test_redis_getter() - - # Test 1: clear_being_calculated with no keys (line 325) - core = _RedisCore(hash_func=None, redis_client=redis_client) - - def mock_func(): - pass - - core.set_func(mock_func) - - # Ensure no keys exist - pattern = f"{core.key_prefix}:{core._func_str}:*" - for key in redis_client.scan_iter(match=pattern): - redis_client.delete(key) - - # Should handle empty key set gracefully - core.clear_being_calculated() - - # Test 2: delete_stale_entries with special keys (line 352) - core2 = _RedisCore(hash_func=None, redis_client=redis_client) - core2.stale_after = timedelta(seconds=1) - - def mock_func2(): - pass - - core2.set_func(mock_func2) - - # Add stale entries - for i in range(2): - key = core2._get_redis_key(f"entry{i}") - redis_client.hset(key, "value", pickle.dumps(f"value{i}")) - redis_client.hset( - key, - "timestamp", - (datetime.now() - timedelta(seconds=2)).isoformat(), - ) - - # Add special cache size key - redis_client.set(core2._cache_size_key, "100") - - # Delete stale - should skip special keys - core2.delete_stale_entries(timedelta(seconds=1)) - - # Special key should still exist - assert redis_client.exists(core2._cache_size_key) - - # Test 3: Non-bytes timestamp (line 364) - key = core2._get_redis_key("nonbytes") - redis_client.hset(key, "value", pickle.dumps("test")) - # String timestamp instead of bytes - redis_client.hset( - key, - "timestamp", - str((datetime.now() - timedelta(seconds=2)).isoformat()), - ) - - core2.delete_stale_entries(timedelta(seconds=1)) - # Should handle string timestamp - assert not redis_client.exists(key) - - -@pytest.mark.redis -def test_redis_delete_stale_size_handling(): - """Test Redis delete_stale_entries size handling.""" - from cachier.cores.redis import _RedisCore - - redis_client = _test_redis_getter() - - # Test 1: Corrupted size data (lines 374-375) - core = _RedisCore( - hash_func=None, redis_client=redis_client, cache_size_limit=1000 - ) - core.stale_after = timedelta(seconds=1) - - def mock_func(): - pass - - core.set_func(mock_func) - - # Add entries with one having corrupted size - for i in range(3): - key = core._get_redis_key(f"item{i}") - value = pickle.dumps(f"result{i}") - redis_client.hset(key, "value", value) - redis_client.hset( - key, - "time", - pickle.dumps((datetime.now() - timedelta(seconds=2)).timestamp()), - ) - if i == 1: - redis_client.hset(key, "size", "invalid_size") - else: - redis_client.hset(key, "size", str(len(value))) - - # Should handle corrupted size gracefully - core.delete_stale_entries(timedelta(seconds=1)) - - # Test 2: No cache_size_limit (line 380) - core2 = _RedisCore(hash_func=None, redis_client=redis_client) - core2.stale_after = timedelta(seconds=1) - core2.cache_size_limit = None - - def mock_func2(): - pass - - core2.set_func(mock_func2) - - # Add stale entries - for i in range(2): - key = core2._get_redis_key(f"old{i}") - redis_client.hset(key, "value", pickle.dumps(f"old{i}")) - redis_client.hset( - key, - "time", - pickle.dumps((datetime.now() - timedelta(seconds=2)).timestamp()), - ) - redis_client.hset(key, "size", "50") - - core2.delete_stale_entries(timedelta(seconds=1)) - - # Test 3: Nothing to delete (line 380) - core3 = _RedisCore( - hash_func=None, redis_client=redis_client, cache_size_limit=1000 - ) - core3.stale_after = timedelta(days=1) - - def mock_func3(): - pass - - core3.set_func(mock_func3) - - # Add fresh entries - for i in range(2): - key = core3._get_redis_key(f"fresh{i}") - redis_client.hset(key, "value", pickle.dumps(f"fresh{i}")) - redis_client.hset( - key, "time", pickle.dumps(datetime.now().timestamp()) - ) - redis_client.hset(key, "size", "30") - - # Nothing should be deleted - core3.delete_stale_entries(timedelta(days=1)) diff --git a/tests/test_sql_core.py b/tests/test_sql_core.py deleted file mode 100644 index a7fad33d..00000000 --- a/tests/test_sql_core.py +++ /dev/null @@ -1,537 +0,0 @@ -import os -import queue -import sys -import threading -from datetime import datetime, timedelta -from random import random -from time import sleep - -import pytest - -from cachier import cachier -from cachier.cores.base import RecalculationNeeded, _get_func_str -from cachier.cores.sql import _SQLCore - -SQL_CONN_STR = os.environ.get("SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:") - - -@pytest.mark.sql -def test_sql_core_basic(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def f(x, y): - return random() + x + y - - f.clear_cache() - v1 = f(1, 2) - v2 = f(1, 2) - assert v1 == v2 - v3 = f(1, 2, cachier__skip_cache=True) - assert v3 != v1 - v4 = f(1, 2) - assert v4 == v1 - v5 = f(1, 2, cachier__overwrite_cache=True) - assert v5 != v1 - v6 = f(1, 2) - assert v6 == v5 - - -@pytest.mark.sql -def test_sql_core_keywords(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def f(x, y): - return random() + x + y - - f.clear_cache() - v1 = f(1, y=2) - v2 = f(1, y=2) - assert v1 == v2 - v3 = f(1, y=2, cachier__skip_cache=True) - assert v3 != v1 - v4 = f(1, y=2) - assert v4 == v1 - v5 = f(1, y=2, cachier__overwrite_cache=True) - assert v5 != v1 - v6 = f(1, y=2) - assert v6 == v5 - - -@pytest.mark.sql -def test_sql_stale_after(): - @cachier( - backend="sql", - sql_engine=SQL_CONN_STR, - stale_after=timedelta(seconds=2), - next_time=False, - ) - def f(x, y): - return random() + x + y - - f.clear_cache() - v1 = f(1, 2) - v2 = f(1, 2) - assert v1 == v2 - sleep(2) - v3 = f(1, 2) - assert v3 != v1 - - -@pytest.mark.sql -def test_sql_overwrite_and_skip_cache(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def f(x): - return random() + x - - f.clear_cache() - v1 = f(1) - v2 = f(1) - assert v1 == v2 - v3 = f(1, cachier__skip_cache=True) - assert v3 != v1 - v4 = f(1, cachier__overwrite_cache=True) - assert v4 != v1 - v5 = f(1) - assert v5 == v4 - - -@pytest.mark.sql -def test_sql_concurrency(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def slow_func(x): - sleep(1) - return random() + x - - slow_func.clear_cache() - res_queue = queue.Queue() - - def call(): - res = slow_func(5) - res_queue.put(res) - - t1 = threading.Thread(target=call) - t2 = threading.Thread(target=call) - t1.start() - sleep(0.2) - t2.start() - t1.join(timeout=3) - t2.join(timeout=3) - assert res_queue.qsize() == 2 - r1 = res_queue.get() - r2 = res_queue.get() - assert r1 == r2 - - -@pytest.mark.sql -def test_sql_clear_being_calculated(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def slow_func(x): - sleep(1) - return random() + x - - slow_func.clear_cache() - slow_func(1) - slow_func.clear_being_calculated() - # Should not raise - slow_func(1) - - -@pytest.mark.sql -def test_sql_missing_entry(): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def f(x): - return x - - f.clear_cache() - # Should not raise - assert f(123) == 123 - - -class DummyWriteError(Exception): - pass - - -@pytest.mark.sql -def test_sql_failed_write(monkeypatch): - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def f(x): - return x - - f.clear_cache() - # Simulate DB failure by monkeypatching set_entry - orig = _SQLCore.set_entry - - def fail_set_entry(self, key, func_res): - raise DummyWriteError("fail") - - monkeypatch.setattr(_SQLCore, "set_entry", fail_set_entry) - with pytest.raises(DummyWriteError, match="fail"): - f(1) - monkeypatch.setattr(_SQLCore, "set_entry", orig) - - -@pytest.mark.sql -def test_import_cachier_without_sqlalchemy(monkeypatch): - """Test that importing cachier works when SQLAlchemy is missing. - - This should work unless SQL core is used. - - """ - # Simulate SQLAlchemy not installed - modules_backup = sys.modules.copy() - sys.modules["sqlalchemy"] = None - sys.modules["sqlalchemy.orm"] = None - sys.modules["sqlalchemy.engine"] = None - try: - import importlib # noqa: F401 - - import cachier # noqa: F401 - - # Should import fine - finally: - sys.modules.clear() - sys.modules.update(modules_backup) - - -@pytest.mark.sql -def test_sqlcore_importerror_without_sqlalchemy(monkeypatch): - """Test that using SQL core without SQLAlchemy raises an ImportError.""" - # Remove sql module from sys.modules to force reimport - if "cachier.cores.sql" in sys.modules: - del sys.modules["cachier.cores.sql"] - - # Mock the sqlalchemy import to raise ImportError - import builtins - - original_import = builtins.__import__ - - def mock_import(name, *args, **kwargs): - if name.startswith("sqlalchemy"): - raise ImportError(f"No module named '{name}'") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", mock_import) - - try: - # Now import sql - it should set SQLALCHEMY_AVAILABLE = False - import importlib - - sql_mod = importlib.import_module("cachier.cores.sql") - - # Verify that SQLALCHEMY_AVAILABLE is False - assert not sql_mod.SQLALCHEMY_AVAILABLE - - # Now trying to create _SQLCore should raise ImportError - with pytest.raises(ImportError) as excinfo: - sql_mod._SQLCore(hash_func=None, sql_engine="sqlite:///:memory:") - assert "SQLAlchemy is required" in str(excinfo.value) - finally: - # Clean up - remove the module so next tests reimport it fresh - if "cachier.cores.sql" in sys.modules: - del sys.modules["cachier.cores.sql"] - - -@pytest.mark.sql -def test_sqlcore_invalid_sql_engine(): - with pytest.raises( - ValueError, match="sql_engine must be a SQLAlchemy Engine" - ): - _SQLCore(hash_func=None, sql_engine=12345) - - -@pytest.mark.sql -def test_sqlcore_get_entry_by_key_none_value(): - import pytest - - pytest.importorskip("sqlalchemy") - import cachier.cores.sql as sql_mod - from cachier.cores.sql import _SQLCore - - CacheTable = getattr(sql_mod, "CacheTable", None) - if CacheTable is None: - pytest.skip("CacheTable not available (SQLAlchemy missing)") - core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) - core.set_func(lambda x: x) - # Insert a row with value=None - with core._Session() as session: - session.add( - CacheTable( - id="testfunc:abc", - function_id=core._func_str, - key="abc", - value=None, - timestamp=datetime.now(), - stale=False, - processing=False, - completed=True, - ) - ) - session.commit() - key, entry = core.get_entry_by_key("abc") - assert entry is not None - assert entry.value is None - - -@pytest.mark.sql -def test_sqlcore_set_entry_fallback(monkeypatch): - from sqlalchemy.orm import Session - from sqlalchemy.sql.dml import Insert, Update - from sqlalchemy.sql.selectable import Select - - core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) - core.set_func(lambda x: x) - # Monkeypatch Session.execute to simulate fallback path - orig_execute = Session.execute - - def fake_execute(self, stmt, *args, **kwargs): - if isinstance(stmt, (Insert, Update)): - - class FakeInsert: - pass - - return FakeInsert() - elif isinstance(stmt, Select): - - class FakeSelectResult: - def scalar_one_or_none(self): - return None # Simulate no row found - - return FakeSelectResult() - - class Dummy: - pass - - return Dummy() - - monkeypatch.setattr(Session, "execute", fake_execute) - # Should not raise - core.set_entry("fallback", 123) - monkeypatch.setattr(Session, "execute", orig_execute) - - -@pytest.mark.sql -def test_sqlcore_wait_on_entry_calc_recalculation(): - core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) - core.set_func(lambda x: x) - with pytest.raises(RecalculationNeeded): - core.wait_on_entry_calc("missing_key") - - -@pytest.mark.sql -def test_sqlcore_clear_being_calculated_empty(): - core = _SQLCore(hash_func=None, sql_engine=SQL_CONN_STR) - core.set_func(lambda x: x) - # Should not raise even if nothing is being calculated - core.clear_being_calculated() - - -@pytest.mark.sql -def test_sqlcore_accepts_engine_instance(): - from sqlalchemy import create_engine - - engine = create_engine(SQL_CONN_STR) - core = _SQLCore(hash_func=None, sql_engine=engine) - core.set_func(lambda x: x) - core.set_entry("engine_test", 456) - key, entry = core.get_entry_by_key("engine_test") - assert entry.value == 456 - - -@pytest.mark.sql -def test_sqlcore_accepts_engine_callable(): - from sqlalchemy import create_engine - - def engine_factory(): - return create_engine(SQL_CONN_STR) - - core = _SQLCore(hash_func=None, sql_engine=engine_factory) - core.set_func(lambda x: x) - core.set_entry("callable_test", 789) - key, entry = core.get_entry_by_key("callable_test") - assert entry.value == 789 - - -# Test SQL allow_none=False -@pytest.mark.sql -def test_sql_allow_none_false_not_stored(): - """Test SQL doesn't store None when allow_none=False.""" - SQL_CONN_STR = os.environ.get( - "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" - ) - call_count = 0 - - @cachier(backend="sql", sql_engine=SQL_CONN_STR, allow_none=False) - def returns_none(): - nonlocal call_count - call_count += 1 - return None - - returns_none.clear_cache() - - # First call - result1 = returns_none() - assert result1 is None - assert call_count == 1 - - # Second call should also execute - result2 = returns_none() - assert result2 is None - assert call_count == 2 - - returns_none.clear_cache() - - -# Test SQL delete_stale_entries direct call -@pytest.mark.sql -def test_sql_delete_stale_direct(): - """Test SQL stale entry deletion method.""" - from cachier.cores.sql import _SQLCore - - # Get the engine from environment or use default - SQL_CONN_STR = os.environ.get( - "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" - ) - - @cachier( - backend="sql", - sql_engine=SQL_CONN_STR, - stale_after=timedelta(seconds=0.5), - ) - def test_func(x): - return x * 2 - - test_func.clear_cache() - - # Create entries - test_func(1) - test_func(2) - - # Wait for staleness - sleep(0.6) - - # Create core instance for direct testing - core = _SQLCore( - hash_func=None, - sql_engine=SQL_CONN_STR, - wait_for_calc_timeout=0, - ) - core.set_func(test_func) - - # Delete stale entries - core.delete_stale_entries(timedelta(seconds=0.5)) - - test_func.clear_cache() - - -# Test Non-standard SQL database fallback -@pytest.mark.sql -def test_sql_non_standard_db(): - """Test SQL backend code coverage for set_entry method.""" - # This test improves coverage for the SQL set_entry method - SQL_CONN_STR = os.environ.get( - "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" - ) - - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def test_func(x): - return x * 3 - - test_func.clear_cache() - - # Test basic set/get functionality - result1 = test_func(10) - assert result1 == 30 - - # Test overwriting existing entry - result2 = test_func(10, cachier__overwrite_cache=True) - assert result2 == 30 - - # Test with None value when allow_none is True (default) - @cachier(backend="sql", sql_engine=SQL_CONN_STR, allow_none=True) - def returns_none_allowed(): - return None - - returns_none_allowed.clear_cache() - result3 = returns_none_allowed() - assert result3 is None - - # Second call should use cache - result4 = returns_none_allowed() - assert result4 is None - - test_func.clear_cache() - returns_none_allowed.clear_cache() - - -@pytest.mark.sql -def test_sql_should_store_false(): - """Test SQL set_entry when _should_store returns False (line 128).""" - from cachier.cores.sql import _SQLCore - - # Create core with entry size limit - core = _SQLCore( - sql_engine=SQL_CONN_STR, hash_func=None, entry_size_limit=10 - ) - - def mock_func(x): - return x - - core.set_func(mock_func) - - # Create a large object that exceeds the size limit - large_object = "x" * 1000 # Much larger than 10 bytes - - # set_entry with large object should return False - result = core.set_entry("test_key", large_object) - assert result is False - - -@pytest.mark.sql -def test_sql_on_conflict_do_update(): - """Test SQL on_conflict_do_update path (line 158).""" - # When running with PostgreSQL, this will test the - # on_conflict_do_update path - # With SQLite in memory, it will also support on_conflict_do_update - - @cachier(backend="sql", sql_engine=SQL_CONN_STR) - def test_func(x): - return x * 2 - - test_func.clear_cache() - - # First call - result1 = test_func(5) - assert result1 == 10 - - # Force an update scenario by marking stale - if "postgresql" in SQL_CONN_STR or "sqlite" in SQL_CONN_STR: - # Direct table manipulation to force update path - from sqlalchemy import create_engine, update - from sqlalchemy.orm import sessionmaker - - from cachier.cores.sql import CacheTable - - engine = create_engine(SQL_CONN_STR) - Session = sessionmaker(bind=engine) - session = Session() - - func_str = _get_func_str(test_func) - - # Mark as stale to force update - stmt = ( - update(CacheTable) - .where(CacheTable.function_id == func_str) - .values(stale=True) - ) - - try: - session.execute(stmt) - session.commit() - except Exception: - # If table doesn't exist or other issue, skip - # This is expected in some test configurations - pass - finally: - session.close() - - # Second call - will use on_conflict_do_update - result2 = test_func(5) - assert result2 == 10 From 00da69af25d72fc502e443c150322c073f607f05 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Sat, 28 Feb 2026 23:04:40 +0200 Subject: [PATCH 03/22] small fix --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2e68cd1c..8fd35f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Other/Nonlisted Topic", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", From ac3321777d220feb2762053de107b04f78175982 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 21:06:34 +0000 Subject: [PATCH 04/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyproject.toml | 16 ++++++++-------- tests/conftest.py | 33 +++++++++++---------------------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8fd35f5d..8a46104a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -207,6 +207,14 @@ markers = [ "seriallocal: local core tests that should run serially", ] +[tool.coverage.report] +show_missing = true +# Regexes for lines to exclude from consideration +exclude_lines = [ + "pragma: no cover", # Have to re-enable the standard pragma + "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: + "if TYPE_CHECKING:", # Is only true when running mypy, not tests +] # Parallel test execution configuration # Use: pytest -n auto (for automatic worker detection) # Or: pytest -n 4 (for specific number of workers) @@ -225,11 +233,3 @@ omit = [ "src/cachier/__init__.py", "**/scripts/**", ] -[tool.coverage.report] -show_missing = true -# Regexes for lines to exclude from consideration -exclude_lines = [ - "pragma: no cover", # Have to re-enable the standard pragma - "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: - "if TYPE_CHECKING:", # Is only true when running mypy, not tests -] diff --git a/tests/conftest.py b/tests/conftest.py index 49dc3cbd..136f35bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,9 +13,8 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): """Automatically inject worker-specific schema into SQL connection string. - This fixture enables parallel SQL test execution by giving each pytest- - xdist worker its own PostgreSQL schema, preventing table creation - conflicts. + This fixture enables parallel SQL test execution by giving each pytest- xdist worker its own PostgreSQL schema, + preventing table creation conflicts. """ # Only apply to SQL tests @@ -31,9 +30,7 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): return # Get the original SQL connection string - original_url = os.environ.get( - "SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:" - ) + original_url = os.environ.get("SQLALCHEMY_DATABASE_URL", "sqlite:///:memory:") if "postgresql" in original_url: # Create worker-specific schema name @@ -84,9 +81,7 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): # Use original URL to create schema (without search_path) engine = create_engine(original_url) with engine.connect() as conn: - conn.execute( - text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") - ) + conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}")) conn.commit() engine.dispose() except Exception as e: @@ -149,8 +144,8 @@ def worker_id(request): def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): """Ensure each test gets an isolated cache directory. - This is especially important for pickle tests when running in parallel. - Each pytest-xdist worker gets its own cache directory to avoid conflicts. + This is especially important for pickle tests when running in parallel. Each pytest-xdist worker gets its own cache + directory to avoid conflicts. """ if "pickle" in request.node.keywords: @@ -167,9 +162,7 @@ def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): # Monkeypatch the global cache directory for this test import cachier.config - monkeypatch.setattr( - cachier.config._global_params, "cache_dir", str(cache_dir) - ) + monkeypatch.setattr(cachier.config._global_params, "cache_dir", str(cache_dir)) # Also set environment variable as a backup monkeypatch.setenv("CACHIER_TEST_CACHE_DIR", str(cache_dir)) @@ -179,8 +172,8 @@ def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): def cleanup_test_schemas(request): """Clean up test schemas after all tests complete. - This fixture ensures that worker-specific PostgreSQL schemas created during - parallel test execution are properly cleaned up. + This fixture ensures that worker-specific PostgreSQL schemas created during parallel test execution are properly + cleaned up. """ yield # Let all tests run first @@ -206,9 +199,7 @@ def cleanup_test_schemas(request): query_params.pop("options", None) # Rebuild clean URL - clean_query = ( - urlencode(query_params, doseq=True) if query_params else "" - ) + clean_query = urlencode(query_params, doseq=True) if query_params else "" clean_url = urlunparse( ( parsed.scheme, @@ -223,9 +214,7 @@ def cleanup_test_schemas(request): engine = create_engine(clean_url) with engine.connect() as conn: # Drop the schema and all its contents - conn.execute( - text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE") - ) + conn.execute(text(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE")) conn.commit() engine.dispose() except Exception as e: From 233d105f2037b56bcb94b8602600f1974fb9e1a4 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 23:15:38 +0200 Subject: [PATCH 05/22] Fix ruff SIM108 lint error in conftest.py (#350) * Initial plan * Fix SIM108: use ternary operator for cache_dir assignment in conftest.py Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 136f35bc..509f5897 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -150,12 +150,7 @@ def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): """ if "pickle" in request.node.keywords: # Create a unique cache directory for this test - if worker_id == "master": - # Not running in parallel mode - cache_dir = tmp_path / "cachier_cache" - else: - # Running with pytest-xdist - use worker-specific directory - cache_dir = tmp_path / f"cachier_cache_{worker_id}" + cache_dir = tmp_path / "cachier_cache" if worker_id == "master" else tmp_path / f"cachier_cache_{worker_id}" cache_dir.mkdir(exist_ok=True, parents=True) From 1a1c5b92febf1bfda31d5a0e327808b4f9d7b06b Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:16:37 +0200 Subject: [PATCH 06/22] Update tests/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/README.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/README.md b/tests/README.md index 07b54a88..c1637a7d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -41,17 +41,19 @@ The Cachier test suite is designed to comprehensively test all caching backends tests/ ├── conftest.py # Shared fixtures and configuration ├── requirements.txt # Base test dependencies (includes pytest-rerunfailures) -├── mongodb_requirements.txt # MongoDB-specific dependencies -├── redis_requirements.txt # Redis-specific dependencies -├── sql_requirements.txt # SQL-specific dependencies +├── requirements_mongodb.txt # MongoDB-specific test dependencies +├── requirements_redis.txt # Redis-specific test dependencies +├── requirements_postgres.txt # PostgreSQL/SQL-specific test dependencies │ -├── test_*.py # Test modules -├── test_mongo_core.py # MongoDB-specific tests -├── test_redis_core.py # Redis-specific tests -├── test_sql_core.py # SQL-specific tests -├── test_memory_core.py # Memory backend tests -├── test_pickle_core.py # Pickle backend tests -├── test_general.py # Cross-backend tests +├── test_*.py # Backend-agnostic test modules +├── mongo_tests/ # MongoDB-specific tests +│ └── test_mongo_core.py +├── sql_tests/ # SQL-specific tests +│ └── test_sql_core.py +├── test_redis_core.py # Redis backend tests +├── test_memory_core.py # Memory backend tests +├── test_pickle_core.py # Pickle backend tests +├── test_general.py # Cross-backend tests └── ... ``` From 2d62c99204d6826bb6c1af19a354653fb70cb500 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:17:00 +0200 Subject: [PATCH 07/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index b39a3a51..debab1b3 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -540,7 +540,14 @@ main() { # Build pytest command PYTEST_CMD="pytest" # and the specific pytest command for running serial pickle tests - SERIAL_PYTEST_CMD="pytest -m seriallocal -n0" + SERIAL_PYTEST_CMD="pytest -m seriallocal" + # Only add -n0 if pytest-xdist is available; otherwise, plain pytest is already serial + if python - << 'EOF' >/dev/null 2>&1 +import xdist # noqa: F401 +EOF + then + SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD -n0" + fi # Add test files if specified if [ -n "$TEST_FILES" ]; then From 123a7941f1dce79478e009d163b8f53247a8eafb Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Sat, 28 Feb 2026 23:18:15 +0200 Subject: [PATCH 08/22] documet flaky test marker in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8a46104a..0b3c80ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,6 +204,7 @@ markers = [ "maxage: test the max_age functionality", "asyncio: marks tests as async", "smoke: fast smoke tests with no external service dependencies", + "flaky: tests that are known to be flaky and should be retried", "seriallocal: local core tests that should run serially", ] From 5f33ab17ef994914c32734be0314102f626bc6ee Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:19:17 +0200 Subject: [PATCH 09/22] Update pyproject.toml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0b3c80ca..13759f81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,7 +192,7 @@ addopts = [ "-v", "-s", "-W error", - # Note: parallel execution is opt-in via --parallel flag or -n option + # Note: parallel execution is opt-in via -n/--numprocesses (pytest-xdist) ] markers = [ "mongo: test the MongoDB core", From a2032e58b9f93de199a5b9f8175bbc6370ec5702 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:19:43 +0200 Subject: [PATCH 10/22] Update tests/conftest.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/conftest.py | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 509f5897..d9ca8ea0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,49 +91,6 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): yield -@pytest.fixture(scope="session", autouse=True) -def cleanup_mongo_clients(): - """Clean up any MongoDB clients created during tests. - - This fixture runs automatically after all tests complete. - - """ - # Let tests run - yield - - # Cleanup after all tests - import contextlib - - try: - from tests.test_mongo_core import _mongo_clients, _test_mongetter - - # Close all tracked MongoDB clients - for client in _mongo_clients: - with contextlib.suppress(Exception): - client.close() - - # Clear the list for next test run - _mongo_clients.clear() - - # Also clean up _test_mongetter specifically - if hasattr(_test_mongetter, "client"): - # Remove the client attribute so future test runs start fresh - delattr(_test_mongetter, "client") - - # Clean up any _custom_mongetter functions that may have been created - import tests.test_mongo_core - - for attr_name in dir(tests.test_mongo_core): - attr = getattr(tests.test_mongo_core, attr_name) - if callable(attr) and hasattr(attr, "client"): - delattr(attr, "client") - - except (ImportError, AttributeError): - # If the module wasn't imported or client wasn't created, - # then there's nothing to clean up - pass - - @pytest.fixture def worker_id(request): """Get the pytest-xdist worker ID.""" From c4548e0fb922b7be621b6a1822729ffebf5ea3e2 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:20:13 +0200 Subject: [PATCH 11/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index debab1b3..63bd24dc 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -443,7 +443,7 @@ stop_postgres() { } test_sql() { - export SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" + export SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" } # Main execution From 52114a6bc1e4d974224219faf682f7c6d62d16e6 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:20:34 +0200 Subject: [PATCH 12/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index 63bd24dc..f3e693e9 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -269,7 +269,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "mongo"; then if ! python -c "import pymongo" 2>/dev/null; then print_message $YELLOW "Installing MongoDB test requirements..." - pip install -r tests/mongodb_requirements.txt || { + pip install -r tests/requirements_mongodb.txt || { print_message $RED "Failed to install MongoDB requirements" exit 1 } From 5a9cd4dd0bd502d8f62ab791fd770d2908cd4b63 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:21:22 +0200 Subject: [PATCH 13/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index f3e693e9..61873558 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -280,7 +280,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "redis"; then if ! python -c "import redis" 2>/dev/null; then print_message $YELLOW "Installing Redis test requirements..." - pip install -r tests/redis_requirements.txt || { + pip install -r tests/requirements_redis.txt || { print_message $RED "Failed to install Redis requirements" exit 1 } From 3cd61834b8f276c2187df3d47d201ef64f1514ac Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:21:33 +0200 Subject: [PATCH 14/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index 61873558..3fd1e271 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -291,7 +291,7 @@ check_dependencies() { if echo "$SELECTED_CORES" | grep -qw "sql"; then if ! python -c "import sqlalchemy" 2>/dev/null; then print_message $YELLOW "Installing SQL test requirements..." - pip install -r tests/sql_requirements.txt || { + pip install -r tests/requirements_postgres.txt || { print_message $RED "Failed to install SQL requirements" exit 1 } From 00f22d6fdaa81abdda186a4e1aeacfb389d1b3e1 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Sat, 28 Feb 2026 23:21:49 +0200 Subject: [PATCH 15/22] Update scripts/test-local.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/test-local.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index 3fd1e271..1f03d765 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -534,7 +534,11 @@ main() { sql) test_sql ;; esac done - pytest_markers="$pytest_markers and not seriallocal" + if [ -n "$pytest_markers" ]; then + pytest_markers="($pytest_markers) and not seriallocal" + else + pytest_markers="not seriallocal" + fi # Run pytest # Build pytest command From c30e2ca6032533f6368aadfe7b3f8e1086c1a008 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 23:23:02 +0200 Subject: [PATCH 16/22] Fix CI/CD Integration section in tests/README.md to match actual workflow (#351) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index c1637a7d..acb8214f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -335,18 +335,29 @@ pytest -n auto ### GitHub Actions -The CI pipeline tests all backends: +The CI pipeline runs a matrix job per backend. Each backend uses the commands below: -```yaml -# Local backends run in parallel -pytest -m "memory or pickle" -n auto +```bash +# Local backends (memory, pickle, and other non-external tests) +pytest -m "not mongo and not sql and not redis and not s3" -# External backends run sequentially for stability +# MongoDB backend pytest -m mongo -pytest -m redis + +# PostgreSQL/SQL backend pytest -m sql + +# Redis backend +pytest -m redis + +# S3 backend +pytest -m s3 ``` +Note: local tests do not use `pytest-xdist` (`-n`) in CI. External backends +(MongoDB, PostgreSQL, Redis, S3) each run in their own isolated matrix job with +the corresponding Docker service started beforehand. + ### Environment Variables - `CACHIER_TEST_VS_DOCKERIZED_MONGO`: Use real MongoDB in CI From 78bd3547ac29d2c1f7a0128ab122111a10a5b590 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 23:23:39 +0200 Subject: [PATCH 17/22] Fix SQL test module import path in parallel test fixture (#352) * Fix SQL test module path in conftest.py fixture Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> * Initial plan --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d9ca8ea0..dce44dc7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -70,9 +70,9 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): monkeypatch.setenv("SQLALCHEMY_DATABASE_URL", new_url) # Also patch the SQL_CONN_STR constant used in tests - import tests.test_sql_core + import tests.sql_tests.test_sql_core - monkeypatch.setattr(tests.test_sql_core, "SQL_CONN_STR", new_url) + monkeypatch.setattr(tests.sql_tests.test_sql_core, "SQL_CONN_STR", new_url) # Ensure schema creation by creating it before tests run try: From 12d6b2aaf74243cf09d804ef2e97676b3677b5bc Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Feb 2026 23:40:48 +0200 Subject: [PATCH 18/22] Extend isolated_cache_directory fixture to cover maxage-marked tests (#354) * Initial plan * Extend isolated_cache_directory fixture to cover maxage-marked tests Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dce44dc7..d138449e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,11 +101,11 @@ def worker_id(request): def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): """Ensure each test gets an isolated cache directory. - This is especially important for pickle tests when running in parallel. Each pytest-xdist worker gets its own cache - directory to avoid conflicts. + This is especially important for pickle and maxage tests when running in parallel. Each pytest-xdist worker gets its + own cache directory to avoid conflicts. """ - if "pickle" in request.node.keywords: + if "pickle" in request.node.keywords or "maxage" in request.node.keywords: # Create a unique cache directory for this test cache_dir = tmp_path / "cachier_cache" if worker_id == "master" else tmp_path / f"cachier_cache_{worker_id}" From c4b2e31f691085fa29456bf340ed8682faf53005 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 00:20:28 +0200 Subject: [PATCH 19/22] Restore pytest-asyncio to base test requirements (#355) * Initial plan * Add pytest-asyncio back to tests/requirements.txt Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index c0fe4d4c..f4e666d0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ # todo: add some version range or pinning latest versions # tests and coverages pytest +pytest-asyncio pytest-xdist # for parallel test execution pytest-rerunfailures # for retrying flaky tests coverage From 4a4f2ca05e0d1f40f6c15fe9af69ed8b37aec618 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 11:04:06 +0200 Subject: [PATCH 20/22] Fix serial-mode regression in pickle cache file manipulation tests (#357) * Initial plan * Fix regression: only apply cache isolation in parallel (xdist) mode Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> * Apply ruff format to conftest.py to fix pre-commit.ci formatting error Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: shaypal5 <917954+shaypal5@users.noreply.github.com> --- tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d138449e..e2657784 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -104,10 +104,12 @@ def isolated_cache_directory(tmp_path, monkeypatch, request, worker_id): This is especially important for pickle and maxage tests when running in parallel. Each pytest-xdist worker gets its own cache directory to avoid conflicts. + Only applies when running in parallel mode (pytest-xdist), to avoid breaking tests that use module-level path + constants computed from the default cache directory at import time. + """ - if "pickle" in request.node.keywords or "maxage" in request.node.keywords: - # Create a unique cache directory for this test - cache_dir = tmp_path / "cachier_cache" if worker_id == "master" else tmp_path / f"cachier_cache_{worker_id}" + if worker_id != "master" and ("pickle" in request.node.keywords or "maxage" in request.node.keywords): + cache_dir = tmp_path / f"cachier_cache_{worker_id}" cache_dir.mkdir(exist_ok=True, parents=True) From fd63d61f5544df60857eda9c62e9bd7e7314d0d7 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 6 Mar 2026 09:52:43 +0100 Subject: [PATCH 21/22] Apply suggestions from code review --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e2657784..57d5dba0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,7 +62,7 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): parsed.path, parsed.params, new_query, - parsed.fragment, + parsed.fragment ) ) @@ -161,7 +161,7 @@ def cleanup_test_schemas(request): parsed.path, parsed.params, clean_query, - parsed.fragment, + parsed.fragment ) ) From 441ad802fe5484dfc63a913bade7bead3ad8cd26 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Mar 2026 10:10:49 +0100 Subject: [PATCH 22/22] Enable parallel test execution in CI with -n auto (#359) * Initial plan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add parallel test execution (-n auto) to CI workflow Co-authored-by: Borda <6035284+Borda@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Borda <6035284+Borda@users.noreply.github.com> --- .github/workflows/ci-test.yml | 10 +++++----- tests/README.md | 18 ++++++++++-------- tests/conftest.py | 20 ++------------------ 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 409701cc..b22b613a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -115,7 +115,7 @@ jobs: - name: Unit tests (local) if: matrix.backend == 'local' - run: pytest -m "not mongo and not sql and not redis and not s3" --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m "not mongo and not sql and not redis and not s3" -n auto --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Setup docker (missing on MacOS) if: runner.os == 'macOS' && matrix.backend == 'mongodb' @@ -148,7 +148,7 @@ jobs: - name: Unit tests (DB) if: matrix.backend == 'mongodb' - run: pytest -m "mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m "mongo" -n auto --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Speed eval run: python tests/speed_eval.py @@ -169,7 +169,7 @@ jobs: if: matrix.backend == 'postgres' env: SQLALCHEMY_DATABASE_URL: postgresql+psycopg://testuser:testpass@localhost:5432/testdb - run: pytest -m sql --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m sql -n auto --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Start Redis in docker if: matrix.backend == 'redis' @@ -183,11 +183,11 @@ jobs: - name: Unit tests (Redis) if: matrix.backend == 'redis' - run: pytest -m redis --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m redis -n auto --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Unit tests (S3) if: matrix.backend == 's3' - run: pytest -m s3 --cov=cachier --cov-report=term --cov-report=xml:cov.xml + run: pytest -m s3 -n auto --cov=cachier --cov-report=term --cov-report=xml:cov.xml - name: Upload coverage to Codecov (non PRs) continue-on-error: true diff --git a/tests/README.md b/tests/README.md index acb8214f..de92da27 100644 --- a/tests/README.md +++ b/tests/README.md @@ -339,24 +339,26 @@ The CI pipeline runs a matrix job per backend. Each backend uses the commands be ```bash # Local backends (memory, pickle, and other non-external tests) -pytest -m "not mongo and not sql and not redis and not s3" +pytest -m "not mongo and not sql and not redis and not s3" -n auto # MongoDB backend -pytest -m mongo +pytest -m mongo -n auto # PostgreSQL/SQL backend -pytest -m sql +pytest -m sql -n auto # Redis backend -pytest -m redis +pytest -m redis -n auto # S3 backend -pytest -m s3 +pytest -m s3 -n auto ``` -Note: local tests do not use `pytest-xdist` (`-n`) in CI. External backends -(MongoDB, PostgreSQL, Redis, S3) each run in their own isolated matrix job with -the corresponding Docker service started beforehand. +All backends use `pytest-xdist` (`-n auto`) in CI for parallel test execution. +Each backend runs in its own isolated matrix job with the corresponding Docker +service started beforehand. Per-worker isolation is handled automatically by +the fixtures in `conftest.py` (separate cache directories for pickle/maxage +tests, separate PostgreSQL schemas for SQL tests). ### Environment Variables diff --git a/tests/conftest.py b/tests/conftest.py index 57d5dba0..9f0d47f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,16 +55,7 @@ def inject_worker_schema_for_sql_tests(monkeypatch, request): # Rebuild the URL with updated query parameters new_query = urlencode(query_params, doseq=True) - new_url = urlunparse( - ( - parsed.scheme, - parsed.netloc, - parsed.path, - parsed.params, - new_query, - parsed.fragment - ) - ) + new_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, new_query, parsed.fragment)) # Override both the environment variable and the module constant monkeypatch.setenv("SQLALCHEMY_DATABASE_URL", new_url) @@ -155,14 +146,7 @@ def cleanup_test_schemas(request): # Rebuild clean URL clean_query = urlencode(query_params, doseq=True) if query_params else "" clean_url = urlunparse( - ( - parsed.scheme, - parsed.netloc, - parsed.path, - parsed.params, - clean_query, - parsed.fragment - ) + (parsed.scheme, parsed.netloc, parsed.path, parsed.params, clean_query, parsed.fragment) ) engine = create_engine(clean_url)