PYTHON-5683: Spike: Investigate using Rust for Extension Modules

.evergreen/generated_configs/functions.yml

@@ -111,6 +111,8 @@ functions:
           - LOAD_BALANCER
           - LOCAL_ATLAS
           - NO_EXT
+          - PYMONGO_BUILD_RUST
+          - PYMONGO_USE_RUST
       type: test
     - command: expansions.update
       params:
@@ -152,6 +154,8 @@ functions:
           - IS_WIN32
           - REQUIRE_FIPS
           - TEST_MIN_DEPS
+          - PYMONGO_BUILD_RUST
+          - PYMONGO_USE_RUST
       type: test
     - command: subprocess.exec
       params:

.evergreen/generated_configs/tasks.yml

@@ -2554,6 +2554,21 @@ tasks:
       - func: attach benchmark test results
       - func: send dashboard data
     tags: [perf]
+  - name: perf-8.0-standalone-ssl-rust
+    commands:
+      - func: run server
+        vars:
+          VERSION: v8.0-perf
+          SSL: ssl
+      - func: run tests
+        vars:
+          TEST_NAME: perf
+          SUB_TEST_NAME: rust
+          PYMONGO_BUILD_RUST: "1"
+          PYMONGO_USE_RUST: "1"
+      - func: attach benchmark test results
+      - func: send dashboard data
+    tags: [perf]
   - name: perf-8.0-standalone
     commands:
       - func: run server
@@ -2580,6 +2595,21 @@ tasks:
       - func: attach benchmark test results
       - func: send dashboard data
     tags: [perf]
+  - name: perf-8.0-standalone-rust
+    commands:
+      - func: run server
+        vars:
+          VERSION: v8.0-perf
+          SSL: nossl
+      - func: run tests
+        vars:
+          TEST_NAME: perf
+          SUB_TEST_NAME: rust
+          PYMONGO_BUILD_RUST: "1"
+          PYMONGO_USE_RUST: "1"
+      - func: attach benchmark test results
+      - func: send dashboard data
+    tags: [perf]
 
   # Search index tests
   - name: test-search-index-helpers

.evergreen/generated_configs/variants.yml

@@ -477,6 +477,40 @@ buildvariants:
     expansions:
       SUB_TEST_NAME: pyopenssl
 
+  # Rust tests
+  - name: test-with-rust-extension
+    tasks:
+      - name: .test-standard .server-latest .pr
+    display_name: Test with Rust Extension
+    run_on:
+      - rhel87-small
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust, pr]
+  - name: test-with-rust-extension---macos-arm64
+    tasks:
+      - name: .test-standard .server-latest !.pr
+    display_name: Test with Rust Extension - macOS ARM64
+    run_on:
+      - macos-14-arm64
+    batchtime: 10080
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust]
+  - name: test-with-rust-extension---windows
+    tasks:
+      - name: .test-standard .server-latest !.pr
+    display_name: Test with Rust Extension - Windows
+    run_on:
+      - windows-64-vsMulti-small
+    batchtime: 10080
+    expansions:
+      PYMONGO_BUILD_RUST: "1"
+      PYMONGO_USE_RUST: "1"
+    tags: [rust]
+
   # Search index tests
   - name: search-index-helpers-rhel8
     tasks:

.evergreen/scripts/configure-env.sh

@@ -14,6 +14,7 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
+RUSTUP_HOME=${RUSTUP_HOME:-${CARGO_HOME}}
 UV_TOOL_DIR=$PROJECT_DIRECTORY/.local/uv/tools
 UV_CACHE_DIR=$PROJECT_DIRECTORY/.local/uv/cache
 DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
@@ -27,13 +28,14 @@ else
   PYMONGO_BIN_DIR=$HOME/cli_bin
 fi
 
-PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:\$PATH"
+PATH_EXT="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PYMONGO_BIN_DIR:$CARGO_HOME/bin:\$PATH"
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
+    RUSTUP_HOME=$(cygpath -m $RUSTUP_HOME)
     UV_TOOL_DIR=$(cygpath -m "$UV_TOOL_DIR")
     UV_CACHE_DIR=$(cygpath -m "$UV_CACHE_DIR")
     DRIVERS_TOOLS_BINARIES=$(cygpath -m "$DRIVERS_TOOLS_BINARIES")
@@ -62,6 +64,7 @@ export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 
 export CARGO_HOME="$CARGO_HOME"
+export RUSTUP_HOME="$RUSTUP_HOME"
 export UV_TOOL_DIR="$UV_TOOL_DIR"
 export UV_CACHE_DIR="$UV_CACHE_DIR"
 export UV_TOOL_BIN_DIR="$DRIVERS_TOOLS_BINARIES"

.evergreen/scripts/generate_config.py

@@ -958,18 +958,24 @@ def create_search_index_tasks():
 
 def create_perf_tasks():
     tasks = []
-    for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]):
+    for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async", "rust"]):
         vars = dict(VERSION=f"v{version}-perf", SSL=ssl)
         server_func = FunctionCall(func="run server", vars=vars)
-        vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync)
-        test_func = FunctionCall(func="run tests", vars=vars)
+        test_vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync)
+        # Enable Rust for rust perf tests
+        if sync == "rust":
+            test_vars["PYMONGO_BUILD_RUST"] = "1"
+            test_vars["PYMONGO_USE_RUST"] = "1"
+        test_func = FunctionCall(func="run tests", vars=test_vars)
         attach_func = FunctionCall(func="attach benchmark test results")
         send_func = FunctionCall(func="send dashboard data")
         task_name = f"perf-{version}-standalone"
         if ssl == "ssl":
             task_name += "-ssl"
         if sync == "async":
             task_name += "-async"
+        elif sync == "rust":
+            task_name += "-rust"
         tags = ["perf"]
         commands = [server_func, test_func, attach_func, send_func]
         tasks.append(EvgTask(name=task_name, tags=tags, commands=commands))
@@ -1189,6 +1195,8 @@ def create_run_server_func():
         "LOAD_BALANCER",
         "LOCAL_ATLAS",
         "NO_EXT",
+        "PYMONGO_BUILD_RUST",
+        "PYMONGO_USE_RUST",
     ]
     args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"]
     sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
@@ -1222,6 +1230,8 @@ def create_run_tests_func():
         "IS_WIN32",
         "REQUIRE_FIPS",
         "TEST_MIN_DEPS",
+        "PYMONGO_BUILD_RUST",
+        "PYMONGO_USE_RUST",
     ]
     args = [".evergreen/just.sh", "setup-tests", "${TEST_NAME}", "${SUB_TEST_NAME}"]
     setup_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args)
@@ -1283,6 +1293,55 @@ def create_send_dashboard_data_func():
     return "send dashboard data", cmds
 
 
+def create_rust_variants():
+    """Create build variants that test with Rust extension alongside C extension."""
+    variants = []
+
+    # Test Rust on Linux (primary platform) - runs on PRs
+    # Run standard tests with Rust enabled (both sync and async)
+    variant = create_variant(
+        [".test-standard .server-latest .pr"],
+        "Test with Rust Extension",
+        host=DEFAULT_HOST,
+        tags=["rust", "pr"],
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    # Test on macOS ARM64 (important for M1/M2 Macs)
+    variant = create_variant(
+        [".test-standard .server-latest !.pr"],
+        "Test with Rust Extension - macOS ARM64",
+        host=HOSTS["macos-arm64"],
+        tags=["rust"],
+        batchtime=BATCHTIME_WEEK,
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    # Test on Windows (important for cross-platform compatibility)
+    variant = create_variant(
+        [".test-standard .server-latest !.pr"],
+        "Test with Rust Extension - Windows",
+        host=HOSTS["win64"],
+        tags=["rust"],
+        batchtime=BATCHTIME_WEEK,
+        expansions=dict(
+            PYMONGO_BUILD_RUST="1",
+            PYMONGO_USE_RUST="1",
+        ),
+    )
+    variants.append(variant)
+
+    return variants
+
+
 mod = sys.modules[__name__]
 write_variants_to_file(mod)
 write_tasks_to_file(mod)

.evergreen/scripts/install-dependencies.sh

@@ -30,7 +30,7 @@ fi
 
 # Ensure just is installed.
 if ! command -v just &>/dev/null; then
-  uv tool install rust-just
+  uv tool install rust-just || uv tool install --force rust-just
 fi
 
 popd > /dev/null

.evergreen/scripts/install-rust.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Install Rust toolchain for building the Rust BSON extension.
+set -eu
+
+echo "Installing Rust toolchain..."
+
+# Check if Rust is already installed
+if command -v cargo &> /dev/null; then
+    echo "Rust is already installed:"
+    rustc --version
+    cargo --version
+    echo "Updating Rust toolchain..."
+    rustup update stable
+else
+    echo "Rust not found. Installing Rust..."
+
+    # Install Rust using rustup
+    if [ "Windows_NT" = "${OS:-}" ]; then
+        # Windows installation
+        curl --proto '=https' --tlsv1.2 -sSf https://win.rustup.rs/x86_64 -o rustup-init.exe
+        ./rustup-init.exe -y --default-toolchain stable
+        rm rustup-init.exe
+
+        # Add to PATH for current session
+        export PATH="$HOME/.cargo/bin:$PATH"
+    else
+        # Unix-like installation (Linux, macOS)
+        # Ensure CARGO_HOME is exported so rustup uses it
+        export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+        export RUSTUP_HOME="${RUSTUP_HOME:-${CARGO_HOME}}"
+
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+
+        # Source cargo env from the installation location
+        # On CI, CARGO_HOME is set to ${DRIVERS_TOOLS}/.cargo by configure-env.sh
+        CARGO_ENV_PATH="${CARGO_HOME}/env"
+
+        if [ -f "${CARGO_ENV_PATH}" ]; then
+            source "${CARGO_ENV_PATH}"
+        else
+            echo "Error: Cargo env file not found at ${CARGO_ENV_PATH}"
+            echo "CARGO_HOME=${CARGO_HOME}"
+            echo "RUSTUP_HOME=${RUSTUP_HOME}"
+            echo "HOME=${HOME}"
+            exit 1
+        fi
+    fi
+
+    echo "Rust installation complete:"
+    rustc --version
+    cargo --version
+fi
+
+# Ensure default toolchain is set (needed for rustup to work properly)
+echo "Setting default toolchain to stable..."
+rustup default stable
+
+# Install maturin if not already installed
+if ! command -v maturin &> /dev/null; then
+    echo "Installing maturin..."
+    # Use pip instead of cargo to avoid yanked dependency issues
+    # (e.g., maturin 1.12.2 depends on cargo-xwin which has yanked xwin versions)
+    pip install maturin
+    echo "maturin installation complete:"
+    maturin --version
+else
+    echo "maturin is already installed:"
+    maturin --version
+fi
+
+echo "Rust toolchain setup complete."

.evergreen/scripts/run_tests.py

@@ -151,6 +151,30 @@ def run() -> None:
     if os.environ.get("PYMONGOCRYPT_LIB"):
         handle_pymongocrypt()
 
+    # Check if Rust extension is being used
+    LOGGER.info(f"PYMONGO_USE_RUST={os.environ.get('PYMONGO_USE_RUST', 'not set')}")
+    LOGGER.info(f"PYMONGO_BUILD_RUST={os.environ.get('PYMONGO_BUILD_RUST', 'not set')}")
+
+    if os.environ.get("PYMONGO_USE_RUST") or os.environ.get("PYMONGO_BUILD_RUST"):
+        try:
+            import bson
+
+            impl = bson.get_bson_implementation()
+            has_rust = bson.has_rust()
+            has_c = bson.has_c()
+
+            LOGGER.info(f"BSON implementation in use: {impl}")
+            LOGGER.info(f"Has Rust: {has_rust}, Has C: {has_c}")
+
+            if impl == "rust":
+                LOGGER.info("✓ Rust extension is ACTIVE")
+            elif impl == "c":
+                LOGGER.info("✓ C extension is ACTIVE")
+            else:
+                LOGGER.info("✓ Pure Python implementation is ACTIVE")
+        except Exception as e:
+            LOGGER.warning(f"Could not check BSON implementation: {e}")
+
     LOGGER.info(f"Test setup:\n{AUTH=}\n{SSL=}\n{UV_ARGS=}\n{TEST_ARGS=}")
 
     # Record the start time for a perf test.

.evergreen/scripts/setup-dev-env.sh

@@ -22,6 +22,11 @@ bash $HERE/install-dependencies.sh
 # Handle the value for UV_PYTHON.
 . $HERE/setup-uv-python.sh
 
+# Show Rust toolchain status for debugging
+echo "Rust toolchain: $(rustc --version 2>/dev/null || echo 'not found')"
+echo "Cargo: $(cargo --version 2>/dev/null || echo 'not found')"
+echo "Maturin: $(maturin --version 2>/dev/null || echo 'not found')"
+
 # Only run the next part if not running on CI.
 if [ -z "${CI:-}" ]; then
   # Add the default install path to the path if needed.

.evergreen/scripts/setup-tests.sh

@@ -13,6 +13,8 @@ set -eu
 #  MONGODB_API_VERSION  The mongodb api version to use in tests.
 #  MONGODB_URI          If non-empty, use as the MONGODB_URI in tests.
 #  USE_ACTIVE_VENV      If non-empty, use the active virtual environment.
+#  PYMONGO_BUILD_RUST   If non-empty, build and test with Rust extension.
+#  PYMONGO_USE_RUST     If non-empty, use the Rust extension for tests.
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
 
@@ -21,6 +23,12 @@ if [ -f $SCRIPT_DIR/env.sh ]; then
   source $SCRIPT_DIR/env.sh
 fi
 
+# Install Rust toolchain if building Rust extension
+if [ -n "${PYMONGO_BUILD_RUST:-}" ]; then
+  echo "PYMONGO_BUILD_RUST is set, installing Rust toolchain..."
+  bash $SCRIPT_DIR/install-rust.sh
+fi
+
 echo "Setting up tests with args \"$*\"..."
 uv run ${USE_ACTIVE_VENV:+--active} "$SCRIPT_DIR/setup_tests.py" "$@"
 echo "Setting up tests with args \"$*\"... done."