Upgrade to DataFusion 53

Author: nuno-faria

Cargo.lock

@@ -48,15 +48,15 @@ tokio = { version = "1.49", features = [
   "rt-multi-thread",
   "sync",
 ] }
-pyo3 = { version = "0.26", features = [
+pyo3 = { version = "0.28", features = [
   "extension-module",
   "abi3",
   "abi3-py310",
 ] }
-pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
+pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
 pyo3-log = "0.13.3"
-arrow = { version = "57", features = ["pyarrow"] }
-arrow-select = { version = "57" }
+arrow = { version = "58", features = ["pyarrow"] }
+arrow-select = { version = "58" }
 datafusion = { version = "52", features = ["avro", "unicode_expressions"] }
 datafusion-substrait = { version = "52", optional = true }
 datafusion-proto = { version = "52" }
@@ -70,7 +70,7 @@ mimalloc = { version = "0.1", optional = true, default-features = false, feature
 async-trait = "0.1.89"
 futures = "0.3"
 cstr = "0.2"
-object_store = { version = "0.12.4", features = [
+object_store = { version = "0.13.1", features = [
   "aws",
   "gcp",
   "azure",
@@ -82,7 +82,7 @@ parking_lot = "0.12"
 
 [build-dependencies]
 prost-types = "0.14.3"     # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.26"
+pyo3-build-config = "0.28"
 
 [lib]
 name = "datafusion_python"
@@ -91,3 +91,10 @@ crate-type = ["cdylib", "rlib"]
 [profile.release]
 lto = true
 codegen-units = 1
+
+# TODO: remove when datafusion-53 is released
+[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion.git", rev = "6713439497561fa74a94177e5b8632322fb7cea5" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "6713439497561fa74a94177e5b8632322fb7cea5" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "6713439497561fa74a94177e5b8632322fb7cea5" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "6713439497561fa74a94177e5b8632322fb7cea5" }

Cargo.toml

@@ -146,7 +146,7 @@ PyO3 class mutability guidelines
 
 PyO3 bindings should present immutable wrappers whenever a struct stores shared or
 interior-mutable state. In practice this means that any ``#[pyclass]`` containing an
-``Arc<RwLock<_>>`` or similar synchronized primitive must opt into ``#[pyclass(frozen)]``
+``Arc<RwLock<_>>`` or similar synchronized primitive must opt into ``#[pyclass(from_py_object, frozen)]``
 unless there is a compelling reason not to.
 
 The :mod:`datafusion` configuration helpers illustrate the preferred pattern. The
@@ -156,7 +156,7 @@ instead of mutating the container directly:
 
 .. code-block:: rust
 
-    #[pyclass(name = "Config", module = "datafusion", subclass, frozen)]
+    #[pyclass(from_py_object, name = "Config", module = "datafusion", subclass, frozen)]
     #[derive(Clone)]
     pub(crate) struct PyConfig {
         config: Arc<RwLock<ConfigOptions>>,
@@ -170,7 +170,7 @@ existing instance in place:
 
 .. code-block:: rust
 
-    #[pyclass(frozen, name = "SessionContext", module = "datafusion", subclass)]
+    #[pyclass(from_py_object, frozen, name = "SessionContext", module = "datafusion", subclass)]
     #[derive(Clone)]
     pub struct PySessionContext {
         pub ctx: SessionContext,
@@ -186,7 +186,7 @@ field updates:
 
     // TODO: This looks like this needs pyo3 tracking so leaving unfrozen for now
     #[derive(Debug, Clone)]
-    #[pyclass(name = "DataTypeMap", module = "datafusion.common", subclass)]
+    #[pyclass(from_py_object, name = "DataTypeMap", module = "datafusion.common", subclass)]
     pub struct DataTypeMap {
         #[pyo3(get, set)]
         pub arrow_type: PyDataType,
@@ -232,8 +232,11 @@ can then be turned into a ``ForeignTableProvider`` the associated code is:
 
 .. code-block:: rust
 
-    let capsule = capsule.downcast::<PyCapsule>()?;
-    let provider = unsafe { capsule.reference::<FFI_TableProvider>() };
+    let capsule = capsule.cast::<PyCapsule>()?;
+    let data: NonNull<FFI_TableProvider> = capsule
+        .pointer_checked(Some(name))?
+        .cast();
+    let codec = unsafe { data.as_ref() };
 
 By convention the ``datafusion-python`` library expects a Python object that has a
 ``TableProvider`` PyCapsule to have this capsule accessible by calling a function named

docs/source/contributor-guide/ffi.rst

@@ -18,6 +18,24 @@
 Upgrade Guides
 ==============
 
+DataFusion 53.0.0
+-----------------
+
+This version includes an upgraded version of `pyo3`, which changed the way to extract an FFI object.
+Example:
+
+Before:
+.. code-block:: rust
+        let codec = unsafe { capsule.reference::<FFI_LogicalExtensionCodec>() };
+
+Now:
+.. code-block:: rust
+        let data: NonNull<FFI_LogicalExtensionCodec> = capsule
+            .pointer_checked(Some(c_str!("datafusion_logical_extension_codec")))?
+            .cast();
+        let codec = unsafe { data.as_ref() };
+
+
 DataFusion 52.0.0
 -----------------