From b6e40bbd4507068e66e10578f139b51549f650db Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 11:37:58 -0500 Subject: [PATCH 1/5] feat: add with_child convenience method to dyn Array Adds a method to replace a single child of an array by index, building on the existing with_children infrastructure. This is needed by the upcoming iterative execution scheduler which replaces children one at a time as they are executed. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/src/array/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index d5159edbc9f..1f57b177bca 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -352,6 +352,19 @@ impl dyn Array + '_ { pub fn is_canonical(&self) -> bool { self.is::() } + + /// Returns a new array with the child at `child_idx` replaced by `replacement`. + pub fn with_child(&self, child_idx: usize, replacement: ArrayRef) -> VortexResult { + let mut children: Vec = self.children(); + vortex_ensure!( + child_idx < children.len(), + "child index {} out of bounds for array with {} children", + child_idx, + children.len() + ); + children[child_idx] = replacement; + self.with_children(children) + } } /// Trait for converting a type into a Vortex [`ArrayRef`]. From 0516da20353913a8ff567fd1f3314c9742401180 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 11:39:53 -0500 Subject: [PATCH 2/5] feat: add ExecutionStep enum for iterative array execution Adds the ExecutionStep enum (ExecuteChild, ColumnarizeChild, Done) that encodings will return from VTable::execute instead of ArrayRef. This is infrastructure for the upcoming iterative execution scheduler. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/src/executor.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 58e128d253f..8dfe549dd1a 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -195,6 +195,33 @@ impl Executable for ArrayRef { } } +/// The result of a single execution step on an array encoding. +/// +/// Instead of recursively executing children, encodings return an `ExecutionStep` that tells the +/// scheduler what to do next. This enables the scheduler to manage execution iteratively using +/// an explicit work stack, run cross-step optimizations, and cache shared sub-expressions. +#[derive(Debug)] +pub enum ExecutionStep { + /// Request that the scheduler execute child at index `i` to columnar form, replace it in + /// this array, then re-enter execution on the updated array. + /// + /// Between steps, the scheduler runs reduce/reduce_parent rules to fixpoint, enabling + /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). + ExecuteChild(usize), + + /// Request that the scheduler execute child at index `i` to columnar form, replace it in + /// this array, then re-enter execution on the updated array. + /// + /// Unlike [`ExecuteChild`](Self::ExecuteChild), the scheduler does **not** run cross-step + /// optimizations when popping back up. Use this when the parent knows it will consume the + /// child directly (e.g., Dict taking from its values). + ColumnarizeChild(usize), + + /// Execution is complete. The result may be in any encoding — not necessarily canonical. + /// The scheduler will continue executing the result if it is not yet columnar. + Done(ArrayRef), +} + /// Extension trait for creating an execution context from a session. pub trait VortexSessionExecute { /// Create a new execution context from this session. From a675c0439435aa4d9a7a3885952e3731608d4445 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 12:27:38 -0500 Subject: [PATCH 3/5] feat: change VTable::execute to return ExecutionStep + iterative scheduler Changes the VTable::execute signature to return ExecutionStep instead of ArrayRef, and replaces the Executable for Columnar implementation with an iterative work-stack scheduler. The ExecutionStep enum has three variants: - ExecuteChild(i): ask the scheduler to execute child i to columnar - ColumnarizeChild(i): same but skip cross-step optimization - Done(result): execution complete The new scheduler in Executable for Columnar uses an explicit stack instead of recursion, and runs reduce/reduce_parent rules between steps via the existing optimizer infrastructure. All encoding implementations are mechanically wrapped in ExecutionStep::Done(...) to preserve existing behavior. Individual encodings will be migrated to use ExecuteChild/ColumnarizeChild in follow-up PRs. Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/alp/src/alp/array.rs | 7 +- encodings/alp/src/alp_rd/array.rs | 5 +- encodings/bytebool/src/array.rs | 7 +- encodings/datetime-parts/src/array.rs | 7 +- .../src/decimal_byte_parts/mod.rs | 5 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 5 +- encodings/fastlanes/src/delta/vtable/mod.rs | 7 +- encodings/fastlanes/src/for/vtable/mod.rs | 5 +- encodings/fastlanes/src/rle/vtable/mod.rs | 7 +- encodings/fsst/src/array.rs | 5 +- encodings/pco/src/array.rs | 5 +- encodings/runend/src/array.rs | 5 +- encodings/sequence/src/array.rs | 5 +- encodings/sparse/src/lib.rs | 5 +- encodings/zigzag/src/array.rs | 7 +- encodings/zstd/src/array.rs | 8 +- encodings/zstd/src/zstd_buffers.rs | 7 +- vortex-array/src/arrays/bool/vtable/mod.rs | 5 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 5 +- .../src/arrays/constant/vtable/mod.rs | 7 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 5 +- vortex-array/src/arrays/dict/vtable/mod.rs | 9 +- .../src/arrays/extension/vtable/mod.rs | 5 +- vortex-array/src/arrays/filter/vtable.rs | 9 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 5 +- vortex-array/src/arrays/list/vtable/mod.rs | 7 +- .../src/arrays/listview/vtable/mod.rs | 5 +- vortex-array/src/arrays/masked/vtable/mod.rs | 11 +- vortex-array/src/arrays/null/mod.rs | 5 +- .../src/arrays/primitive/vtable/mod.rs | 5 +- .../src/arrays/scalar_fn/vtable/mod.rs | 5 +- vortex-array/src/arrays/shared/vtable.rs | 7 +- vortex-array/src/arrays/slice/vtable.rs | 7 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 5 +- vortex-array/src/arrays/varbin/vtable/mod.rs | 7 +- .../src/arrays/varbinview/vtable/mod.rs | 5 +- vortex-array/src/columnar.rs | 110 +++++++++++++++--- vortex-array/src/executor.rs | 29 +++-- vortex-array/src/vtable/dyn_.rs | 47 ++++---- vortex-array/src/vtable/mod.rs | 27 ++--- vortex-python/src/arrays/py/vtable.rs | 3 +- 41 files changed, 297 insertions(+), 140 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 055bc8583db..da337d97f91 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -234,9 +235,11 @@ impl VTable for ALPVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // TODO(joe): take by value - Ok(execute_decompress(array.clone(), ctx)?.into_array()) + Ok(ExecutionStep::Done( + execute_decompress(array.clone(), ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 3f351fec007..3bd7cdc6812 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -11,6 +11,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -295,7 +296,7 @@ impl VTable for ALPRDVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let left_parts = array.left_parts().clone().execute::(ctx)?; let right_parts = array.right_parts().clone().execute::(ctx)?; @@ -334,7 +335,7 @@ impl VTable for ALPRDVTable { ) }; - Ok(decoded_array.into_array()) + Ok(ExecutionStep::Done(decoded_array.into_array())) } fn reduce_parent( diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index c7e21795a12..8c5c7c9fa31 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::arrays::BoolArray; @@ -182,10 +183,12 @@ impl VTable for ByteBoolVTable { crate::rules::RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { let boolean_buffer = BitBuffer::from(array.as_slice()); let validity = array.validity().clone(); - Ok(BoolArray::new(boolean_buffer, validity).into_array()) + Ok(ExecutionStep::Done( + BoolArray::new(boolean_buffer, validity).into_array(), + )) } fn execute_parent( diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 0290ec9d0b9..c9e292d5498 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -221,8 +222,10 @@ impl VTable for DateTimePartsVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(decode_to_temporal(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + decode_to_temporal(array, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 51a5e62c10e..9bfbd9c21de 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -13,6 +13,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -189,8 +190,8 @@ impl VTable for DecimalBytePartsVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - to_canonical_decimal(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + to_canonical_decimal(array, ctx).map(ExecutionStep::Done) } fn execute_parent( diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index d213fb9f1ed..facb2aa5a4f 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -353,8 +354,8 @@ impl VTable for BitPackedVTable { }) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(unpack_array(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(unpack_array(array, ctx)?.into_array())) } fn execute_parent( diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 96c0e0dd248..5bf67a561c5 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -189,8 +190,10 @@ impl VTable for DeltaVTable { DeltaArray::try_new(bases, deltas, metadata.0.offset as usize, len) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(delta_decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + delta_decompress(array, ctx)?.into_array(), + )) } } diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 37373456746..79740ca3061 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::buffer::BufferHandle; @@ -165,8 +166,8 @@ impl VTable for FoRVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(decompress(array, ctx)?.into_array())) } fn execute_parent( diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 0a682d3abf7..ab12a22294d 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -230,8 +231,10 @@ impl VTable for RLEVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(rle_decompress(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + rle_decompress(array, ctx)?.into_array(), + )) } } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 0831f25002a..066d1b9fbe3 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -17,6 +17,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -332,8 +333,8 @@ impl VTable for FSSTVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - canonicalize_fsst(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + canonicalize_fsst(array, ctx).map(ExecutionStep::Done) } fn execute_parent( diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index ba94651b360..eead34ee852 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -19,6 +19,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -262,8 +263,8 @@ impl VTable for PcoVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.decompress()?.into_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.decompress()?.into_array())) } fn reduce_parent( diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 65d2c57cb5d..ed2ba204bf4 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -200,8 +201,8 @@ impl VTable for RunEndVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - run_end_canonicalize(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + run_end_canonicalize(array, ctx).map(ExecutionStep::Done) } } diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 207cff5c5a6..f5681841f95 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -7,6 +7,7 @@ use num_traits::cast::FromPrimitive; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -363,7 +364,7 @@ impl VTable for SequenceVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { let prim = match_each_native_ptype!(array.ptype(), |P| { let base = array.base().cast::

()?; let multiplier = array.multiplier().cast::

()?; @@ -374,7 +375,7 @@ impl VTable for SequenceVTable { PrimitiveArray::new(values, array.dtype.nullability().into()) }); - Ok(prim.into_array()) + Ok(ExecutionStep::Done(prim.into_array())) } fn execute_parent( diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index df41d26a49f..b1cc667a33b 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -11,6 +11,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -229,8 +230,8 @@ impl VTable for SparseVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - execute_sparse(array, ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + execute_sparse(array, ctx).map(ExecutionStep::Done) } } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 38155892d24..b7229031151 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -9,6 +9,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::buffer::BufferHandle; @@ -148,8 +149,10 @@ impl VTable for ZigZagVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(zigzag_decode(array.encoded().clone().execute(ctx)?).into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + zigzag_decode(array.encoded().clone().execute(ctx)?).into_array(), + )) } fn reduce_parent( diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 5d0cd8f6503..f7263939e4d 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -12,6 +12,7 @@ use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ProstMetadata; @@ -271,8 +272,11 @@ impl VTable for ZstdVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array.decompress()?.execute(ctx) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + array + .decompress()? + .execute::(ctx) + .map(ExecutionStep::Done) } fn reduce_parent( diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 30c26bbf30a..ff733475a95 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; +use vortex_array::ExecutionStep; use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::buffer::BufferHandle; @@ -466,10 +467,12 @@ impl VTable for ZstdBuffersVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); let inner_array = array.decompress_and_build_inner(session)?; - inner_array.execute::(ctx) + inner_array + .execute::(ctx) + .map(ExecutionStep::Done) } } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index ae82d592c72..567d017c627 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -12,6 +12,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::BoolArray; @@ -183,8 +184,8 @@ impl VTable for BoolVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 073282c769a..e30eaf3cd50 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -15,6 +15,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::ToCanonical; @@ -239,8 +240,8 @@ impl VTable for ChunkedVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(_canonicalize(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(_canonicalize(array, ctx)?.into_array())) } fn reduce(array: &Self::Array) -> VortexResult> { diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 3e69fdfd218..9c7a0ea4f6d 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; @@ -165,7 +166,9 @@ impl VTable for ConstantVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(constant_canonicalize(array)?.into_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + constant_canonicalize(array)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 3dd71efc924..f7b8fb8dbdf 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::DecimalArray; @@ -205,8 +206,8 @@ impl VTable for DecimalVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 8c2c9197019..37d88aaf7ee 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -29,6 +29,7 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; @@ -190,9 +191,9 @@ impl VTable for DictVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(canonical) = execute_fast_path(array, ctx)? { - return Ok(canonical); + return Ok(ExecutionStep::Done(canonical)); } // TODO(joe): if the values are constant return a constant @@ -208,7 +209,9 @@ impl VTable for DictVTable { // TODO(ngates): if indices min is quite high, we could slice self and offset the indices // such that canonicalize does less work. - Ok(take_canonical(values, &codes, ctx)?.into_array()) + Ok(ExecutionStep::Done( + take_canonical(values, &codes, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 3fe7bbb4cd2..c0404859555 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -18,6 +18,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::extension::ExtensionArray; use crate::arrays::extension::compute::rules::PARENT_RULES; @@ -148,8 +149,8 @@ impl VTable for ExtensionVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 7deae7af8a0..fff8c90b992 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -27,6 +27,7 @@ use crate::arrays::filter::rules::RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; @@ -154,9 +155,9 @@ impl VTable for FilterVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(canonical) = execute_filter_fast_paths(array, ctx)? { - return Ok(canonical); + return Ok(ExecutionStep::Done(canonical)); } let Mask::Values(mask_values) = &array.mask else { unreachable!("`execute_filter_fast_paths` handles AllTrue and AllFalse") @@ -164,7 +165,9 @@ impl VTable for FilterVTable { // We rely on the optimization pass that runs prior to this execution for filter pushdown, // so now we can just execute the filter without worrying. - Ok(execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array()) + Ok(ExecutionStep::Done( + execute_filter(array.child.clone().execute(ctx)?, mask_values).into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index fd377e41d7f..324c20db74d 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::compute::rules::PARENT_RULES; @@ -217,7 +218,7 @@ impl VTable for FixedSizeListVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index d574757b008..2857736af2c 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayRef; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::ProstMetadata; @@ -210,8 +211,10 @@ impl VTable for ListVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(list_view_from_list(array.clone(), ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + list_view_from_list(array.clone(), ctx)?.into_array(), + )) } fn execute_parent( diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 5ee338c781f..e7844b183a0 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::ProstMetadata; use crate::SerializeMetadata; @@ -237,8 +238,8 @@ impl VTable for ListViewVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index faafac1d95c..c63a0687196 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -25,6 +25,7 @@ use crate::arrays::masked::mask_validity_canonical; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; @@ -160,15 +161,15 @@ impl VTable for MaskedVTable { MaskedArray::try_new(child, validity) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { let validity_mask = array.validity_mask()?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { - return Ok( + return Ok(ExecutionStep::Done( ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), - ); + )); } // NB: We intentionally do NOT have a fast path for `validity_mask.all_true()`. @@ -178,7 +179,9 @@ impl VTable for MaskedVTable { // `AllTrue` masks (no data copying), so there's no benefit. let child = array.child().clone().execute::(ctx)?; - Ok(mask_validity_canonical(child, &validity_mask, ctx)?.into_array()) + Ok(ExecutionStep::Done( + mask_validity_canonical(child, &validity_mask, ctx)?.into_array(), + )) } fn reduce_parent( diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 95770dbbac8..dd08cc712be 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -11,6 +11,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::null::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -130,8 +131,8 @@ impl VTable for NullVTable { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 2a6529613aa..6637a51ffb8 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::arrays::PrimitiveArray; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -197,8 +198,8 @@ impl VTable for PrimitiveVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 974923a23a1..e255f74ea11 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -30,6 +30,7 @@ use crate::arrays::scalar_fn::rules::RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::expr::Expression; use crate::matcher::Matcher; use crate::scalar_fn; @@ -194,10 +195,10 @@ impl VTable for ScalarFnVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { ctx.log(format_args!("scalar_fn({}): executing", array.scalar_fn)); let args = VecExecutionArgs::new(array.children.clone(), array.len); - array.scalar_fn.execute(&args, ctx) + array.scalar_fn.execute(&args, ctx).map(ExecutionStep::Done) } fn reduce(array: &Self::Array) -> VortexResult> { diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index d79ca3de4d6..9a7ca38b263 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -12,6 +12,7 @@ use crate::ArrayRef; use crate::Canonical; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::shared::SharedArray; use crate::buffer::BufferHandle; @@ -144,8 +145,10 @@ impl VTable for SharedVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - array.get_or_compute(|source| source.clone().execute::(ctx)) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + array + .get_or_compute(|source| source.clone().execute::(ctx)) + .map(ExecutionStep::Done) } } impl OperationsVTable for SharedVTable { diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index daa445e8b89..26cc932a8ea 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -26,6 +26,7 @@ use crate::arrays::slice::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; +use crate::executor::ExecutionStep; use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; @@ -154,7 +155,7 @@ impl VTable for SliceVTable { Ok(()) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { // Execute the child to get canonical form, then slice it let Some(canonical) = array.child.as_opt::() else { // If the child is not canonical, recurse. @@ -162,13 +163,15 @@ impl VTable for SliceVTable { .child .clone() .execute::(ctx)? - .slice(array.slice_range().clone()); + .slice(array.slice_range().clone()) + .map(ExecutionStep::Done); }; // TODO(ngates): we should inline canonical slice logic here. Canonical::from(canonical) .as_ref() .slice(array.range.clone()) + .map(ExecutionStep::Done) } fn reduce_parent( diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index c81286ef782..b8aa7d01a36 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -15,6 +15,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::arrays::struct_::StructArray; use crate::arrays::struct_::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; @@ -205,8 +206,8 @@ impl VTable for StructVTable { Ok(()) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } fn reduce_parent( diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 9b3d3a1ec71..369f72b9c10 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -10,6 +10,7 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::DeserializeMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::ProstMetadata; use crate::SerializeMetadata; @@ -218,8 +219,10 @@ impl VTable for VarBinVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(varbin_to_canonical(array, ctx)?.into_array()) + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done( + varbin_to_canonical(array, ctx)?.into_array(), + )) } } diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index bf52a8d6e47..95ae0422f22 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -17,6 +17,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::EmptyMetadata; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::Precision; use crate::arrays::BinaryView; use crate::arrays::varbinview::VarBinViewArray; @@ -241,7 +242,7 @@ impl VTable for VarBinViewVTable { PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { - Ok(array.to_array()) + fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + Ok(ExecutionStep::Done(array.to_array())) } } diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 09746a67032..3633028c205 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -4,6 +4,7 @@ use std::env::VarError; use std::sync::LazyLock; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_panic; @@ -15,11 +16,13 @@ use crate::Canonical; use crate::CanonicalView; use crate::Executable; use crate::ExecutionCtx; +use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::ConstantArray; use crate::arrays::ConstantVTable; use crate::dtype::DType; use crate::matcher::Matcher; +use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; /// Represents a columnnar array of data, either in canonical form or as a constant array. @@ -71,14 +74,20 @@ impl IntoArray for Columnar { } } -/// Executing into a [`Columnar`] is implemented by repeatedly executing the array until we -/// converge on either a constant or canonical. +/// Executing into a [`Columnar`] is implemented using an iterative scheduler with an explicit +/// work stack. +/// +/// The scheduler repeatedly: +/// 1. Checks if the current array is columnar (constant or canonical) — if so, pops the stack. +/// 2. Runs reduce/reduce_parent rules to fixpoint. +/// 3. Tries execute_parent on each child. +/// 4. Calls `execute` which returns an [`ExecutionStep`]. /// /// For safety, we will error when the number of execution iterations reaches 128. We may want this /// to be configurable in the future in case of highly complex array trees, but in practice we /// don't expect to ever reach this limit. impl Executable for Columnar { - fn execute(mut array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(root: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { static MAX_ITERATIONS: LazyLock = LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { Ok(val) => val.parse::().unwrap_or_else(|e| { @@ -90,26 +99,101 @@ impl Executable for Columnar { } }); + let mut current = root.optimize()?; + let mut stack: Vec<(ArrayRef, usize)> = Vec::new(); + for _ in 0..*MAX_ITERATIONS { - // Check for termination conditions - if let Some(constant) = array.as_opt::() { - ctx.log(format_args!("-> constant({})", constant.scalar())); - return Ok(Columnar::Constant(constant.clone())); + // Check for columnar termination (constant or canonical) + if let Some(columnar) = try_as_columnar(¤t) { + match stack.pop() { + None => { + // Stack empty — we're done + ctx.log(format_args!("-> columnar {}", current)); + return Ok(columnar); + } + Some((parent, child_idx)) => { + // Replace the child in the parent and continue + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } } - if let Some(canonical) = array.as_opt::() { - ctx.log(format_args!("-> canonical {}", array)); - return Ok(Columnar::Canonical(canonical.into())); + + // Try execute_parent (child-driven optimized execution) + if let Some(rewritten) = try_execute_parent(¤t, ctx)? { + ctx.log(format_args!( + "execute_parent rewrote {} -> {}", + current, rewritten + )); + current = rewritten.optimize()?; + continue; } - // Otherwise execute the array one step - array = array.execute(ctx)?; + // Execute the array itself + match current.vtable().execute(¤t, ctx)? { + ExecutionStep::ExecuteChild(i) => { + let child = current + .nth_child(i) + .vortex_expect("ExecuteChild index in bounds"); + ctx.log(format_args!( + "ExecuteChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i)); + current = child.optimize()?; + } + ExecutionStep::ColumnarizeChild(i) => { + let child = current + .nth_child(i) + .vortex_expect("ColumnarizeChild index in bounds"); + ctx.log(format_args!( + "ColumnarizeChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i)); + // No cross-step optimization for ColumnarizeChild + current = child; + } + ExecutionStep::Done(result) => { + ctx.log(format_args!("Done: {} -> {}", current, result)); + current = result; + } + } } - // If we reach here, we exceeded the maximum number of iterations, so error. vortex_bail!("Exceeded maximum execution iterations while executing to Columnar") } } +/// Try to interpret an array as columnar (constant or canonical). +fn try_as_columnar(array: &ArrayRef) -> Option { + if let Some(constant) = array.as_opt::() { + Some(Columnar::Constant(constant.clone())) + } else { + array + .as_opt::() + .map(|c| Columnar::Canonical(c.into())) + } +} + +/// Try execute_parent on each child of the array. +fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { + for child_idx in 0..array.nchildren() { + let child = array + .nth_child(child_idx) + .vortex_expect("checked nchildren"); + if let Some(result) = child + .vtable() + .execute_parent(&child, array, child_idx, ctx)? + { + result.statistics().inherit_from(array.statistics()); + return Ok(Some(result)); + } + } + Ok(None) +} + pub enum ColumnarView<'a> { Canonical(CanonicalView<'a>), Constant(&'a ConstantArray), diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 8dfe549dd1a..122d7e4b065 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -122,10 +122,10 @@ impl Drop for ExecutionCtx { /// /// The execution steps are as follows: /// 0. Check for canonical. -/// 1. Attempt to call `reduce_parent` on each child. -/// 2. Attempt to `reduce` the array with metadata-only optimizations. +/// 1. Attempt to `reduce` the array with metadata-only optimizations. +/// 2. Attempt to call `reduce_parent` on each child. /// 3. Attempt to call `execute_parent` on each child. -/// 4. Call `execute` on the array itself. +/// 4. Call `execute` on the array itself (which returns an [`ExecutionStep`]). /// /// Most users will not call this method directly, instead preferring to specify an executable /// target such as [`crate::Columnar`], [`Canonical`], or any of the canonical array types (such as @@ -182,16 +182,21 @@ impl Executable for ArrayRef { } } - // 4. execute (optimized execution) + // 4. execute (returns an ExecutionStep) ctx.log(format_args!("executing {}", array)); - let array = array - .vtable() - .execute(&array, ctx) - .map(|c| c.into_array())?; - array.statistics().inherit_from(array.statistics()); - ctx.log(format_args!("-> {}", array.as_ref())); - - Ok(array) + match array.vtable().execute(&array, ctx)? { + ExecutionStep::Done(result) => { + ctx.log(format_args!("-> {}", result.as_ref())); + Ok(result) + } + ExecutionStep::ExecuteChild(i) | ExecutionStep::ColumnarizeChild(i) => { + // For single-step execution, handle ExecuteChild by executing the child, + // replacing it, and returning the updated array. + let child = array.nth_child(i).vortex_expect("valid child index"); + let executed_child = child.execute::(ctx)?; + array.with_child(i, executed_child) + } + } } } diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 242bce3d532..37be712f07b 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -16,6 +16,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayAdapter; use crate::ArrayRef; +use crate::ExecutionStep; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; @@ -59,7 +60,7 @@ pub trait DynVTable: 'static + private::Sealed + Send + Sync + Debug { ) -> VortexResult>; /// See [`VTable::execute`] - fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; + fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; /// See [`VTable::execute_parent`] fn execute_parent( @@ -144,29 +145,31 @@ impl DynVTable for ArrayVTableAdapter { Ok(Some(reduced)) } - fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - let result = V::execute(downcast::(array), ctx)?; - - if cfg!(debug_assertions) { - vortex_ensure!( - result.as_ref().len() == array.len(), - "Result length mismatch for {:?}", - self - ); - vortex_ensure!( - result.as_ref().dtype() == array.dtype(), - "Executed canonical dtype mismatch for {:?}", - self - ); + fn execute(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let step = V::execute(downcast::(array), ctx)?; + + if let ExecutionStep::Done(ref result) = step { + if cfg!(debug_assertions) { + vortex_ensure!( + result.as_ref().len() == array.len(), + "Result length mismatch for {:?}", + self + ); + vortex_ensure!( + result.as_ref().dtype() == array.dtype(), + "Executed canonical dtype mismatch for {:?}", + self + ); + } + + // TODO(ngates): do we want to do this on every execution? We used to in to_canonical. + result + .as_ref() + .statistics() + .inherit_from(array.statistics()); } - // TODO(ngates): do we want to do this on every execution? We used to in to_canonical. - result - .as_ref() - .statistics() - .inherit_from(array.statistics()); - - Ok(result) + Ok(step) } fn execute_parent( diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 08d6868bb4e..2a600f07015 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -22,6 +22,7 @@ use vortex_session::VortexSession; use crate::Array; use crate::ArrayRef; use crate::Canonical; +use crate::ExecutionStep; use crate::IntoArray; use crate::Precision; use crate::arrays::ConstantArray; @@ -180,30 +181,24 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// of children must be expected. fn with_children(array: &mut Self::Array, children: Vec) -> VortexResult<()>; - /// Execute this array to produce an [`ArrayRef`]. + /// Execute this array by returning an [`ExecutionStep`] that tells the scheduler what to + /// do next. + /// + /// Instead of recursively executing children, implementations should return + /// [`ExecutionStep::ExecuteChild(i)`] or [`ExecutionStep::ColumnarizeChild(i)`] to request + /// that the scheduler execute a child first, or [`ExecutionStep::Done(result)`] when the + /// encoding can produce a result directly. /// /// Array execution is designed such that repeated execution of an array will eventually /// converge to a canonical representation. Implementations of this function should therefore /// ensure they make progress towards that goal. /// - /// This includes fully evaluating the array, such us decoding run-end encoding, or executing - /// one of the array's children and re-building the array with the executed child. - /// - /// It is recommended to only perform a single step of execution per call to this function, - /// such that surrounding arrays have an opportunity to perform their own parent reduction - /// or execution logic. - /// - /// The returned array must be logically equivalent to the input array. In other words, the - /// recursively canonicalized forms of both arrays must be equal. + /// The returned array (in `Done`) must be logically equivalent to the input array. In other + /// words, the recursively canonicalized forms of both arrays must be equal. /// /// Debug builds will panic if the returned array is of the wrong type, wrong length, or /// incorrectly contains null values. - /// - // TODO(ngates): in the future, we may pass a "target encoding hint" such that this array - // can produce a more optimal representation for the parent. This could be used to preserve - // varbin vs varbinview or list vs listview encodings when the parent knows it prefers - // one representation over another, such as when exporting to a specific Arrow array. - fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult; + fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult; /// Attempt to execute the parent of this array. /// diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index 83ebfa226cf..be3b2ad26a3 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -10,6 +10,7 @@ use pyo3::prelude::*; use pyo3::types::PyBytes; use vortex::array::ArrayRef; use vortex::array::ExecutionCtx; +use vortex::array::ExecutionStep; use vortex::array::Precision; use vortex::array::RawMetadata; use vortex::array::SerializeMetadata; @@ -155,7 +156,7 @@ impl VTable for PythonVTable { Ok(()) } - fn execute(_array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(_array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult { todo!() } } From 2b9697c6d665fcbef5392dfe4616116f8a99c1a3 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 13:59:21 -0500 Subject: [PATCH 4/5] chore: regenerate public-api.lock files Signed-off-by: Nicholas Gates Co-Authored-By: Claude Opus 4.6 --- encodings/alp/public-api.lock | 4 +- encodings/bytebool/public-api.lock | 2 +- encodings/datetime-parts/public-api.lock | 2 +- encodings/decimal-byte-parts/public-api.lock | 2 +- encodings/fastlanes/public-api.lock | 8 +- encodings/fsst/public-api.lock | 2 +- encodings/pco/public-api.lock | 2 +- encodings/runend/public-api.lock | 2 +- encodings/sequence/public-api.lock | 2 +- encodings/sparse/public-api.lock | 2 +- encodings/zigzag/public-api.lock | 2 +- encodings/zstd/public-api.lock | 2 +- vortex-array/public-api.lock | 96 +++++++++++--------- vortex-array/src/columnar.rs | 12 --- vortex-array/src/executor.rs | 10 +- vortex-array/src/vtable/mod.rs | 4 +- 16 files changed, 72 insertions(+), 82 deletions(-) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index f2a1ae9089d..fe227544ded 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_alp::ALPRDVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_alp::ALPRDVTable::dtype(array: &vortex_alp::ALPRDArray) -> &vortex_array::dtype::DType -pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::ALPRDVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -300,7 +300,7 @@ pub fn vortex_alp::ALPVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::d pub fn vortex_alp::ALPVTable::dtype(array: &vortex_alp::ALPArray) -> &vortex_array::dtype::DType -pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::ALPVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 0a1a01e7ae2..47108e9378a 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -104,7 +104,7 @@ pub fn vortex_bytebool::ByteBoolVTable::deserialize(_bytes: &[u8], _dtype: &vort pub fn vortex_bytebool::ByteBoolVTable::dtype(array: &vortex_bytebool::ByteBoolArray) -> &vortex_array::dtype::DType -pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_bytebool::ByteBoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 04626e436e8..14d850ade0a 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -178,7 +178,7 @@ pub fn vortex_datetime_parts::DateTimePartsVTable::deserialize(bytes: &[u8], _dt pub fn vortex_datetime_parts::DateTimePartsVTable::dtype(array: &vortex_datetime_parts::DateTimePartsArray) -> &vortex_array::dtype::DType -pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 310a47c8c9a..41a2a23a086 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -108,7 +108,7 @@ pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::deserialize(bytes: &[u pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &vortex_array::dtype::DType -pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index a6797c89530..80b77f6bdaa 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -238,7 +238,7 @@ pub fn vortex_fastlanes::BitPackedVTable::deserialize(bytes: &[u8], _dtype: &vor pub fn vortex_fastlanes::BitPackedVTable::dtype(array: &vortex_fastlanes::BitPackedArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPackedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -364,7 +364,7 @@ pub fn vortex_fastlanes::DeltaVTable::deserialize(bytes: &[u8], _dtype: &vortex_ pub fn vortex_fastlanes::DeltaVTable::dtype(array: &vortex_fastlanes::DeltaArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::DeltaVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId @@ -498,7 +498,7 @@ pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_arr pub fn vortex_fastlanes::FoRVTable::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoRVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> @@ -630,7 +630,7 @@ pub fn vortex_fastlanes::RLEVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_fastlanes::RLEVTable::dtype(array: &vortex_fastlanes::RLEArray) -> &vortex_array::dtype::DType -pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::RLEVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 103abc8a38c..57f1098d357 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -142,7 +142,7 @@ pub fn vortex_fsst::FSSTVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_fsst::FSSTVTable::dtype(array: &vortex_fsst::FSSTArray) -> &vortex_array::dtype::DType -pub fn vortex_fsst::FSSTVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fsst::FSSTVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 2cbb6087556..f97ab79123b 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -156,7 +156,7 @@ pub fn vortex_pco::PcoVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::d pub fn vortex_pco::PcoVTable::dtype(array: &vortex_pco::PcoArray) -> &vortex_array::dtype::DType -pub fn vortex_pco::PcoVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_pco::PcoVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::PcoVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index b6dce876336..4318cc7255e 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -176,7 +176,7 @@ pub fn vortex_runend::RunEndVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_runend::RunEndVTable::dtype(array: &vortex_runend::RunEndArray) -> &vortex_array::dtype::DType -pub fn vortex_runend::RunEndVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_runend::RunEndVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::RunEndVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 6d4095649d0..520418d2a3a 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -128,7 +128,7 @@ pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], _dtype: &vorte pub fn vortex_sequence::SequenceVTable::dtype(array: &vortex_sequence::SequenceArray) -> &vortex_array::dtype::DType -pub fn vortex_sequence::SequenceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_sequence::SequenceVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sequence::SequenceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index f4456984bb1..24a0fb1e6c1 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -120,7 +120,7 @@ pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar pub fn vortex_sparse::SparseVTable::dtype(array: &vortex_sparse::SparseArray) -> &vortex_array::dtype::DType -pub fn vortex_sparse::SparseVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_sparse::SparseVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sparse::SparseVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index 60dc4b073f9..1f9fca6c735 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -96,7 +96,7 @@ pub fn vortex_zigzag::ZigZagVTable::deserialize(_bytes: &[u8], _dtype: &vortex_a pub fn vortex_zigzag::ZigZagVTable::dtype(array: &vortex_zigzag::ZigZagArray) -> &vortex_array::dtype::DType -pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZagVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zigzag::ZigZagVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 04d8faf7109..9f351777a4c 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -196,7 +196,7 @@ pub fn vortex_zstd::ZstdVTable::deserialize(bytes: &[u8], _dtype: &vortex_array: pub fn vortex_zstd::ZstdVTable::dtype(array: &vortex_zstd::ZstdArray) -> &vortex_array::dtype::DType -pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 5cbd6cfbfed..0942e30fa63 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -212,7 +212,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -520,7 +520,7 @@ pub fn vortex_array::arrays::BoolVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::BoolVTable::dtype(array: &vortex_array::arrays::BoolArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::BoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -690,7 +690,7 @@ pub fn vortex_array::arrays::ChunkedVTable::deserialize(_bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ChunkedVTable::dtype(array: &vortex_array::arrays::ChunkedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ChunkedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -838,7 +838,7 @@ pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ConstantVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1056,7 +1056,7 @@ pub fn vortex_array::arrays::DecimalVTable::deserialize(bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::DecimalVTable::dtype(array: &vortex_array::arrays::DecimalArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DecimalVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1262,7 +1262,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1450,7 +1450,7 @@ pub fn vortex_array::arrays::ExtensionVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ExtensionVTable::dtype(array: &vortex_array::arrays::ExtensionArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ExtensionVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1600,7 +1600,7 @@ pub fn vortex_array::arrays::FilterVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::FilterVTable::dtype(array: &vortex_array::arrays::FilterArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FilterVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1756,7 +1756,7 @@ pub fn vortex_array::arrays::FixedSizeListVTable::deserialize(_bytes: &[u8], _dt pub fn vortex_array::arrays::FixedSizeListVTable::dtype(array: &vortex_array::arrays::FixedSizeListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -1954,7 +1954,7 @@ pub fn vortex_array::arrays::ListVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::ListVTable::dtype(array: &vortex_array::arrays::ListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2134,7 +2134,7 @@ pub fn vortex_array::arrays::ListViewVTable::deserialize(bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ListViewVTable::dtype(array: &vortex_array::arrays::ListViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2256,7 +2256,7 @@ pub fn vortex_array::arrays::MaskedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::MaskedVTable::dtype(array: &vortex_array::arrays::MaskedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::MaskedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2454,7 +2454,7 @@ pub fn vortex_array::arrays::NullVTable::deserialize(_bytes: &[u8], _dtype: &vor pub fn vortex_array::arrays::NullVTable::dtype(_array: &vortex_array::arrays::NullArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::NullVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2708,7 +2708,7 @@ pub fn vortex_array::arrays::PrimitiveVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::PrimitiveVTable::dtype(array: &vortex_array::arrays::PrimitiveArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::PrimitiveVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2870,7 +2870,7 @@ pub fn vortex_array::arrays::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ScalarFnVTable::dtype(array: &vortex_array::arrays::ScalarFnArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ScalarFnVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2978,7 +2978,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3138,7 +3138,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3326,7 +3326,7 @@ pub fn vortex_array::arrays::StructVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::StructVTable::dtype(array: &vortex_array::arrays::StructArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::StructVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3652,7 +3652,7 @@ pub fn vortex_array::arrays::VarBinVTable::deserialize(bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::VarBinVTable::dtype(array: &vortex_array::arrays::VarBinArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -3870,7 +3870,7 @@ pub fn vortex_array::arrays::VarBinViewVTable::deserialize(_bytes: &[u8], _dtype pub fn vortex_array::arrays::VarBinViewVTable::dtype(array: &vortex_array::arrays::VarBinViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16414,7 +16414,7 @@ pub trait vortex_array::vtable::DynVTable: 'static + vortex_array::vtable::dyn_: pub fn vortex_array::vtable::DynVTable::build(&self, id: vortex_array::vtable::ArrayId, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::vtable::DynVTable::execute(&self, array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::vtable::DynVTable::execute(&self, array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::DynVTable::execute_parent(&self, array: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16538,7 +16538,7 @@ pub fn vortex_array::vtable::VTable::deserialize(bytes: &[u8], _dtype: &vortex_a pub fn vortex_array::vtable::VTable::dtype(array: &Self::Array) -> &vortex_array::dtype::DType -pub fn vortex_array::vtable::VTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::vtable::VTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16592,7 +16592,7 @@ pub fn vortex_array::arrays::BoolVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::BoolVTable::dtype(array: &vortex_array::arrays::BoolArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::BoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::BoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16646,7 +16646,7 @@ pub fn vortex_array::arrays::ChunkedVTable::deserialize(_bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ChunkedVTable::dtype(array: &vortex_array::arrays::ChunkedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ChunkedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ChunkedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16700,7 +16700,7 @@ pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ConstantVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ConstantVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16754,7 +16754,7 @@ pub fn vortex_array::arrays::DecimalVTable::deserialize(bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::DecimalVTable::dtype(array: &vortex_array::arrays::DecimalArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DecimalVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DecimalVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16808,7 +16808,7 @@ pub fn vortex_array::arrays::DictVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::DictVTable::dtype(array: &vortex_array::arrays::DictArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::DictVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::DictVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16862,7 +16862,7 @@ pub fn vortex_array::arrays::ExtensionVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ExtensionVTable::dtype(array: &vortex_array::arrays::ExtensionArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ExtensionVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ExtensionVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16916,7 +16916,7 @@ pub fn vortex_array::arrays::FilterVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::FilterVTable::dtype(array: &vortex_array::arrays::FilterArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FilterVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FilterVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -16970,7 +16970,7 @@ pub fn vortex_array::arrays::FixedSizeListVTable::deserialize(_bytes: &[u8], _dt pub fn vortex_array::arrays::FixedSizeListVTable::dtype(array: &vortex_array::arrays::FixedSizeListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeListVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17024,7 +17024,7 @@ pub fn vortex_array::arrays::ListVTable::deserialize(bytes: &[u8], _dtype: &vort pub fn vortex_array::arrays::ListVTable::dtype(array: &vortex_array::arrays::ListArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17078,7 +17078,7 @@ pub fn vortex_array::arrays::ListViewVTable::deserialize(bytes: &[u8], _dtype: & pub fn vortex_array::arrays::ListViewVTable::dtype(array: &vortex_array::arrays::ListViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17132,7 +17132,7 @@ pub fn vortex_array::arrays::MaskedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::MaskedVTable::dtype(array: &vortex_array::arrays::MaskedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::MaskedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::MaskedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17186,7 +17186,7 @@ pub fn vortex_array::arrays::NullVTable::deserialize(_bytes: &[u8], _dtype: &vor pub fn vortex_array::arrays::NullVTable::dtype(_array: &vortex_array::arrays::NullArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::NullVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::NullVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17240,7 +17240,7 @@ pub fn vortex_array::arrays::PrimitiveVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::PrimitiveVTable::dtype(array: &vortex_array::arrays::PrimitiveArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::PrimitiveVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::PrimitiveVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17294,7 +17294,7 @@ pub fn vortex_array::arrays::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: pub fn vortex_array::arrays::ScalarFnVTable::dtype(array: &vortex_array::arrays::ScalarFnArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ScalarFnVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ScalarFnVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17348,7 +17348,7 @@ pub fn vortex_array::arrays::SharedVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::SharedVTable::dtype(array: &vortex_array::arrays::SharedArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SharedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SharedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17402,7 +17402,7 @@ pub fn vortex_array::arrays::SliceVTable::deserialize(_bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::SliceVTable::dtype(array: &vortex_array::arrays::SliceArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::SliceVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::SliceVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17456,7 +17456,7 @@ pub fn vortex_array::arrays::StructVTable::deserialize(_bytes: &[u8], _dtype: &v pub fn vortex_array::arrays::StructVTable::dtype(array: &vortex_array::arrays::StructArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::StructVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::StructVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17510,7 +17510,7 @@ pub fn vortex_array::arrays::VarBinVTable::deserialize(bytes: &[u8], _dtype: &vo pub fn vortex_array::arrays::VarBinVTable::dtype(array: &vortex_array::arrays::VarBinArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinVTable::execute(array: &Self::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17564,7 +17564,7 @@ pub fn vortex_array::arrays::VarBinViewVTable::deserialize(_bytes: &[u8], _dtype pub fn vortex_array::arrays::VarBinViewVTable::dtype(array: &vortex_array::arrays::VarBinViewArray) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinViewVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinViewVTable::execute_parent(array: &Self::Array, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -17896,7 +17896,7 @@ pub fn vortex_array::Columnar::len(&self) -> usize impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::IntoArray for vortex_array::Columnar @@ -17912,6 +17912,16 @@ impl<'a> core::convert::AsRef for vortex_array::Columna pub fn vortex_array::ColumnarView<'a>::as_ref(&self) -> &dyn vortex_array::Array +pub enum vortex_array::ExecutionStep + +pub vortex_array::ExecutionStep::Done(vortex_array::ArrayRef) + +pub vortex_array::ExecutionStep::ExecuteChild(usize) + +impl core::fmt::Debug for vortex_array::ExecutionStep + +pub fn vortex_array::ExecutionStep::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub enum vortex_array::Precision pub vortex_array::Precision::Ptr @@ -18562,7 +18572,7 @@ pub fn vortex_array::CanonicalValidity::execute(array: vortex_array::ArrayRef, c impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::Executable for vortex_array::RecursiveCanonical diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 3633028c205..7cb60b5f5f5 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -143,18 +143,6 @@ impl Executable for Columnar { stack.push((current, i)); current = child.optimize()?; } - ExecutionStep::ColumnarizeChild(i) => { - let child = current - .nth_child(i) - .vortex_expect("ColumnarizeChild index in bounds"); - ctx.log(format_args!( - "ColumnarizeChild({i}): pushing {}, focusing on {}", - current, child - )); - stack.push((current, i)); - // No cross-step optimization for ColumnarizeChild - current = child; - } ExecutionStep::Done(result) => { ctx.log(format_args!("Done: {} -> {}", current, result)); current = result; diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 122d7e4b065..f17780b31e2 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -189,7 +189,7 @@ impl Executable for ArrayRef { ctx.log(format_args!("-> {}", result.as_ref())); Ok(result) } - ExecutionStep::ExecuteChild(i) | ExecutionStep::ColumnarizeChild(i) => { + ExecutionStep::ExecuteChild(i) => { // For single-step execution, handle ExecuteChild by executing the child, // replacing it, and returning the updated array. let child = array.nth_child(i).vortex_expect("valid child index"); @@ -214,14 +214,6 @@ pub enum ExecutionStep { /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). ExecuteChild(usize), - /// Request that the scheduler execute child at index `i` to columnar form, replace it in - /// this array, then re-enter execution on the updated array. - /// - /// Unlike [`ExecuteChild`](Self::ExecuteChild), the scheduler does **not** run cross-step - /// optimizations when popping back up. Use this when the parent knows it will consume the - /// child directly (e.g., Dict taking from its values). - ColumnarizeChild(usize), - /// Execution is complete. The result may be in any encoding — not necessarily canonical. /// The scheduler will continue executing the result if it is not yet columnar. Done(ArrayRef), diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index 2a600f07015..35184f39b25 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -185,8 +185,8 @@ pub trait VTable: 'static + Sized + Send + Sync + Debug { /// do next. /// /// Instead of recursively executing children, implementations should return - /// [`ExecutionStep::ExecuteChild(i)`] or [`ExecutionStep::ColumnarizeChild(i)`] to request - /// that the scheduler execute a child first, or [`ExecutionStep::Done(result)`] when the + /// [`ExecutionStep::ExecuteChild(i)`] to request that the scheduler execute a child first, + /// or [`ExecutionStep::Done(result)`] when the /// encoding can produce a result directly. /// /// Array execution is designed such that repeated execution of an array will eventually From b5b165268c651e9113abd5b0256a0a828b66cc46 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Mon, 2 Mar 2026 15:06:11 -0500 Subject: [PATCH 5/5] feat: extract execute_until scheduler with per-child done predicates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the iterative execution scheduler into a general-purpose `execute_until` method on `dyn Array`. The scheduler terminates when the root array matches `M`, while each child can specify its own termination condition via a `DonePredicate` carried in `ExecutionStep::ExecuteChild`. `ExecutionStep` now provides constructor methods: - `execute_child::(idx)` — request child execution until M matches - `done(result)` — signal completion Both `Executable for Columnar` and `Executable for Canonical` are simplified to thin wrappers over `execute_until` with `AnyColumnar` and `AnyCanonical` matchers respectively. Signed-off-by: Nick Gates Co-Authored-By: Claude Opus 4.6 --- vortex-array/public-api.lock | 14 +- vortex-array/src/arrays/constant/mod.rs | 1 - vortex-array/src/canonical.rs | 30 ++--- vortex-array/src/columnar.rs | 122 ++--------------- vortex-array/src/executor.rs | 167 +++++++++++++++++++++++- 5 files changed, 193 insertions(+), 141 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 0942e30fa63..b885c08284c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -17896,7 +17896,7 @@ pub fn vortex_array::Columnar::len(&self) -> usize impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::IntoArray for vortex_array::Columnar @@ -17916,7 +17916,13 @@ pub enum vortex_array::ExecutionStep pub vortex_array::ExecutionStep::Done(vortex_array::ArrayRef) -pub vortex_array::ExecutionStep::ExecuteChild(usize) +pub vortex_array::ExecutionStep::ExecuteChild(usize, vortex_array::DonePredicate) + +impl vortex_array::ExecutionStep + +pub fn vortex_array::ExecutionStep::done(result: vortex_array::ArrayRef) -> Self + +pub fn vortex_array::ExecutionStep::execute_child(child_idx: usize) -> Self impl core::fmt::Debug for vortex_array::ExecutionStep @@ -18572,7 +18578,7 @@ pub fn vortex_array::CanonicalValidity::execute(array: vortex_array::ArrayRef, c impl vortex_array::Executable for vortex_array::Columnar -pub fn vortex_array::Columnar::execute(root: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::Columnar::execute(array: vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::Executable for vortex_array::RecursiveCanonical @@ -18829,3 +18835,5 @@ pub fn vortex_session::VortexSession::create_execution_ctx(&self) -> vortex_arra pub type vortex_array::ArrayContext = vortex_session::registry::Context<&'static dyn vortex_array::vtable::DynVTable> pub type vortex_array::ArrayRef = alloc::sync::Arc + +pub type vortex_array::DonePredicate = fn(&dyn vortex_array::Array) -> bool diff --git a/vortex-array/src/arrays/constant/mod.rs b/vortex-array/src/arrays/constant/mod.rs index bb5bb519401..6aa94ab8565 100644 --- a/vortex-array/src/arrays/constant/mod.rs +++ b/vortex-array/src/arrays/constant/mod.rs @@ -8,7 +8,6 @@ pub use arbitrary::ArbitraryConstantArray; mod array; pub use array::ConstantArray; -pub(crate) use vtable::canonical::constant_canonicalize; pub(crate) mod compute; diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index bd90965cf43..21f83511f10 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -14,7 +14,6 @@ use vortex_error::vortex_panic; use crate::Array; use crate::ArrayRef; -use crate::Columnar; use crate::Executable; use crate::ExecutionCtx; use crate::IntoArray; @@ -43,7 +42,6 @@ use crate::arrays::StructVTable; use crate::arrays::VarBinViewArray; use crate::arrays::VarBinViewArrayParts; use crate::arrays::VarBinViewVTable; -use crate::arrays::constant_canonicalize; use crate::builders::builder_with_capacity; use crate::dtype::DType; use crate::dtype::NativePType; @@ -439,28 +437,18 @@ impl From for ArrayRef { } } -/// Recursively execute the array until it reaches canonical form. +/// Execute into [`Canonical`] by running `execute_until` with the [`AnyCanonical`] matcher. /// -/// Callers should prefer to execute into `Columnar` if they are able to optimize their use for -/// constant arrays. +/// Unlike executing into [`crate::Columnar`], this will fully expand constant arrays into their +/// canonical form. Callers should prefer to execute into `Columnar` if they are able to optimize +/// their use for constant arrays. impl Executable for Canonical { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - if let Some(canonical) = array.as_opt::() { - return Ok(canonical.into()); - } - - // Invoke execute directly to avoid logging the call in the execution context. - Ok(match Columnar::execute(array.clone(), ctx)? { - Columnar::Canonical(c) => c, - Columnar::Constant(s) => { - let canonical = constant_canonicalize(&s)?; - canonical - .as_ref() - .statistics() - .inherit_from(array.statistics()); - canonical - } - }) + let result = array.execute_until::(ctx)?; + Ok(result + .as_opt::() + .map(Canonical::from) + .vortex_expect("execute_until:: must return a canonical array")) } } diff --git a/vortex-array/src/columnar.rs b/vortex-array/src/columnar.rs index 7cb60b5f5f5..8d272f8a033 100644 --- a/vortex-array/src/columnar.rs +++ b/vortex-array/src/columnar.rs @@ -1,13 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::env::VarError; -use std::sync::LazyLock; - use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_panic; use crate::AnyCanonical; use crate::Array; @@ -16,13 +11,11 @@ use crate::Canonical; use crate::CanonicalView; use crate::Executable; use crate::ExecutionCtx; -use crate::ExecutionStep; use crate::IntoArray; use crate::arrays::ConstantArray; use crate::arrays::ConstantVTable; use crate::dtype::DType; use crate::matcher::Matcher; -use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; /// Represents a columnnar array of data, either in canonical form or as a constant array. @@ -74,112 +67,21 @@ impl IntoArray for Columnar { } } -/// Executing into a [`Columnar`] is implemented using an iterative scheduler with an explicit -/// work stack. -/// -/// The scheduler repeatedly: -/// 1. Checks if the current array is columnar (constant or canonical) — if so, pops the stack. -/// 2. Runs reduce/reduce_parent rules to fixpoint. -/// 3. Tries execute_parent on each child. -/// 4. Calls `execute` which returns an [`ExecutionStep`]. -/// -/// For safety, we will error when the number of execution iterations reaches 128. We may want this -/// to be configurable in the future in case of highly complex array trees, but in practice we -/// don't expect to ever reach this limit. +/// Execute into [`Columnar`] by running `execute_until` with the [`AnyColumnar`] matcher. impl Executable for Columnar { - fn execute(root: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - static MAX_ITERATIONS: LazyLock = - LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { - Ok(val) => val.parse::().unwrap_or_else(|e| { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") - }), - Err(VarError::NotPresent) => 128, - Err(VarError::NotUnicode(_)) => { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") - } - }); - - let mut current = root.optimize()?; - let mut stack: Vec<(ArrayRef, usize)> = Vec::new(); - - for _ in 0..*MAX_ITERATIONS { - // Check for columnar termination (constant or canonical) - if let Some(columnar) = try_as_columnar(¤t) { - match stack.pop() { - None => { - // Stack empty — we're done - ctx.log(format_args!("-> columnar {}", current)); - return Ok(columnar); - } - Some((parent, child_idx)) => { - // Replace the child in the parent and continue - current = parent.with_child(child_idx, current)?; - current = current.optimize()?; - continue; - } - } - } - - // Try execute_parent (child-driven optimized execution) - if let Some(rewritten) = try_execute_parent(¤t, ctx)? { - ctx.log(format_args!( - "execute_parent rewrote {} -> {}", - current, rewritten - )); - current = rewritten.optimize()?; - continue; - } - - // Execute the array itself - match current.vtable().execute(¤t, ctx)? { - ExecutionStep::ExecuteChild(i) => { - let child = current - .nth_child(i) - .vortex_expect("ExecuteChild index in bounds"); - ctx.log(format_args!( - "ExecuteChild({i}): pushing {}, focusing on {}", - current, child - )); - stack.push((current, i)); - current = child.optimize()?; - } - ExecutionStep::Done(result) => { - ctx.log(format_args!("Done: {} -> {}", current, result)); - current = result; - } - } - } - - vortex_bail!("Exceeded maximum execution iterations while executing to Columnar") - } -} - -/// Try to interpret an array as columnar (constant or canonical). -fn try_as_columnar(array: &ArrayRef) -> Option { - if let Some(constant) = array.as_opt::() { - Some(Columnar::Constant(constant.clone())) - } else { - array - .as_opt::() - .map(|c| Columnar::Canonical(c.into())) - } -} - -/// Try execute_parent on each child of the array. -fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { - for child_idx in 0..array.nchildren() { - let child = array - .nth_child(child_idx) - .vortex_expect("checked nchildren"); - if let Some(result) = child - .vtable() - .execute_parent(&child, array, child_idx, ctx)? - { - result.statistics().inherit_from(array.statistics()); - return Ok(Some(result)); + fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let result = array.execute_until::(ctx)?; + if let Some(constant) = result.as_opt::() { + Ok(Columnar::Constant(constant.clone())) + } else { + Ok(Columnar::Canonical( + result + .as_opt::() + .map(Canonical::from) + .vortex_expect("execute_until:: must return a columnar array"), + )) } } - Ok(None) } pub enum ColumnarView<'a> { diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index f17780b31e2..0f5516f5785 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -1,13 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::env::VarError; use std::fmt; use std::fmt::Display; use std::sync::Arc; +use std::sync::LazyLock; use std::sync::atomic::AtomicUsize; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::AnyCanonical; @@ -15,6 +19,8 @@ use crate::Array; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::matcher::Matcher; +use crate::optimizer::ArrayOptimizer; /// Marker trait for types that an [`ArrayRef`] can be executed into. /// @@ -43,6 +49,110 @@ impl dyn Array + '_ { ) -> VortexResult { E::execute(self, ctx) } + + /// Iteratively execute this array until the [`Matcher`] matches, using an explicit work + /// stack. + /// + /// The scheduler repeatedly: + /// 1. Checks if the current array matches `M` — if so, pops the stack or returns. + /// 2. Runs `execute_parent` on each child for child-driven optimizations. + /// 3. Calls `execute` which returns an [`ExecutionStep`]. + /// + /// Note: the returned array may not match `M`. If execution converges to a canonical form + /// that does not match `M`, the canonical array is returned since no further execution + /// progress is possible. + /// + /// For safety, we will error when the number of execution iterations reaches a configurable + /// maximum (default 128, override with `VORTEX_MAX_ITERATIONS`). + pub fn execute_until( + self: Arc, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + static MAX_ITERATIONS: LazyLock = + LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { + Ok(val) => val.parse::().unwrap_or_else(|e| { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") + }), + Err(VarError::NotPresent) => 128, + Err(VarError::NotUnicode(_)) => { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") + } + }); + + let mut current = self.optimize()?; + // Stack frames: (parent, child_idx, done_predicate_for_child) + let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); + + for _ in 0..*MAX_ITERATIONS { + // Check for termination: use the stack frame's done predicate, or the root matcher. + let is_done = stack + .last() + .map_or(M::matches as DonePredicate, |frame| frame.2); + if is_done(current.as_ref()) { + match stack.pop() { + None => { + ctx.log(format_args!("-> {}", current)); + return Ok(current); + } + Some((parent, child_idx, _)) => { + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } + } + + // If we've reached canonical form, we can't execute any further regardless + // of whether the matcher matched. + if AnyCanonical::matches(current.as_ref()) { + match stack.pop() { + None => { + ctx.log(format_args!("-> canonical (unmatched) {}", current)); + return Ok(current); + } + Some((parent, child_idx, _)) => { + current = parent.with_child(child_idx, current)?; + current = current.optimize()?; + continue; + } + } + } + + // Try execute_parent (child-driven optimized execution) + if let Some(rewritten) = try_execute_parent(¤t, ctx)? { + ctx.log(format_args!( + "execute_parent rewrote {} -> {}", + current, rewritten + )); + current = rewritten.optimize()?; + continue; + } + + // Execute the array itself + match current.vtable().execute(¤t, ctx)? { + ExecutionStep::ExecuteChild(i, done) => { + let child = current + .nth_child(i) + .vortex_expect("ExecuteChild index in bounds"); + ctx.log(format_args!( + "ExecuteChild({i}): pushing {}, focusing on {}", + current, child + )); + stack.push((current, i, done)); + current = child.optimize()?; + } + ExecutionStep::Done(result) => { + ctx.log(format_args!("Done: {} -> {}", current, result)); + current = result; + } + } + } + + vortex_bail!( + "Exceeded maximum execution iterations ({}) while executing array", + *MAX_ITERATIONS, + ) + } } /// Execution context for batch CPU compute. @@ -189,7 +299,7 @@ impl Executable for ArrayRef { ctx.log(format_args!("-> {}", result.as_ref())); Ok(result) } - ExecutionStep::ExecuteChild(i) => { + ExecutionStep::ExecuteChild(i, _) => { // For single-step execution, handle ExecuteChild by executing the child, // replacing it, and returning the updated array. let child = array.nth_child(i).vortex_expect("valid child index"); @@ -200,25 +310,70 @@ impl Executable for ArrayRef { } } +/// Try execute_parent on each child of the array. +fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult> { + for child_idx in 0..array.nchildren() { + let child = array + .nth_child(child_idx) + .vortex_expect("checked nchildren"); + if let Some(result) = child + .vtable() + .execute_parent(&child, array, child_idx, ctx)? + { + result.statistics().inherit_from(array.statistics()); + return Ok(Some(result)); + } + } + Ok(None) +} + +/// A predicate that determines when an array has reached a desired form during execution. +pub type DonePredicate = fn(&dyn Array) -> bool; + /// The result of a single execution step on an array encoding. /// /// Instead of recursively executing children, encodings return an `ExecutionStep` that tells the /// scheduler what to do next. This enables the scheduler to manage execution iteratively using /// an explicit work stack, run cross-step optimizations, and cache shared sub-expressions. -#[derive(Debug)] pub enum ExecutionStep { - /// Request that the scheduler execute child at index `i` to columnar form, replace it in - /// this array, then re-enter execution on the updated array. + /// Request that the scheduler execute child at the given index, using the provided + /// [`DonePredicate`] to determine when the child is "done", then replace the child in this + /// array and re-enter execution. /// /// Between steps, the scheduler runs reduce/reduce_parent rules to fixpoint, enabling /// cross-step optimization (e.g., pushing scalar functions through newly-decoded children). - ExecuteChild(usize), + /// + /// Use [`ExecutionStep::execute_child`] instead of constructing this variant directly. + ExecuteChild(usize, DonePredicate), /// Execution is complete. The result may be in any encoding — not necessarily canonical. - /// The scheduler will continue executing the result if it is not yet columnar. + /// The scheduler will continue executing the result if it has not yet reached the target form. Done(ArrayRef), } +impl ExecutionStep { + /// Request execution of child at `child_idx` until it matches the given [`Matcher`]. + pub fn execute_child(child_idx: usize) -> Self { + ExecutionStep::ExecuteChild(child_idx, M::matches) + } + + /// Signal that execution is complete with the given result. + pub fn done(result: ArrayRef) -> Self { + ExecutionStep::Done(result) + } +} + +impl fmt::Debug for ExecutionStep { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExecutionStep::ExecuteChild(idx, _) => { + f.debug_tuple("ExecuteChild").field(idx).finish() + } + ExecutionStep::Done(result) => f.debug_tuple("Done").field(result).finish(), + } + } +} + /// Extension trait for creating an execution context from a session. pub trait VortexSessionExecute { /// Create a new execution context from this session.