feat(datafusion): allow configuring ExpressionConvertor on VortexFormat and VortexFormatFactory

vortex-datafusion/src/convert/exprs.rs

@@ -52,7 +52,9 @@ use crate::convert::FromDataFusion;
 
 /// Result of splitting a projection into Vortex expressions and leftover DataFusion projections.
 pub struct ProcessedProjection {
+    /// Projection evaluated by the Vortex scan.
     pub scan_projection: Expression,
+    /// Projection evaluated by DataFusion after the Vortex scan.
     pub leftover_projection: ProjectionExprs,
 }
 
@@ -70,6 +72,47 @@ pub(crate) fn make_vortex_predicate(
 }
 
 /// Trait for converting DataFusion expressions to Vortex ones.
+///
+/// # Implementing a custom convertor
+///
+/// ```
+/// use std::sync::Arc;
+///
+/// use arrow_schema::Schema;
+/// use datafusion_common::Result as DFResult;
+/// use datafusion_physical_expr::PhysicalExpr;
+/// use datafusion_physical_expr::projection::ProjectionExprs;
+/// use vortex::expr::Expression;
+/// use vortex_datafusion::convert::DefaultExpressionConvertor;
+/// use vortex_datafusion::convert::ExpressionConvertor;
+/// use vortex_datafusion::convert::ProcessedProjection;
+///
+/// struct CustomExpressionConvertor(DefaultExpressionConvertor);
+///
+/// impl ExpressionConvertor for CustomExpressionConvertor {
+///     fn can_be_pushed_down(&self, expr: &Arc<dyn PhysicalExpr>, schema: &Schema) -> bool {
+///         self.0.can_be_pushed_down(expr, schema)
+///     }
+///
+///     fn convert(&self, expr: &dyn PhysicalExpr) -> DFResult<Expression> {
+///         self.0.convert(expr)
+///     }
+///
+///     fn split_projection(
+///         &self,
+///         source_projection: ProjectionExprs,
+///         input_schema: &Schema,
+///         output_schema: &Schema,
+///     ) -> DFResult<ProcessedProjection> {
+///         self.0
+///             .split_projection(source_projection, input_schema, output_schema)
+///     }
+/// }
+///
+/// let _convertor: Arc<dyn ExpressionConvertor> = Arc::new(CustomExpressionConvertor(
+///     DefaultExpressionConvertor::default(),
+/// ));
+/// ```
 pub trait ExpressionConvertor: Send + Sync {
     /// Can an expression be pushed down given a specific schema
     fn can_be_pushed_down(&self, expr: &Arc<dyn PhysicalExpr>, schema: &Schema) -> bool;

vortex-datafusion/src/convert/mod.rs

@@ -18,6 +18,7 @@ pub(crate) mod stats;
 
 pub use exprs::DefaultExpressionConvertor;
 pub use exprs::ExpressionConvertor;
+pub use exprs::ProcessedProjection;
 
 /// First-party trait for implementing conversion from DataFusion types to Vortex types.
 pub trait FromDataFusion<D: ?Sized>: Sized {

vortex-datafusion/src/lib.rs

@@ -166,12 +166,18 @@ mod common_tests {
     impl TestSessionContext {
         /// Create a new test session context with the given projection pushdown setting.
         pub fn new(projection_pushdown: bool) -> Self {
-            let store = Arc::new(InMemory::new());
             let opts = VortexTableOptions {
                 projection_pushdown,
                 ..Default::default()
             };
             let factory = Arc::new(VortexFormatFactory::new().with_options(opts));
+
+            Self::new_with_factory(factory)
+        }
+
+        /// Create a new test session context with the given Vortex format factory.
+        pub fn new_with_factory(factory: Arc<VortexFormatFactory>) -> Self {
+            let store = Arc::new(InMemory::new());
             let mut session_state_builder = SessionStateBuilder::new()
                 .with_default_features()
                 .with_table_factory(