From 9965e511111b02e29ca7a67f7cb85322bb6d3026 Mon Sep 17 00:00:00 2001
From: Adam Gutglick <adam@spiraldb.com>
Date: Wed, 29 Jul 2026 13:04:36 +0100
Subject: [PATCH 1/4] [WIP] DataFusion morselized scan

Signed-off-by: Adam Gutglick <adam@spiraldb.com>
---
 vortex-datafusion/src/persistent/mod.rs    |   1 +
 vortex-datafusion/src/persistent/morsel.rs | 911 +++++++++++++++++++++
 vortex-datafusion/src/persistent/opener.rs |  42 +-
 vortex-datafusion/src/persistent/source.rs |  31 +
 4 files changed, 964 insertions(+), 21 deletions(-)
 create mode 100644 vortex-datafusion/src/persistent/morsel.rs

diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs
index 53fefacafa4..8a0d1d84f92 100644
--- a/vortex-datafusion/src/persistent/mod.rs
+++ b/vortex-datafusion/src/persistent/mod.rs
@@ -26,6 +26,7 @@ mod access_plan;
 mod cache;
 mod format;
 pub mod metrics;
+pub mod morsel;
 mod opener;
 pub mod reader;
 mod sink;
diff --git a/vortex-datafusion/src/persistent/morsel.rs b/vortex-datafusion/src/persistent/morsel.rs
new file mode 100644
index 00000000000..4ddbc1c1549
--- /dev/null
+++ b/vortex-datafusion/src/persistent/morsel.rs
@@ -0,0 +1,911 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#![allow(missing_docs)]
+
+//! Morsel-driven I/O support for Vortex files.
+
+use std::ops::Range;
+use std::sync::Arc;
+use std::sync::Weak;
+
+use arrow_array::RecordBatch;
+use arrow_array::RecordBatchOptions;
+use arrow_schema::Field;
+use arrow_schema::Schema;
+use arrow_schema::SchemaRef;
+use datafusion_common::DataFusionError;
+use datafusion_common::Result as DFResult;
+use datafusion_common::ScalarValue;
+use datafusion_common::Statistics;
+use datafusion_common::arrow::array::AsArray;
+use datafusion_common::exec_datafusion_err;
+use datafusion_common::format::MetricCategory;
+use datafusion_datasource::PartitionedFile;
+use datafusion_datasource::TableSchema;
+use datafusion_datasource::morsel::Morsel;
+use datafusion_datasource::morsel::MorselPlan;
+use datafusion_datasource::morsel::MorselPlanner;
+use datafusion_datasource::morsel::Morselizer;
+use datafusion_execution::cache::cache_manager::CachedFileMetadataEntry;
+use datafusion_execution::cache::cache_manager::FileMetadataCache;
+use datafusion_physical_expr::PhysicalExprRef;
+use datafusion_physical_expr::projection::ProjectionExprs;
+use datafusion_physical_expr::projection::Projector;
+use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
+use datafusion_physical_expr::split_conjunction;
+use datafusion_physical_expr::utils::collect_columns;
+use datafusion_physical_expr::utils::reassign_expr_columns;
+use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
+use datafusion_physical_expr_adapter::replace_columns_with_literals;
+use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+use datafusion_physical_plan::metrics::MetricBuilder;
+use datafusion_pruning::FilePruner;
+use futures::StreamExt;
+use futures::TryStreamExt;
+use futures::stream::BoxStream;
+use itertools::Itertools;
+use object_store::path::Path;
+use tracing::Instrument;
+use vortex::array::VortexSessionExecute;
+use vortex::dtype::FieldMask;
+use vortex::error::VortexError;
+use vortex::error::VortexExpect;
+use vortex::file::Footer;
+use vortex::file::OpenOptionsSessionExt;
+use vortex::file::VortexFile;
+use vortex::io::InstrumentedReadAt;
+use vortex::io::VortexReadAt;
+use vortex::layout::LayoutReader;
+use vortex::layout::scan::repeated_scan::RepeatedScan;
+use vortex::layout::scan::scan_builder::ScanBuilder;
+use vortex::layout::scan::split_by::SplitBy;
+use vortex::metrics::Label;
+use vortex::metrics::MetricsRegistry;
+use vortex::session::VortexSession;
+use vortex_arrow::ArrowSessionExt;
+use vortex_utils::aliases::dash_map::DashMap;
+use vortex_utils::aliases::dash_map::Entry;
+
+use crate::VortexAccessPlan;
+use crate::convert::ExpressionConvertor;
+use crate::convert::exprs::ProcessedProjection;
+use crate::convert::exprs::make_vortex_predicate;
+use crate::convert::schema::calculate_physical_schema;
+use crate::metrics::PARTITION_LABEL;
+use crate::metrics::PATH_LABEL;
+use crate::persistent::cache::CachedVortexMetadata;
+use crate::persistent::stream::PrunableStream;
+use crate::reader::VortexReaderFactory;
+
+/// Creates morsel planners for Vortex files.
+pub struct VortexMorselizer {
+    pub partition: usize,
+    pub session: VortexSession,
+    pub vortex_reader_factory: Arc<dyn VortexReaderFactory>,
+    /// Optional table schema projection. The indices are w.r.t. the `table_schema`, which is
+    /// all fields in the final scan result not including the partition columns.
+    pub projection: ProjectionExprs,
+    /// Filter expression optimized for pushdown into Vortex scan operations.
+    /// This may be a subset of file_pruning_predicate containing only expressions
+    /// that Vortex can efficiently evaluate.
+    pub filter: Option<PhysicalExprRef>,
+    /// Filter expression used by DataFusion's FilePruner to eliminate files based on
+    /// statistics and partition values without opening them.
+    pub file_pruning_predicate: Option<PhysicalExprRef>,
+    pub expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+    /// This is the table's schema without partition columns. It may contain fields which do
+    /// not exist in the file, and are supplied by the `schema_adapter_factory`.
+    pub table_schema: TableSchema,
+    /// If provided, the scan will not return more than this many rows.
+    pub limit: Option<u64>,
+    /// A metrics object for tracking performance of the scan.
+    pub metrics_registry: Arc<dyn MetricsRegistry>,
+    /// DataFusion-native metrics exposed through `DataSourceExec`.
+    pub df_metrics: ExecutionPlanMetricsSet,
+    /// A shared cache of file readers.
+    ///
+    /// To save on the overhead of reparsing FlatBuffers and rebuilding the layout tree, we cache
+    /// a file reader the first time we read a file.
+    pub layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
+    /// Shared full-file natural split ranges keyed by file path.
+    pub natural_split_ranges: Arc<DashMap<Path, Arc<[Range<u64>]>>>,
+    /// Whether the query has output ordering specified
+    pub has_output_ordering: bool,
+
+    pub expression_convertor: Arc<dyn ExpressionConvertor>,
+    pub file_metadata_cache: Option<Arc<dyn FileMetadataCache>>,
+    /// Whether to enable expression pushdown into the underlying Vortex scan.
+    pub projection_pushdown: bool,
+    pub scan_concurrency: Option<usize>,
+}
+
+impl std::fmt::Debug for VortexMorselizer {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("VortexMorselizer")
+            .field("partition", &self.partition)
+            .field("session", &self.session)
+            .field("vortex_reader_factory", &self.vortex_reader_factory)
+            .field("projection", &self.projection)
+            .field("filter", &self.filter)
+            .field("file_pruning_predicate", &self.file_pruning_predicate)
+            .field("expr_adapter_factory", &self.expr_adapter_factory)
+            .field("table_schema", &self.table_schema)
+            .field("limit", &self.limit)
+            .field("df_metrics", &self.df_metrics)
+            .field("layout_readers", &self.layout_readers)
+            .field("natural_split_ranges", &self.natural_split_ranges)
+            .field("has_output_ordering", &self.has_output_ordering)
+            .field("file_metadata_cache", &self.file_metadata_cache)
+            .field("projection_pushdown", &self.projection_pushdown)
+            .field("scan_concurrency", &self.scan_concurrency)
+            .finish_non_exhaustive()
+    }
+}
+
+impl Morselizer for VortexMorselizer {
+    fn plan_file(&self, file: PartitionedFile) -> DFResult<Box<dyn MorselPlanner>> {
+        Ok(Box::new(VortexMorselPlanner::try_new(self, file)?))
+    }
+}
+
+/// Plans morsels for a Vortex file.
+#[derive(Debug)]
+pub struct VortexMorselPlanner {
+    #[allow(dead_code)]
+    state: State,
+}
+
+impl VortexMorselPlanner {
+    /// Create new [`VortexMorselPlanner`]
+    pub fn try_new(morselizer: &VortexMorselizer, file: PartitionedFile) -> DFResult<Self> {
+        // Calculate the output schema before replacing partition columns with literals so it
+        // retains the table and partition-field metadata declared by the plan.
+        let output_schema = Arc::new(
+            morselizer
+                .projection
+                .project_schema(morselizer.table_schema.table_schema())?,
+        );
+        let session = morselizer.session.clone();
+        let metrics_registry = Arc::clone(&morselizer.metrics_registry);
+        let labels = vec![
+            Label::new(PATH_LABEL, file.path().to_string()),
+            Label::new(PARTITION_LABEL, morselizer.partition.to_string()),
+        ];
+
+        let mut projection = morselizer.projection.clone();
+        let mut filter = morselizer.filter.clone();
+
+        let reader = morselizer
+            .vortex_reader_factory
+            .create_reader(&file, &session)?;
+
+        let reader =
+            InstrumentedReadAt::new_with_labels(reader, metrics_registry.as_ref(), labels.clone());
+
+        let mut file_pruning_predicate = morselizer.file_pruning_predicate.clone();
+        let expr_adapter_factory = Arc::clone(&morselizer.expr_adapter_factory);
+        let file_metadata_cache = morselizer.file_metadata_cache.clone();
+
+        let unified_file_schema = Arc::clone(morselizer.table_schema.file_schema());
+        let limit = morselizer.limit;
+        let layout_readers = Arc::clone(&morselizer.layout_readers);
+        let natural_split_ranges = Arc::clone(&morselizer.natural_split_ranges);
+        let has_output_ordering = morselizer.has_output_ordering;
+        let scan_concurrency = morselizer.scan_concurrency;
+
+        let expr_convertor = Arc::clone(&morselizer.expression_convertor);
+        let projection_pushdown = morselizer.projection_pushdown;
+
+        let predicate_creation_errors = MetricBuilder::new(&morselizer.df_metrics)
+            .with_category(MetricCategory::Rows)
+            .global_counter("num_predicate_creation_errors");
+
+        // Replace column access for partition columns with literals
+        #[expect(clippy::disallowed_types)]
+        let literal_value_cols = morselizer
+            .table_schema
+            .table_partition_cols()
+            .iter()
+            .map(|f| f.name())
+            .cloned()
+            .zip(file.partition_values.clone())
+            .collect::<std::collections::HashMap<String, ScalarValue>>();
+
+        let predicate_uses_partition_columns =
+            file_pruning_predicate.as_ref().is_some_and(|predicate| {
+                collect_columns(predicate)
+                    .iter()
+                    .any(|column| literal_value_cols.contains_key(column.name()))
+            });
+
+        if !literal_value_cols.is_empty() {
+            projection = projection.try_map_exprs(|expr| {
+                replace_columns_with_literals(Arc::clone(&expr), &literal_value_cols)
+            })?;
+            filter = filter
+                .map(|p| replace_columns_with_literals(p, &literal_value_cols))
+                .transpose()?;
+            file_pruning_predicate = file_pruning_predicate
+                .map(|p| replace_columns_with_literals(p, &literal_value_cols))
+                .transpose()?;
+        }
+
+        // FilePruner requires a statistics object even when the rewritten predicate
+        // only contains partition literals. Supply unknown file-column statistics in
+        // that case so static and dynamic partition predicates can still prune.
+        let synthetic_statistics = (!file.has_statistics() && predicate_uses_partition_columns)
+            .then(|| {
+                file.clone()
+                    .with_statistics(Arc::new(Statistics::new_unknown(&unified_file_schema)))
+            });
+        let pruning_file = synthetic_statistics.as_ref().unwrap_or(&file);
+
+        let file_pruner = file_pruning_predicate
+            .filter(|_| file.has_statistics() || predicate_uses_partition_columns)
+            .and_then(|predicate| {
+                FilePruner::try_new(
+                    Arc::clone(&predicate),
+                    &unified_file_schema,
+                    pruning_file,
+                    predicate_creation_errors,
+                )
+            });
+
+        Ok(Self {
+            state: State::Start {
+                state: FileOpenState {
+                    file,
+                    output_schema,
+                    session,
+                    metrics_registry,
+                    labels,
+                    projection,
+                    filter,
+                    reader,
+                    file_pruner,
+                    expr_adapter_factory,
+                    file_metadata_cache,
+                    unified_file_schema,
+                    limit,
+                    layout_readers,
+                    natural_split_ranges,
+                    has_output_ordering,
+                    expr_convertor,
+                    projection_pushdown,
+                    scan_concurrency,
+                },
+            },
+        })
+    }
+}
+
+#[allow(dead_code)]
+struct FileOpenState {
+    file: PartitionedFile,
+    output_schema: SchemaRef,
+    session: VortexSession,
+    metrics_registry: Arc<dyn MetricsRegistry>,
+    labels: Vec<Label>,
+    projection: ProjectionExprs,
+    filter: Option<PhysicalExprRef>,
+    reader: InstrumentedReadAt<Arc<dyn VortexReadAt>>,
+    file_pruner: Option<FilePruner>,
+    expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+    file_metadata_cache: Option<Arc<dyn FileMetadataCache>>,
+    unified_file_schema: SchemaRef,
+    limit: Option<u64>,
+    layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
+    natural_split_ranges: Arc<DashMap<Path, Arc<[Range<u64>]>>>,
+    has_output_ordering: bool,
+    expr_convertor: Arc<dyn ExpressionConvertor>,
+    projection_pushdown: bool,
+    scan_concurrency: Option<usize>,
+}
+
+enum State {
+    Start {
+        state: FileOpenState,
+    },
+    OpenFooter {
+        state: FileOpenState,
+    },
+    OpenFile {
+        state: FileOpenState,
+        footer: Option<Footer>,
+    },
+    BuildScan {
+        state: FileOpenState,
+        vxf: VortexFile,
+    },
+    PreparedScan {
+        scan: RepeatedScan,
+        file_pruner: Option<FilePruner>,
+        output_schema: SchemaRef,
+        session: VortexSession,
+        stream_target_field: Field,
+        file_location: Path,
+        projector: Projector,
+    },
+    Done,
+}
+
+impl State {
+    fn transition(self) -> DFResult<Self> {
+        match self {
+            State::Start { mut state } => {
+                if let Some(pruner) = state.file_pruner.as_mut()
+                    && pruner.should_prune()?
+                {
+                    Ok(Self::Done)
+                } else {
+                    Ok(Self::OpenFooter { state })
+                }
+            }
+            State::OpenFooter { state } => {
+                let footer = state
+                    .file_metadata_cache
+                    .as_ref()
+                    .and_then(|cache| cache.get(state.file.path()))
+                    .filter(|entry| entry.is_valid_for(&state.file.object_meta))
+                    .and_then(|entry| {
+                        entry
+                            .file_metadata
+                            .as_any()
+                            .downcast_ref::<CachedVortexMetadata>()
+                            .map(|vortex_metadata| vortex_metadata.footer().clone())
+                    });
+
+                Ok(State::OpenFile { state, footer })
+            }
+            State::OpenFile { state, footer } => Ok(State::OpenFile { state, footer }),
+            State::BuildScan { state, vxf } => Ok(State::BuildScan { state, vxf }),
+            State::PreparedScan {
+                scan,
+                file_pruner,
+                output_schema,
+                session,
+                stream_target_field,
+                file_location,
+                projector,
+            } => Ok(State::PreparedScan {
+                scan,
+                file_pruner,
+                output_schema,
+                session,
+                stream_target_field,
+                file_location,
+                projector,
+            }),
+            State::Done => Ok(State::Done),
+        }
+    }
+}
+
+impl std::fmt::Debug for State {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Start { state } => f.debug_struct("Start").field("state", state).finish(),
+            Self::OpenFooter { state } => {
+                f.debug_struct("OpenFooter").field("state", state).finish()
+            }
+            Self::OpenFile { state, footer } => f
+                .debug_struct("OpenFile")
+                .field("state", state)
+                .field("footer", &footer.as_ref().map(|_| "<footer>"))
+                .finish(),
+            Self::BuildScan { state, .. } => f
+                .debug_struct("BuildScan")
+                .field("state", state)
+                .field("vxf", &"<vortex_file>")
+                .finish(),
+            Self::PreparedScan {
+                file_pruner,
+                output_schema,
+                stream_target_field,
+                file_location,
+                projector,
+                ..
+            } => f
+                .debug_struct("PreparedScan")
+                .field("scan", &"<repeated_scan>")
+                .field(
+                    "file_pruner",
+                    &file_pruner.as_ref().map(|_| "<file_pruner>"),
+                )
+                .field("output_schema", output_schema)
+                .field("stream_target_field", stream_target_field)
+                .field("file_location", file_location)
+                .field("projector", projector)
+                .finish(),
+            Self::Done => f.debug_struct("Done").finish(),
+        }
+    }
+}
+
+impl std::fmt::Debug for FileOpenState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("FileOpenState")
+            .field("file", &self.file)
+            .field("output_schema", &self.output_schema)
+            .field("session", &self.session)
+            .field("labels", &self.labels)
+            .field("projection", &self.projection)
+            .field("filter", &self.filter)
+            .field(
+                "file_pruner",
+                &self.file_pruner.as_ref().map(|_| "<file_pruner>"),
+            )
+            .field("expr_adapter_factory", &self.expr_adapter_factory)
+            .field("file_metadata_cache", &self.file_metadata_cache)
+            .field("unified_file_schema", &self.unified_file_schema)
+            .field("limit", &self.limit)
+            .field("layout_readers", &self.layout_readers)
+            .field("natural_split_ranges", &self.natural_split_ranges)
+            .field("has_output_ordering", &self.has_output_ordering)
+            .field("expr_convertor", &"<expr_convertor>")
+            .field("projection_pushdown", &self.projection_pushdown)
+            .field("scan_concurrency", &self.scan_concurrency)
+            .finish_non_exhaustive()
+    }
+}
+
+struct VortexStreamMorsel {
+    inner: BoxStream<'static, DFResult<RecordBatch>>,
+}
+
+impl std::fmt::Debug for VortexStreamMorsel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("VortexStreamMorsel").finish_non_exhaustive()
+    }
+}
+
+impl Morsel for VortexStreamMorsel {
+    fn into_stream(self: Box<Self>) -> BoxStream<'static, DFResult<RecordBatch>> {
+        self.inner
+    }
+}
+
+impl MorselPlanner for VortexMorselPlanner {
+    #[allow(clippy::cognitive_complexity)]
+    fn plan(self: Box<Self>) -> DFResult<Option<MorselPlan>> {
+        let VortexMorselPlanner { state } = *self;
+        let state = match state {
+            State::Done => return Ok(None),
+            state => state.transition()?,
+        };
+
+        match state {
+            State::OpenFile { state, footer } => {
+                let footer_cache_hit = footer.is_some();
+                let mut open_opts = state
+                    .session
+                    .open_options()
+                    .with_file_size(state.file.object_meta.size)
+                    .with_metrics_registry(Arc::clone(&state.metrics_registry))
+                    .with_labels(state.labels.clone());
+
+                if let Some(footer) = footer {
+                    open_opts = open_opts.with_footer(footer);
+                }
+
+                Ok(Some(
+                    MorselPlan::new().with_pending_planner(
+                        async move {
+                            let vxf =
+                                open_opts
+                                    .open_read(state.reader.clone())
+                                    .await
+                                    .map_err(|e| {
+                                        exec_datafusion_err!("Failed to open Vortex file {e}")
+                                    })?;
+
+                            // On a miss, cache the parsed footer so other partitions and later executions
+                            // skip the footer fetch and parse. `infer_schema`/`infer_stats` also populate
+                            // this cache, but only when planning goes through `VortexFormat`.
+                            if !footer_cache_hit && let Some(cache) = &state.file_metadata_cache {
+                                cache.put(
+                                    state.file.path(),
+                                    CachedFileMetadataEntry::new(
+                                        state.file.object_meta.clone(),
+                                        Arc::new(CachedVortexMetadata::new(&vxf)),
+                                    ),
+                                );
+                            }
+
+                            let new_state = if vxf.row_count() == 0 {
+                                State::Done
+                            } else {
+                                State::BuildScan { state, vxf }
+                            };
+
+                            Ok(Box::new(VortexMorselPlanner { state: new_state })
+                                as Box<dyn MorselPlanner>)
+                        }
+                        .in_current_span(),
+                    ),
+                ))
+            }
+            State::BuildScan { state, vxf } => {
+                let FileOpenState {
+                    file,
+                    output_schema,
+                    session,
+                    metrics_registry,
+                    projection,
+                    filter,
+                    file_pruner,
+                    expr_adapter_factory,
+                    unified_file_schema,
+                    limit,
+                    layout_readers,
+                    natural_split_ranges,
+                    has_output_ordering,
+                    expr_convertor,
+                    projection_pushdown,
+                    scan_concurrency,
+                    ..
+                } = state;
+
+                // This is the expected arrow types of the actual columns in the file, which might have different types
+                // from the unified logical schema or miss
+                let this_file_schema = Arc::new(calculate_physical_schema(
+                    vxf.dtype(),
+                    &unified_file_schema,
+                    &session.arrow(),
+                )?);
+
+                let expr_adapter = expr_adapter_factory.create(
+                    Arc::clone(&unified_file_schema),
+                    Arc::clone(&this_file_schema),
+                )?;
+
+                let simplifier = PhysicalExprSimplifier::new(&this_file_schema);
+
+                // The adapter rewrites the expressions to the local file schema, allowing
+                // for schema evolution and divergence between the table's schema and individual files.
+                let filter = filter
+                    .map(|filter| {
+                        // Expression might now reference columns that don't exist in the file, so we can give it
+                        // another simplification pass.
+                        simplifier.simplify(expr_adapter.rewrite(filter)?)
+                    })
+                    .transpose()?;
+                let projection =
+                    projection.try_map_exprs(|p| simplifier.simplify(expr_adapter.rewrite(p)?))?;
+
+                let ProcessedProjection {
+                    scan_projection,
+                    leftover_projection,
+                } = if projection_pushdown {
+                    expr_convertor.split_projection(
+                        projection.clone(),
+                        &this_file_schema,
+                        output_schema.as_ref(),
+                    )?
+                } else {
+                    // When projection pushdown is disabled, read only the required columns
+                    // and apply the full projection after the scan.
+                    expr_convertor.no_pushdown_projection(projection.clone(), &this_file_schema)?
+                };
+
+                // The schema of the stream returned from the vortex scan.
+                // We use a reference schema for types that don't roundtrip (Dictionary, Utf8, etc.).
+                let scan_dtype = scan_projection.return_dtype(vxf.dtype()).map_err(|_e| {
+                    exec_datafusion_err!("Couldn't get the dtype for the underlying Vortex scan")
+                })?;
+
+                // When projection pushdown is enabled, the scan outputs the projected columns.
+                // When disabled, the scan outputs raw columns and the projection is applied after.
+                let scan_reference_schema = if projection_pushdown {
+                    (*output_schema).clone()
+                } else {
+                    // Build schema from the raw columns being read
+                    let column_indices = projection.column_indices();
+                    let fields: Vec<_> = column_indices
+                        .into_iter()
+                        .map(|idx| this_file_schema.field(idx).clone())
+                        .collect();
+                    Schema::new_with_metadata(fields, this_file_schema.metadata().clone())
+                };
+                let stream_schema = calculate_physical_schema(
+                    &scan_dtype,
+                    &scan_reference_schema,
+                    &session.arrow(),
+                )?;
+
+                let leftover_projection = leftover_projection
+                    .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;
+                let projector = leftover_projection.make_projector(&stream_schema)?;
+
+                // We share our layout readers with others partitions in the scan, so we can only need to read each layout in each file once.
+                let layout_reader = match layout_readers.entry(file.object_meta.location.clone()) {
+                    Entry::Occupied(mut occupied_entry) => {
+                        if let Some(reader) = occupied_entry.get().upgrade() {
+                            tracing::trace!("reusing layout reader for {}", occupied_entry.key());
+                            reader
+                        } else {
+                            tracing::trace!("creating layout reader for {}", occupied_entry.key());
+                            let reader = vxf.layout_reader().map_err(|e| {
+                                DataFusionError::Execution(format!(
+                                    "Failed to create layout reader: {e}"
+                                ))
+                            })?;
+                            occupied_entry.insert(Arc::downgrade(&reader));
+                            reader
+                        }
+                    }
+                    Entry::Vacant(vacant_entry) => {
+                        tracing::trace!("creating layout reader for {}", vacant_entry.key());
+                        let reader = vxf.layout_reader().map_err(|e| {
+                            DataFusionError::Execution(format!(
+                                "Failed to create layout reader: {e}"
+                            ))
+                        })?;
+                        vacant_entry.insert(Arc::downgrade(&reader));
+
+                        reader
+                    }
+                };
+
+                let mut scan_builder =
+                    ScanBuilder::new(session.clone(), Arc::clone(&layout_reader));
+
+                if let Some(vortex_plan) = file.extensions.get::<VortexAccessPlan>() {
+                    scan_builder = vortex_plan.apply_to_builder(scan_builder);
+                }
+
+                if let Some(file_range) = file.range {
+                    let byte_range = Range {
+                        start: u64::try_from(file_range.start).map_err(|_| {
+                            exec_datafusion_err!("Vortex file range start is negative")
+                        })?,
+                        end: u64::try_from(file_range.end).map_err(|_| {
+                            exec_datafusion_err!("Vortex file range end is negative")
+                        })?,
+                    };
+                    if byte_range.start != 0 || byte_range.end != file.object_meta.size {
+                        // Full-file scans already cover every natural split. Only translate the
+                        // byte range back into row boundaries when DataFusion has trimmed the file.
+                        let natural_split_ranges = natural_split_ranges_for_file(
+                            natural_split_ranges.as_ref(),
+                            &file.object_meta.location,
+                            &layout_reader,
+                        )?;
+
+                        let Some(row_range) = split_aligned_row_range(
+                            byte_range,
+                            file.object_meta.size,
+                            natural_split_ranges.as_ref(),
+                        ) else {
+                            return Ok(None);
+                        };
+
+                        scan_builder = scan_builder.with_row_range(row_range);
+                    }
+                }
+
+                let filter = filter
+                    .and_then(|f| {
+                        // Verify that all filters we've accepted from DataFusion get pushed down.
+                        // This will only fail if the user has not configured a suitable
+                        // PhysicalExprAdapterFactory on the file source to handle rewriting the
+                        // expression to handle missing/reordered columns in the Vortex file.
+                        let (pushed, unpushed): (Vec<PhysicalExprRef>, Vec<PhysicalExprRef>) =
+                            split_conjunction(&f)
+                                .into_iter()
+                                .cloned()
+                                .partition(|expr| {
+                                    expr_convertor.can_be_pushed_down(expr, &this_file_schema)
+                                });
+
+                        if !unpushed.is_empty() {
+                            return Some(Err(exec_datafusion_err!(
+                                r#"VortexSource accepted but failed to push {} filters.
+                            This should never happen if you have a properly configured
+                            PhysicalExprAdapterFactory configured on the source.
+
+                            Failed filters:
+
+                            {unpushed:#?}
+                            "#,
+                                unpushed.len()
+                            )));
+                        }
+
+                        make_vortex_predicate(expr_convertor.as_ref(), &pushed).transpose()
+                    })
+                    .transpose()?;
+
+                if let Some(limit) = limit {
+                    scan_builder = scan_builder.with_limit(limit);
+                }
+
+                if let Some(concurrency) = scan_concurrency {
+                    scan_builder = scan_builder.with_concurrency(concurrency);
+                }
+
+                let scan = scan_builder
+                    .with_metrics_registry(metrics_registry)
+                    .with_projection(scan_projection)
+                    .with_some_filter(filter)
+                    .with_ordered(has_output_ordering)
+                    .prepare()
+                    .map_err(|e| exec_datafusion_err!("Failed to prepare Vortex scan: {e}"))?;
+
+                let stream_target_field =
+                    Field::new_struct("", stream_schema.fields().clone(), false);
+                let file_location = file.object_meta.location;
+
+                Ok(Some(MorselPlan::new().with_planners(vec![Box::new(
+                    Self {
+                        state: State::PreparedScan {
+                            scan,
+                            file_pruner,
+                            output_schema,
+                            session,
+                            stream_target_field,
+                            file_location,
+                            projector,
+                        },
+                    },
+                )])))
+            }
+            State::PreparedScan {
+                scan,
+                file_pruner,
+                output_schema,
+                session,
+                stream_target_field,
+                file_location,
+                projector,
+            } => {
+                let stream = scan
+                    .execute_array_stream(None)
+                    .map_err(|e| exec_datafusion_err!("Failed to create Vortex stream: {e}"))?
+                    // Convert to Arrow inline on the polling thread: DataFusion sources are expected
+                    // to do their CPU work inside `poll_next`, and spawning this onto the blocking
+                    // pool oversubscribes the CPU.
+                    .map(move |chunk| {
+                        let mut ctx = session.create_execution_ctx();
+                        chunk.and_then(|chunk| {
+                            let arrow_session = ctx.session().clone();
+                            let arrow = arrow_session.arrow().execute_arrow(
+                                chunk,
+                                Some(&stream_target_field),
+                                &mut ctx,
+                            )?;
+                            Ok(RecordBatch::from(arrow.as_struct().clone()))
+                        })
+                    })
+                    .map_err(move |e: VortexError| vortex_file_read_error(&file_location, e))
+                    .map(move |batch| {
+                        let batch = if projector.projection().as_ref().is_empty() {
+                            batch
+                        } else {
+                            batch.and_then(|b| projector.project_batch(&b))
+                        }?;
+
+                        let (_, columns, row_count) = batch.into_parts();
+                        RecordBatch::try_new_with_options(
+                            Arc::clone(&output_schema),
+                            columns,
+                            &RecordBatchOptions::new().with_row_count(Some(row_count)),
+                        )
+                        .map_err(Into::into)
+                    })
+                    .boxed();
+
+                let stream = if let Some(file_pruner) = file_pruner {
+                    PrunableStream::new(file_pruner, stream).boxed()
+                } else {
+                    stream
+                };
+
+                Ok(Some(MorselPlan::new().with_morsels(vec![
+                    Box::new(VortexStreamMorsel { inner: stream }) as Box<dyn Morsel>,
+                ])))
+            }
+            State::Done => Ok(None),
+            new_state => Ok(Some(
+                MorselPlan::new().with_planners(vec![Box::new(Self { state: new_state })]),
+            )),
+        }
+    }
+}
+
+fn natural_split_ranges_for_file(
+    natural_split_ranges: &DashMap<Path, Arc<[Range<u64>]>>,
+    path: &Path,
+    layout_reader: &Arc<dyn LayoutReader>,
+) -> DFResult<Arc<[Range<u64>]>> {
+    if let Some(split_ranges) = natural_split_ranges.get(path) {
+        return Ok(Arc::clone(split_ranges.value()));
+    }
+
+    let split_ranges = compute_natural_split_ranges(layout_reader.as_ref())?;
+
+    match natural_split_ranges.entry(path.clone()) {
+        Entry::Occupied(entry) => Ok(Arc::clone(entry.get())),
+        Entry::Vacant(entry) => {
+            entry.insert(Arc::clone(&split_ranges));
+            Ok(split_ranges)
+        }
+    }
+}
+
+fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<Arc<[Range<u64>]>> {
+    let row_count = layout_reader.row_count();
+    let row_range = 0..row_count;
+    let split_points: Vec<_> = SplitBy::Layout
+        .splits(layout_reader, &row_range, &[FieldMask::All])
+        .map_err(|e| exec_datafusion_err!("Failed to compute Vortex natural splits: {e}"))?
+        .into_iter()
+        .tuple_windows()
+        .map(|(s, e)| s..e)
+        .collect::<Vec<_>>();
+
+    Ok(split_points.into())
+}
+
+/// Translate a DataFusion byte range to the contiguous natural split ranges it owns.
+/// Most splits are assigned by midpoint, but the leading split stays with the range that owns
+/// byte 0 so a tiny first byte range still claims the first rows.
+fn split_aligned_row_range(
+    byte_range: Range<u64>,
+    total_size: u64,
+    split_ranges: &[Range<u64>],
+) -> Option<Range<u64>> {
+    if byte_range.start >= byte_range.end {
+        return None;
+    }
+
+    let row_count = split_ranges.last().map(|split| split.end)?;
+    if row_count == 0 {
+        return None;
+    }
+
+    let mut owned_splits = split_ranges
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, split_range)| {
+            let assignment_byte = split_assignment_byte(idx, split_range, row_count, total_size);
+            byte_range.contains(&assignment_byte).then_some(split_range)
+        });
+
+    let first_split = owned_splits.next()?;
+    let mut row_range = first_split.start..first_split.end;
+    for split_range in owned_splits {
+        row_range.end = split_range.end;
+    }
+
+    Some(row_range)
+}
+
+fn split_assignment_byte(
+    idx: usize,
+    split_range: &Range<u64>,
+    row_count: u64,
+    total_size: u64,
+) -> u64 {
+    if idx == 0 && split_range.start == 0 {
+        // Byte 0 is the only stable representative for the leading split. A midpoint can fall
+        // into the next DataFusion byte range and leave the first range with no rows to read.
+        0
+    } else {
+        split_midpoint_to_byte(split_range, row_count, total_size)
+    }
+}
+
+fn split_midpoint_to_byte(split_range: &Range<u64>, row_count: u64, total_size: u64) -> u64 {
+    let midpoint_row = split_range.start + (split_range.end - split_range.start) / 2;
+    let midpoint_byte = (u128::from(midpoint_row) * u128::from(total_size)) / u128::from(row_count);
+
+    u64::try_from(midpoint_byte).vortex_expect("midpoint byte projection should fit into u64")
+}
+
+fn vortex_file_read_error(path: &Path, error: VortexError) -> DataFusionError {
+    DataFusionError::External(Box::new(
+        error.with_context(format!("Failed to read Vortex file: {path}")),
+    ))
+}
diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs
index f1eb05d969e..d720382a5e8 100644
--- a/vortex-datafusion/src/persistent/opener.rs
+++ b/vortex-datafusion/src/persistent/opener.rs
@@ -180,28 +180,28 @@ impl FileOpener for VortexOpener {
                 .transpose()?;
         }
 
-        Ok(async move {
-            // FilePruner requires a statistics object even when the rewritten predicate
-            // only contains partition literals. Supply unknown file-column statistics in
-            // that case so static and dynamic partition predicates can still prune.
-            let synthetic_statistics = (!file.has_statistics() && predicate_uses_partition_columns)
-                .then(|| {
-                    file.clone()
-                        .with_statistics(Arc::new(Statistics::new_unknown(&unified_file_schema)))
-                });
-            let pruning_file = synthetic_statistics.as_ref().unwrap_or(&file);
-
-            let mut file_pruner = file_pruning_predicate
-                .filter(|_| file.has_statistics() || predicate_uses_partition_columns)
-                .and_then(|predicate| {
-                    FilePruner::try_new(
-                        Arc::clone(&predicate),
-                        &unified_file_schema,
-                        pruning_file,
-                        predicate_creation_errors,
-                    )
-                });
+        // FilePruner requires a statistics object even when the rewritten predicate
+        // only contains partition literals. Supply unknown file-column statistics in
+        // that case so static and dynamic partition predicates can still prune.
+        let synthetic_statistics = (!file.has_statistics() && predicate_uses_partition_columns)
+            .then(|| {
+                file.clone()
+                    .with_statistics(Arc::new(Statistics::new_unknown(&unified_file_schema)))
+            });
+        let pruning_file = synthetic_statistics.as_ref().unwrap_or(&file);
+
+        let mut file_pruner = file_pruning_predicate
+            .filter(|_| file.has_statistics() || predicate_uses_partition_columns)
+            .and_then(|predicate| {
+                FilePruner::try_new(
+                    Arc::clone(&predicate),
+                    &unified_file_schema,
+                    pruning_file,
+                    predicate_creation_errors,
+                )
+            });
 
+        Ok(async move {
             // Check if this file should be pruned based on statistics/partition values.
             // Returns empty stream if file can be skipped entirely.
             if let Some(file_pruner) = file_pruner.as_mut()
diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs
index 81fcf937955..d24a77ba13e 100644
--- a/vortex-datafusion/src/persistent/source.rs
+++ b/vortex-datafusion/src/persistent/source.rs
@@ -35,6 +35,7 @@ use vortex::metrics::MetricsRegistry;
 use vortex::session::VortexSession;
 use vortex_utils::aliases::dash_map::DashMap;
 
+use super::morsel::VortexMorselizer;
 use super::opener::NaturalSplits;
 use super::opener::VortexOpener;
 use crate::VortexTableOptions;
@@ -387,6 +388,36 @@ impl FileSource for VortexSource {
         Arc::new(self.clone())
     }
 
+    fn create_morselizer(
+        &self,
+        object_store: Arc<dyn ObjectStore>,
+        base_config: &FileScanConfig,
+        partition: usize,
+    ) -> DFResult<Box<dyn datafusion_datasource::morsel::Morselizer>> {
+        let opener = self.create_vortex_opener(object_store, base_config, partition)?;
+
+        Ok(Box::new(VortexMorselizer {
+            partition: opener.partition,
+            session: opener.session,
+            vortex_reader_factory: opener.vortex_reader_factory,
+            projection: opener.projection,
+            filter: opener.filter,
+            file_pruning_predicate: opener.file_pruning_predicate,
+            expr_adapter_factory: opener.expr_adapter_factory,
+            table_schema: opener.table_schema,
+            limit: opener.limit,
+            metrics_registry: opener.metrics_registry,
+            df_metrics: opener.df_metrics,
+            layout_readers: opener.layout_readers,
+            natural_split_ranges: opener.natural_split_ranges,
+            has_output_ordering: opener.has_output_ordering,
+            expression_convertor: opener.expression_convertor,
+            file_metadata_cache: opener.file_metadata_cache,
+            projection_pushdown: opener.projection_pushdown,
+            scan_concurrency: opener.scan_concurrency,
+        }))
+    }
+
     fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {
         self.vortex_predicate.clone()
     }

From d33cd48c10752662734e3ed488bedce90dc5eca9 Mon Sep 17 00:00:00 2001
From: Adam Gutglick <adam@spiraldb.com>
Date: Thu, 30 Jul 2026 11:34:58 +0100
Subject: [PATCH 2/4] Remove opener in favor of Morselizer

Signed-off-by: Adam Gutglick <adam@spiraldb.com>
---
 .../src/persistent/access_plan.rs             |    6 +-
 vortex-datafusion/src/persistent/mod.rs       |    1 -
 .../persistent/{morsel.rs => morsel/mod.rs}   |  282 ++-
 .../src/persistent/morsel/tests.rs            | 1239 ++++++++++++
 vortex-datafusion/src/persistent/opener.rs    | 1792 -----------------
 vortex-datafusion/src/persistent/source.rs    |   76 +-
 6 files changed, 1456 insertions(+), 1940 deletions(-)
 rename vortex-datafusion/src/persistent/{morsel.rs => morsel/mod.rs} (82%)
 create mode 100644 vortex-datafusion/src/persistent/morsel/tests.rs
 delete mode 100644 vortex-datafusion/src/persistent/opener.rs

diff --git a/vortex-datafusion/src/persistent/access_plan.rs b/vortex-datafusion/src/persistent/access_plan.rs
index 0e14b6730b0..3fa6ebec280 100644
--- a/vortex-datafusion/src/persistent/access_plan.rs
+++ b/vortex-datafusion/src/persistent/access_plan.rs
@@ -9,8 +9,8 @@ use vortex::scan::selection::Selection;
 ///
 /// `VortexAccessPlan` is the hook to use when an external index or planner
 /// already knows that only part of a file needs to be scanned. The plan is
-/// attached as `extensions` on `PartitionedFile`, and the internal
-/// `VortexOpener` applies it before building the Vortex scan.
+/// attached as `extensions` on `PartitionedFile`, and the internal Vortex
+/// morsel planner applies it before building the Vortex scan.
 ///
 /// The current access plan surface is intentionally small: it lets callers
 /// provide a [`Selection`] that narrows the rows considered by the scan.
@@ -56,7 +56,7 @@ impl VortexAccessPlan {
 
     /// Applies this access plan to a [`ScanBuilder`].
     ///
-    /// This is used internally by the file opener after it has translated a
+    /// This is used internally by the morsel planner after it has translated a
     /// `PartitionedFile` into a Vortex scan.
     pub fn apply_to_builder(&self, mut scan_builder: ScanBuilder) -> ScanBuilder {
         let Self { selection } = self;
diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs
index 8a0d1d84f92..3d98ffd43d8 100644
--- a/vortex-datafusion/src/persistent/mod.rs
+++ b/vortex-datafusion/src/persistent/mod.rs
@@ -27,7 +27,6 @@ mod cache;
 mod format;
 pub mod metrics;
 pub mod morsel;
-mod opener;
 pub mod reader;
 mod sink;
 mod source;
diff --git a/vortex-datafusion/src/persistent/morsel.rs b/vortex-datafusion/src/persistent/morsel/mod.rs
similarity index 82%
rename from vortex-datafusion/src/persistent/morsel.rs
rename to vortex-datafusion/src/persistent/morsel/mod.rs
index 4ddbc1c1549..d43a8403e4f 100644
--- a/vortex-datafusion/src/persistent/morsel.rs
+++ b/vortex-datafusion/src/persistent/morsel/mod.rs
@@ -46,6 +46,7 @@ use futures::TryStreamExt;
 use futures::stream::BoxStream;
 use itertools::Itertools;
 use object_store::path::Path;
+use tokio::sync::OnceCell;
 use tracing::Instrument;
 use vortex::array::VortexSessionExecute;
 use vortex::dtype::FieldMask;
@@ -78,6 +79,9 @@ use crate::persistent::cache::CachedVortexMetadata;
 use crate::persistent::stream::PrunableStream;
 use crate::reader::VortexReaderFactory;
 
+pub(crate) type NaturalSplitRanges = Arc<[Range<u64>]>;
+pub(crate) type NaturalSplitCache = DashMap<Path, Arc<OnceCell<NaturalSplitRanges>>>;
+
 /// Creates morsel planners for Vortex files.
 pub struct VortexMorselizer {
     pub partition: usize,
@@ -97,6 +101,7 @@ pub struct VortexMorselizer {
     /// This is the table's schema without partition columns. It may contain fields which do
     /// not exist in the file, and are supplied by the `schema_adapter_factory`.
     pub table_schema: TableSchema,
+    /// Desired row count for record batches returned from the scan.
     /// If provided, the scan will not return more than this many rows.
     pub limit: Option<u64>,
     /// A metrics object for tracking performance of the scan.
@@ -109,7 +114,7 @@ pub struct VortexMorselizer {
     /// a file reader the first time we read a file.
     pub layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
     /// Shared full-file natural split ranges keyed by file path.
-    pub natural_split_ranges: Arc<DashMap<Path, Arc<[Range<u64>]>>>,
+    pub natural_split_ranges: Arc<NaturalSplitCache>,
     /// Whether the query has output ordering specified
     pub has_output_ordering: bool,
 
@@ -152,7 +157,6 @@ impl Morselizer for VortexMorselizer {
 /// Plans morsels for a Vortex file.
 #[derive(Debug)]
 pub struct VortexMorselPlanner {
-    #[allow(dead_code)]
     state: State,
 }
 
@@ -280,7 +284,6 @@ impl VortexMorselPlanner {
     }
 }
 
-#[allow(dead_code)]
 struct FileOpenState {
     file: PartitionedFile,
     output_schema: SchemaRef,
@@ -296,7 +299,7 @@ struct FileOpenState {
     unified_file_schema: SchemaRef,
     limit: Option<u64>,
     layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
-    natural_split_ranges: Arc<DashMap<Path, Arc<[Range<u64>]>>>,
+    natural_split_ranges: Arc<NaturalSplitCache>,
     has_output_ordering: bool,
     expr_convertor: Arc<dyn ExpressionConvertor>,
     projection_pushdown: bool,
@@ -318,6 +321,19 @@ enum State {
         state: FileOpenState,
         vxf: VortexFile,
     },
+    CalculateLayoutSplits {
+        state: FileOpenState,
+        vxf: VortexFile,
+        layout_reader: Arc<dyn LayoutReader>,
+        byte_range: Range<u64>,
+        split_ranges: Arc<OnceCell<NaturalSplitRanges>>,
+    },
+    PrepareScan {
+        state: FileOpenState,
+        vxf: VortexFile,
+        layout_reader: Arc<dyn LayoutReader>,
+        row_range: Option<Range<u64>>,
+    },
     PreparedScan {
         scan: RepeatedScan,
         file_pruner: Option<FilePruner>,
@@ -358,26 +374,7 @@ impl State {
 
                 Ok(State::OpenFile { state, footer })
             }
-            State::OpenFile { state, footer } => Ok(State::OpenFile { state, footer }),
-            State::BuildScan { state, vxf } => Ok(State::BuildScan { state, vxf }),
-            State::PreparedScan {
-                scan,
-                file_pruner,
-                output_schema,
-                session,
-                stream_target_field,
-                file_location,
-                projector,
-            } => Ok(State::PreparedScan {
-                scan,
-                file_pruner,
-                output_schema,
-                session,
-                stream_target_field,
-                file_location,
-                projector,
-            }),
-            State::Done => Ok(State::Done),
+            state => Ok(state),
         }
     }
 }
@@ -399,6 +396,25 @@ impl std::fmt::Debug for State {
                 .field("state", state)
                 .field("vxf", &"<vortex_file>")
                 .finish(),
+            Self::CalculateLayoutSplits {
+                state, byte_range, ..
+            } => f
+                .debug_struct("CalculateLayoutSplits")
+                .field("state", state)
+                .field("vxf", &"<vortex_file>")
+                .field("layout_reader", &"<layout_reader>")
+                .field("byte_range", byte_range)
+                .field("split_ranges", &"<once_cell>")
+                .finish(),
+            Self::PrepareScan {
+                state, row_range, ..
+            } => f
+                .debug_struct("PrepareScan")
+                .field("state", state)
+                .field("vxf", &"<vortex_file>")
+                .field("layout_reader", &"<layout_reader>")
+                .field("row_range", row_range)
+                .finish(),
             Self::PreparedScan {
                 file_pruner,
                 output_schema,
@@ -527,6 +543,88 @@ impl MorselPlanner for VortexMorselPlanner {
                 ))
             }
             State::BuildScan { state, vxf } => {
+                let layout_reader = layout_reader_for_file(
+                    state.layout_readers.as_ref(),
+                    &state.file.object_meta.location,
+                    &vxf,
+                )?;
+
+                let Some(byte_range) = partial_file_byte_range(&state.file)? else {
+                    return Ok(Some(MorselPlan::new().with_planners(vec![Box::new(
+                        Self {
+                            state: State::PrepareScan {
+                                state,
+                                vxf,
+                                layout_reader,
+                                row_range: None,
+                            },
+                        },
+                    )])));
+                };
+
+                let split_ranges = natural_split_cell_for_file(
+                    state.natural_split_ranges.as_ref(),
+                    &state.file.object_meta.location,
+                );
+
+                Ok(Some(MorselPlan::new().with_planners(vec![Box::new(
+                    Self {
+                        state: State::CalculateLayoutSplits {
+                            state,
+                            vxf,
+                            layout_reader,
+                            byte_range,
+                            split_ranges,
+                        },
+                    },
+                )])))
+            }
+            State::CalculateLayoutSplits {
+                state,
+                vxf,
+                layout_reader,
+                byte_range,
+                split_ranges,
+            } => {
+                let total_size = state.file.object_meta.size;
+                let compute_layout_reader = Arc::clone(&layout_reader);
+
+                Ok(Some(
+                    MorselPlan::new().with_pending_planner(
+                        async move {
+                            let split_ranges = split_ranges
+                                .get_or_try_init(|| {
+                                    compute_natural_split_ranges_async(compute_layout_reader)
+                                })
+                                .await?;
+
+                            let new_state = match split_aligned_row_range(
+                                byte_range,
+                                total_size,
+                                split_ranges.as_ref(),
+                            ) {
+                                Some(row_range) => State::PrepareScan {
+                                    state,
+                                    vxf,
+                                    layout_reader,
+                                    row_range: Some(row_range),
+                                },
+                                None => State::Done,
+                            };
+
+                            Ok(Box::new(VortexMorselPlanner { state: new_state })
+                                as Box<dyn MorselPlanner>)
+                        }
+                        .in_current_span(),
+                    ),
+                ))
+            }
+            State::PrepareScan {
+                state,
+                vxf,
+                layout_reader,
+                row_range,
+            } => {
                 let FileOpenState {
                     file,
                     output_schema,
@@ -538,8 +636,6 @@ impl MorselPlanner for VortexMorselPlanner {
                     expr_adapter_factory,
                     unified_file_schema,
                     limit,
-                    layout_readers,
-                    natural_split_ranges,
                     has_output_ordering,
                     expr_convertor,
                     projection_pushdown,
@@ -618,36 +714,6 @@ impl MorselPlanner for VortexMorselPlanner {
                     .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;
                 let projector = leftover_projection.make_projector(&stream_schema)?;
 
-                // We share our layout readers with others partitions in the scan, so we can only need to read each layout in each file once.
-                let layout_reader = match layout_readers.entry(file.object_meta.location.clone()) {
-                    Entry::Occupied(mut occupied_entry) => {
-                        if let Some(reader) = occupied_entry.get().upgrade() {
-                            tracing::trace!("reusing layout reader for {}", occupied_entry.key());
-                            reader
-                        } else {
-                            tracing::trace!("creating layout reader for {}", occupied_entry.key());
-                            let reader = vxf.layout_reader().map_err(|e| {
-                                DataFusionError::Execution(format!(
-                                    "Failed to create layout reader: {e}"
-                                ))
-                            })?;
-                            occupied_entry.insert(Arc::downgrade(&reader));
-                            reader
-                        }
-                    }
-                    Entry::Vacant(vacant_entry) => {
-                        tracing::trace!("creating layout reader for {}", vacant_entry.key());
-                        let reader = vxf.layout_reader().map_err(|e| {
-                            DataFusionError::Execution(format!(
-                                "Failed to create layout reader: {e}"
-                            ))
-                        })?;
-                        vacant_entry.insert(Arc::downgrade(&reader));
-
-                        reader
-                    }
-                };
-
                 let mut scan_builder =
                     ScanBuilder::new(session.clone(), Arc::clone(&layout_reader));
 
@@ -655,34 +721,8 @@ impl MorselPlanner for VortexMorselPlanner {
                     scan_builder = vortex_plan.apply_to_builder(scan_builder);
                 }
 
-                if let Some(file_range) = file.range {
-                    let byte_range = Range {
-                        start: u64::try_from(file_range.start).map_err(|_| {
-                            exec_datafusion_err!("Vortex file range start is negative")
-                        })?,
-                        end: u64::try_from(file_range.end).map_err(|_| {
-                            exec_datafusion_err!("Vortex file range end is negative")
-                        })?,
-                    };
-                    if byte_range.start != 0 || byte_range.end != file.object_meta.size {
-                        // Full-file scans already cover every natural split. Only translate the
-                        // byte range back into row boundaries when DataFusion has trimmed the file.
-                        let natural_split_ranges = natural_split_ranges_for_file(
-                            natural_split_ranges.as_ref(),
-                            &file.object_meta.location,
-                            &layout_reader,
-                        )?;
-
-                        let Some(row_range) = split_aligned_row_range(
-                            byte_range,
-                            file.object_meta.size,
-                            natural_split_ranges.as_ref(),
-                        ) else {
-                            return Ok(None);
-                        };
-
-                        scan_builder = scan_builder.with_row_range(row_range);
-                    }
+                if let Some(row_range) = row_range {
+                    scan_builder = scan_builder.with_row_range(row_range);
                 }
 
                 let filter = filter
@@ -779,7 +819,7 @@ impl MorselPlanner for VortexMorselPlanner {
                         })
                     })
                     .map_err(move |e: VortexError| vortex_file_read_error(&file_location, e))
-                    .map(move |batch| {
+                    .map(move |batch| -> DFResult<RecordBatch> {
                         let batch = if projector.projection().as_ref().is_empty() {
                             batch
                         } else {
@@ -814,27 +854,76 @@ impl MorselPlanner for VortexMorselPlanner {
     }
 }
 
-fn natural_split_ranges_for_file(
-    natural_split_ranges: &DashMap<Path, Arc<[Range<u64>]>>,
+fn layout_reader_for_file(
+    layout_readers: &DashMap<Path, Weak<dyn LayoutReader>>,
     path: &Path,
-    layout_reader: &Arc<dyn LayoutReader>,
-) -> DFResult<Arc<[Range<u64>]>> {
-    if let Some(split_ranges) = natural_split_ranges.get(path) {
-        return Ok(Arc::clone(split_ranges.value()));
+    vxf: &VortexFile,
+) -> DFResult<Arc<dyn LayoutReader>> {
+    match layout_readers.entry(path.clone()) {
+        Entry::Occupied(mut entry) => {
+            if let Some(reader) = entry.get().upgrade() {
+                tracing::trace!("reusing layout reader for {}", entry.key());
+                Ok(reader)
+            } else {
+                tracing::trace!("creating layout reader for {}", entry.key());
+                let reader = vxf.layout_reader().map_err(|error| {
+                    DataFusionError::Execution(format!("Failed to create layout reader: {error}"))
+                })?;
+                entry.insert(Arc::downgrade(&reader));
+                Ok(reader)
+            }
+        }
+        Entry::Vacant(entry) => {
+            tracing::trace!("creating layout reader for {}", entry.key());
+            let reader = vxf.layout_reader().map_err(|error| {
+                DataFusionError::Execution(format!("Failed to create layout reader: {error}"))
+            })?;
+            entry.insert(Arc::downgrade(&reader));
+            Ok(reader)
+        }
     }
+}
+
+fn partial_file_byte_range(file: &PartitionedFile) -> DFResult<Option<Range<u64>>> {
+    let Some(file_range) = file.range.as_ref() else {
+        return Ok(None);
+    };
 
-    let split_ranges = compute_natural_split_ranges(layout_reader.as_ref())?;
+    let byte_range = Range {
+        start: u64::try_from(file_range.start)
+            .map_err(|_| exec_datafusion_err!("Vortex file range start is negative"))?,
+        end: u64::try_from(file_range.end)
+            .map_err(|_| exec_datafusion_err!("Vortex file range end is negative"))?,
+    };
+
+    Ok((byte_range.start != 0 || byte_range.end != file.object_meta.size).then_some(byte_range))
+}
 
+fn natural_split_cell_for_file(
+    natural_split_ranges: &NaturalSplitCache,
+    path: &Path,
+) -> Arc<OnceCell<NaturalSplitRanges>> {
     match natural_split_ranges.entry(path.clone()) {
-        Entry::Occupied(entry) => Ok(Arc::clone(entry.get())),
+        Entry::Occupied(entry) => Arc::clone(entry.get()),
         Entry::Vacant(entry) => {
+            let split_ranges = Arc::new(OnceCell::new());
             entry.insert(Arc::clone(&split_ranges));
-            Ok(split_ranges)
+            split_ranges
         }
     }
 }
 
-fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<Arc<[Range<u64>]>> {
+async fn compute_natural_split_ranges_async(
+    layout_reader: Arc<dyn LayoutReader>,
+) -> DFResult<NaturalSplitRanges> {
+    tokio::task::spawn_blocking(move || compute_natural_split_ranges(layout_reader.as_ref()))
+        .await
+        .map_err(|error| {
+            exec_datafusion_err!("Vortex natural split calculation task failed: {error}")
+        })?
+}
+
+fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<NaturalSplitRanges> {
     let row_count = layout_reader.row_count();
     let row_range = 0..row_count;
     let split_points: Vec<_> = SplitBy::Layout
@@ -909,3 +998,6 @@ fn vortex_file_read_error(path: &Path, error: VortexError) -> DataFusionError {
         error.with_context(format!("Failed to read Vortex file: {path}")),
     ))
 }
+
+#[cfg(test)]
+mod tests;
diff --git a/vortex-datafusion/src/persistent/morsel/tests.rs b/vortex-datafusion/src/persistent/morsel/tests.rs
new file mode 100644
index 00000000000..0f533579fb4
--- /dev/null
+++ b/vortex-datafusion/src/persistent/morsel/tests.rs
@@ -0,0 +1,1239 @@
+use std::collections::VecDeque;
+use std::fmt;
+use std::sync::Arc;
+use std::sync::LazyLock;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+
+use arrow_schema::Field;
+use arrow_schema::Fields;
+use arrow_schema::SchemaRef;
+use datafusion::arrow::array::DictionaryArray;
+use datafusion::arrow::array::Int32Array;
+use datafusion::arrow::array::RecordBatch;
+use datafusion::arrow::array::StringArray;
+use datafusion::arrow::array::StructArray;
+use datafusion::arrow::datatypes::DataType;
+use datafusion::arrow::datatypes::Schema;
+use datafusion::arrow::datatypes::UInt32Type;
+use datafusion::arrow::util::display::FormatOptions;
+use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
+use datafusion::common::record_batch;
+use datafusion::logical_expr::col;
+use datafusion::logical_expr::lit;
+use datafusion::physical_expr::planner::logical2physical;
+use datafusion::physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
+use datafusion::scalar::ScalarValue;
+use datafusion_common::stats::Precision;
+use datafusion_execution::cache::DefaultFilesMetadataCache;
+use datafusion_expr::Operator;
+use datafusion_physical_expr::PhysicalExpr;
+use datafusion_physical_expr::expressions as df_expr;
+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
+use datafusion_physical_expr::projection::ProjectionExpr;
+use futures::FutureExt;
+use futures::TryStreamExt;
+use futures::future::BoxFuture;
+use futures::stream;
+use futures::stream::BoxStream;
+use insta::assert_snapshot;
+use itertools::Itertools;
+use object_store::ObjectStore;
+use object_store::memory::InMemory;
+use rstest::rstest;
+use vortex::VortexSessionDefault;
+use vortex::array::ArrayRef;
+use vortex::buffer::Buffer;
+use vortex::file::WriteOptionsSessionExt;
+use vortex::io::VortexWrite;
+use vortex::io::object_store::ObjectStoreWrite;
+use vortex::metrics::DefaultMetricsRegistry;
+use vortex::scan::selection::Selection;
+use vortex::session::VortexSession;
+use vortex_arrow::FromArrowArray;
+
+use super::*;
+use crate::VortexAccessPlan;
+use crate::convert::exprs::DefaultExpressionConvertor;
+use crate::persistent::reader::DefaultVortexReaderFactory;
+
+static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::default);
+
+type TestOpenFuture = BoxFuture<'static, DFResult<BoxStream<'static, DFResult<RecordBatch>>>>;
+
+trait TestMorselizerExt {
+    fn open(&self, file: PartitionedFile) -> DFResult<TestOpenFuture>;
+}
+
+impl TestMorselizerExt for VortexMorselizer {
+    fn open(&self, file: PartitionedFile) -> DFResult<TestOpenFuture> {
+        let planner = self.plan_file(file)?;
+        Ok(read_morsels(planner).boxed())
+    }
+}
+
+async fn read_morsels(
+    planner: Box<dyn MorselPlanner>,
+) -> DFResult<BoxStream<'static, DFResult<RecordBatch>>> {
+    let mut planners = VecDeque::from([planner]);
+    let mut batches = Vec::new();
+
+    while let Some(planner) = planners.pop_front() {
+        let Some(mut plan) = planner.plan()? else {
+            continue;
+        };
+
+        for morsel in plan.take_morsels() {
+            batches.extend(morsel.into_stream().try_collect::<Vec<_>>().await?);
+        }
+        planners.extend(plan.take_ready_planners());
+        if let Some(pending_planner) = plan.take_pending_planner() {
+            planners.push_back(pending_planner.await?);
+        }
+    }
+
+    Ok(stream::iter(batches.into_iter().map(Ok)).boxed())
+}
+
+/// Test-only expr used to test error reporting.
+#[derive(Debug, Eq, Hash, PartialEq)]
+struct SnapshotErrorExpr;
+
+impl fmt::Display for SnapshotErrorExpr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "snapshot_error")
+    }
+}
+
+impl PhysicalExpr for SnapshotErrorExpr {
+    fn data_type(&self, _input_schema: &Schema) -> DFResult<DataType> {
+        Ok(DataType::Boolean)
+    }
+
+    fn nullable(&self, _input_schema: &Schema) -> DFResult<bool> {
+        Ok(false)
+    }
+
+    fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+
+    fn evaluate(&self, _batch: &RecordBatch) -> DFResult<datafusion_expr::ColumnarValue> {
+        Err(DataFusionError::Internal(
+            "intentional snapshot error".to_owned(),
+        ))
+    }
+
+    fn children(&self) -> Vec<&PhysicalExprRef> {
+        Vec::new()
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<PhysicalExprRef>,
+    ) -> DFResult<PhysicalExprRef> {
+        assert!(children.is_empty());
+        Ok(self)
+    }
+
+    fn snapshot(&self) -> DFResult<Option<PhysicalExprRef>> {
+        Err(DataFusionError::Internal(
+            "intentional snapshot error".to_owned(),
+        ))
+    }
+}
+
+#[rstest]
+#[case(0..3, 10, vec![0..2, 2..5, 5..10], Some(0..2))]
+#[case(3..7, 10, vec![0..2, 2..5, 5..10], Some(2..5))]
+#[case(1..8, 10, vec![0..1, 1..9, 9..10], Some(1..9))]
+#[case(1..4, 16, vec![0..1, 1..2, 2..3, 3..4], None)]
+#[case(0..1, 10, vec![0..2, 2..10], Some(0..2))]
+fn test_split_aligned_row_range(
+    #[case] byte_range: Range<u64>,
+    #[case] total_size: u64,
+    #[case] split_ranges: Vec<Range<u64>>,
+    #[case] expected: Option<Range<u64>>,
+) {
+    assert_eq!(
+        split_aligned_row_range(byte_range, total_size, &split_ranges),
+        expected
+    );
+}
+
+#[test]
+fn test_split_aligned_ranges_cover_splits_exactly_once() {
+    let split_ranges = vec![0..1, 1..4, 4..10, 10..13];
+    let byte_ranges = [0..4, 4..8, 8..12, 12..16];
+
+    let assigned = byte_ranges
+        .into_iter()
+        .filter_map(|byte_range| split_aligned_row_range(byte_range, 16, &split_ranges))
+        .collect::<Vec<_>>();
+
+    assert_eq!(assigned, vec![0..4, 4..10, 10..13]);
+    assert_eq!(
+        assigned
+            .iter()
+            .map(|range| range.end - range.start)
+            .sum::<u64>(),
+        13
+    );
+
+    let split_starts = split_ranges
+        .iter()
+        .map(|range| range.start)
+        .collect::<Vec<_>>();
+    let split_ends = split_ranges
+        .iter()
+        .map(|range| range.end)
+        .collect::<Vec<_>>();
+
+    for range in &assigned {
+        assert!(split_starts.contains(&range.start));
+        assert!(split_ends.contains(&range.end));
+    }
+
+    for (left, right) in assigned.iter().tuple_windows() {
+        assert_eq!(left.end, right.start);
+    }
+}
+
+#[tokio::test]
+async fn test_natural_split_cell_is_shared_by_file_path() {
+    let cache = NaturalSplitCache::default();
+    let path = Path::from("shared.vortex");
+    let first = natural_split_cell_for_file(&cache, &path);
+    let second = natural_split_cell_for_file(&cache, &path);
+    assert!(Arc::ptr_eq(&first, &second));
+
+    let initializations = Arc::new(AtomicUsize::new(0));
+    let first_initializations = Arc::clone(&initializations);
+    let second_initializations = Arc::clone(&initializations);
+    let first_cell = Arc::clone(&first);
+    let second_cell = Arc::clone(&second);
+
+    let (first_ranges, second_ranges) = tokio::join!(
+        first_cell.get_or_init(|| async move {
+            first_initializations.fetch_add(1, Ordering::Relaxed);
+            tokio::task::yield_now().await;
+            Arc::from([0..5, 5..10])
+        }),
+        second_cell.get_or_init(|| async move {
+            second_initializations.fetch_add(1, Ordering::Relaxed);
+            Arc::from([10..15, 15..20])
+        }),
+    );
+
+    assert_eq!(initializations.load(Ordering::Relaxed), 1);
+    assert!(Arc::ptr_eq(first_ranges, second_ranges));
+    assert_eq!(first_ranges.as_ref(), [0..5, 5..10]);
+}
+
+async fn write_arrow_to_vortex(
+    object_store: Arc<dyn ObjectStore>,
+    path: &str,
+    rb: RecordBatch,
+) -> anyhow::Result<u64> {
+    let array = ArrayRef::from_arrow(rb, false)?;
+    let path = Path::parse(path)?;
+
+    let mut write = ObjectStoreWrite::new(object_store, &path).await?;
+    let summary = SESSION
+        .write_options()
+        .write(&mut write, array.to_array_stream())
+        .await?;
+    write.shutdown().await?;
+
+    Ok(summary.size())
+}
+
+fn make_morselizer(
+    object_store: Arc<dyn ObjectStore>,
+    table_schema: TableSchema,
+    filter: Option<PhysicalExprRef>,
+) -> VortexMorselizer {
+    VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
+        projection: ProjectionExprs::from_indices(&[0], table_schema.file_schema()),
+        filter,
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema,
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    }
+}
+
+#[tokio::test]
+async fn test_open() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "part=1/file.vortex";
+    let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let file_schema = batch.schema();
+    let mut file = PartitionedFile::new(file_path.to_string(), data_size);
+    file.partition_values = vec![ScalarValue::Int32(Some(1))];
+
+    let table_schema = TableSchema::new(
+        Arc::clone(&file_schema),
+        vec![Arc::new(Field::new("part", DataType::Int32, false))],
+    );
+
+    // filter matches partition value
+    let filter = col("part").eq(lit(1));
+    let filter = logical2physical(&filter, table_schema.table_schema());
+
+    let morselizer = make_morselizer(
+        Arc::clone(&object_store),
+        table_schema.clone(),
+        Some(filter),
+    );
+    let stream = morselizer.open(file.clone()).unwrap().await.unwrap();
+
+    let data = stream.try_collect::<Vec<_>>().await?;
+    let num_batches = data.len();
+    let num_rows = data.iter().map(|rb| rb.num_rows()).sum::<usize>();
+
+    assert_eq!((num_batches, num_rows), (1, 3));
+
+    // filter doesn't matches partition value
+    let filter = col("part").eq(lit(2));
+    let filter = logical2physical(&filter, table_schema.table_schema());
+
+    let morselizer = make_morselizer(
+        Arc::clone(&object_store),
+        table_schema.clone(),
+        Some(filter),
+    );
+    let stream = morselizer.open(file.clone()).unwrap().await.unwrap();
+
+    let data = stream.try_collect::<Vec<_>>().await?;
+    let num_batches = data.len();
+    let num_rows = data.iter().map(|rb| rb.num_rows()).sum::<usize>();
+    assert_eq!((num_batches, num_rows), (0, 0));
+
+    Ok(())
+}
+#[tokio::test]
+async fn test_open_preserves_declared_schema_metadata() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "part=1/file.vortex";
+    let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)]))?;
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let file_schema = Arc::new(
+        batch.schema().as_ref().clone().with_metadata(
+            [("table".to_string(), "metadata".to_string())]
+                .into_iter()
+                .collect(),
+        ),
+    );
+    let table_schema = TableSchema::new(
+        file_schema,
+        vec![Arc::new(
+            Field::new("part", DataType::Int32, false).with_metadata(
+                [("partition".to_string(), "metadata".to_string())]
+                    .into_iter()
+                    .collect(),
+            ),
+        )],
+    );
+    let projection = ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
+    let expected_schema = Arc::new(projection.project_schema(table_schema.table_schema())?);
+
+    assert_eq!(
+        expected_schema.metadata().get("table"),
+        Some(&"metadata".to_string())
+    );
+    assert_eq!(
+        expected_schema.field(1).metadata().get("partition"),
+        Some(&"metadata".to_string())
+    );
+
+    for projection_pushdown in [false, true] {
+        let mut morselizer = make_morselizer(Arc::clone(&object_store), table_schema.clone(), None);
+        morselizer.projection = projection.clone();
+        morselizer.projection_pushdown = projection_pushdown;
+
+        let mut file = PartitionedFile::new(file_path.to_string(), data_size);
+        file.partition_values = vec![ScalarValue::Int32(Some(1))];
+        let batches = morselizer
+            .open(file)?
+            .await?
+            .try_collect::<Vec<_>>()
+            .await?;
+
+        assert!(!batches.is_empty());
+        for batch in batches {
+            assert_eq!(batch.schema().as_ref(), expected_schema.as_ref());
+        }
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_open_all_valid_nullable_columns_with_nonnullable_table_schema() -> anyhow::Result<()>
+{
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "nullable/file.vortex";
+    let batch = RecordBatch::try_new(
+        Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),
+        vec![Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)]))],
+    )?;
+    let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
+
+    let expected_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+    let table_schema = TableSchema::from_file_schema(Arc::clone(&expected_schema));
+
+    for projection_pushdown in [false, true] {
+        let mut morselizer = make_morselizer(Arc::clone(&object_store), table_schema.clone(), None);
+        morselizer.projection_pushdown = projection_pushdown;
+
+        let file = PartitionedFile::new(file_path.to_string(), data_size);
+        let batches = morselizer
+            .open(file)?
+            .await?
+            .try_collect::<Vec<_>>()
+            .await?;
+
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].schema().as_ref(), expected_schema.as_ref());
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_pruning_replaces_partition_columns_without_file_statistics() -> anyhow::Result<()>
+{
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+    let table_schema = TableSchema::new(
+        Arc::clone(&file_schema),
+        vec![Arc::new(Field::new("part", DataType::Int32, false))],
+    );
+
+    let partition_column = Arc::new(df_expr::Column::new("part", 1)) as PhysicalExprRef;
+    let predicate = Arc::new(df_expr::BinaryExpr::new(
+        Arc::clone(&partition_column),
+        Operator::Gt,
+        df_expr::lit(ScalarValue::Int32(Some(1))),
+    )) as PhysicalExprRef;
+    let dynamic_predicate = Arc::new(DynamicFilterPhysicalExpr::new(
+        vec![partition_column],
+        predicate,
+    )) as PhysicalExprRef;
+
+    let mut morselizer = make_morselizer(object_store, table_schema, None);
+    morselizer.file_pruning_predicate = Some(dynamic_predicate);
+    let df_metrics = morselizer.df_metrics.clone();
+
+    // The file does not exist and has no statistics. Replacing `part` with 1
+    // makes the predicate false, so pruning must happen before any file I/O.
+    let mut file = PartitionedFile::new("missing.vortex", 1);
+    file.partition_values = vec![ScalarValue::Int32(Some(1))];
+    let batches = morselizer
+        .open(file)?
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    assert!(batches.is_empty());
+    assert_eq!(
+        df_metrics
+            .clone_inner()
+            .sum_by_name("num_predicate_creation_errors")
+            .map(|metric| metric.as_usize()),
+        Some(0)
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_pruning_creation_errors_are_reported() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "metrics/file.vortex";
+    let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+    let mut statistics = Statistics::new_unknown(batch.schema().as_ref());
+    statistics.column_statistics[0].null_count = Precision::Exact(0);
+    let file = PartitionedFile::new(file_path, data_size).with_statistics(Arc::new(statistics));
+
+    let mut morselizer = make_morselizer(
+        object_store,
+        TableSchema::from_file_schema(batch.schema()),
+        None,
+    );
+    morselizer.file_pruning_predicate = Some(Arc::new(SnapshotErrorExpr));
+    let df_metrics = morselizer.df_metrics.clone();
+
+    let batches = morselizer
+        .open(file)?
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
+    assert_eq!(
+        df_metrics
+            .clone_inner()
+            .sum_by_name("num_predicate_creation_errors")
+            .map(|metric| metric.as_usize()),
+        Some(1)
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_open_applies_limit_after_filtering() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "filtered-limit/file.vortex";
+    let batch = record_batch!((
+        "a",
+        Int32,
+        vec![Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)]
+    ))
+    .unwrap();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+    let file = PartitionedFile::new(file_path.to_string(), data_size);
+    let table_schema = TableSchema::from_file_schema(batch.schema());
+    // `a > 3` excludes the first three rows, so a limit applied *before* filtering would take
+    // rows [1, 2, 3] and filter them all out (yielding nothing), whereas a limit applied
+    // *after* filtering yields the first three matching rows [4, 5, 6]. Asserting the values
+    // (not just the count) is what makes this test able to detect a pre-filter regression.
+    let filter = logical2physical(&col("a").gt(lit(3_i32)), table_schema.table_schema());
+
+    let mut morselizer = make_morselizer(object_store, table_schema, Some(filter));
+    morselizer.limit = Some(3);
+
+    let batches = morselizer
+        .open(file)?
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+    let values = batches
+        .iter()
+        .flat_map(|batch| {
+            batch
+                .column(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .expect("projected column should be Int32")
+                .values()
+                .to_vec()
+        })
+        .collect::<Vec<i32>>();
+
+    assert_eq!(values, [4, 5, 6]);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_open_empty_file() -> anyhow::Result<()> {
+    use futures::TryStreamExt;
+
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let data_batch = record_batch!(("a", Int32, Vec::<i32>::new())).unwrap();
+    let file_path = "part=1/empty.vortex";
+    let file_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, data_batch.clone()).await?;
+
+    let file_schema = data_batch.schema();
+    // Parallel scans may attach a byte range even for empty files; the
+    // morselizer must return early before attempting split-aligned translation.
+    let file =
+        PartitionedFile::new_with_range(file_path.to_string(), file_size, 0, file_size as i64);
+
+    let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));
+
+    let morselizer = make_morselizer(object_store, table_schema, None);
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+
+    assert_eq!(data.len(), 0);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_open_populates_file_metadata_cache() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "cached/file.vortex";
+    let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let file = PartitionedFile::new(file_path.to_string(), data_size);
+    let table_schema = TableSchema::from_file_schema(batch.schema());
+
+    let cache: Arc<dyn FileMetadataCache> =
+        Arc::new(DefaultFilesMetadataCache::new(64 * 1024 * 1024));
+    let mut morselizer = make_morselizer(Arc::clone(&object_store), table_schema, None);
+    morselizer.file_metadata_cache = Some(Arc::clone(&cache));
+
+    // The first open misses the cache and must write the parsed footer back.
+    let stream = morselizer.open(file.clone())?.await?;
+    stream.try_collect::<Vec<_>>().await?;
+
+    let entry = cache
+        .get(file.path())
+        .ok_or_else(|| anyhow::anyhow!("footer was not cached after open"))?;
+    assert!(entry.is_valid_for(&file.object_meta));
+    assert!(
+        entry
+            .file_metadata
+            .as_any()
+            .downcast_ref::<CachedVortexMetadata>()
+            .is_some()
+    );
+
+    // The second open hits the cache and still returns the same data.
+    let stream = morselizer.open(file.clone())?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+    assert_eq!(data.iter().map(|rb| rb.num_rows()).sum::<usize>(), 3);
+
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn test_open_files_different_table_schema() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+
+    let file1 = {
+        let file1_path = "/path/file1.vortex";
+        let batch1 = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
+        let data_size1 =
+            write_arrow_to_vortex(Arc::clone(&object_store), file1_path, batch1).await?;
+        PartitionedFile::new(file1_path.to_string(), data_size1)
+    };
+
+    let file2 = {
+        let file2_path = "/path/file2.vortex";
+        let batch2 = record_batch!(("a", Int16, vec![Some(-1), Some(-2), Some(-3)])).unwrap();
+        let data_size2 =
+            write_arrow_to_vortex(Arc::clone(&object_store), file2_path, batch2).await?;
+        PartitionedFile::new(file2_path.to_string(), data_size2)
+    };
+
+    // Table schema has can accommodate both files
+    let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new(
+        "a",
+        DataType::Int32,
+        true,
+    )])));
+
+    let make_morselizer = |filter| VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(&object_store))),
+        projection: ProjectionExprs::from_indices(&[0], table_schema.file_schema()),
+        filter: Some(filter),
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema: table_schema.clone(),
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    };
+
+    let filter = col("a").lt(lit(100_i32));
+    let filter = logical2physical(&filter, table_schema.table_schema());
+
+    let morselizer1 = make_morselizer(Arc::clone(&filter));
+    let stream = morselizer1.open(file1)?.await?;
+
+    let format_opts = FormatOptions::new().with_types_info(true);
+
+    let data = stream.try_collect::<Vec<_>>().await?;
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+
+    | a     |
+    | Int32 |
+    +-------+
+    | 1     |
+    | 2     |
+    | 3     |
+    +-------+
+    ");
+
+    let morselizer2 = make_morselizer(Arc::clone(&filter));
+    let stream = morselizer2.open(file2)?.await?;
+
+    let data = stream.try_collect::<Vec<_>>().await?;
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+
+    | a     |
+    | Int32 |
+    +-------+
+    | -1    |
+    | -2    |
+    | -3    |
+    +-------+
+    ");
+
+    Ok(())
+}
+
+#[tokio::test]
+// This test verifies that files with different column order than the
+// table schema can be opened without errors. The fix ensures that the
+// schema mapper is only used for type casting, not for reordering,
+// since the vortex projection already handles reordering.
+async fn test_schema_different_column_order() -> anyhow::Result<()> {
+    use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
+
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    // File has columns in order: c, b, a
+    let batch = record_batch!(
+        ("c", Int32, vec![Some(300), Some(301), Some(302)]),
+        ("b", Int32, vec![Some(200), Some(201), Some(202)]),
+        ("a", Int32, vec![Some(100), Some(101), Some(102)])
+    )
+    .unwrap();
+    let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
+    let file = PartitionedFile::new(file_path.to_string(), data_size);
+
+    // Table schema has columns in different order: a, b, c
+    let table_schema = Arc::new(Schema::new(vec![
+        Field::new("a", DataType::Int32, true),
+        Field::new("b", DataType::Int32, true),
+        Field::new("c", DataType::Int32, true),
+    ]));
+
+    let morselizer = VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
+        projection: ProjectionExprs::from_indices(&[0, 1, 2], &table_schema),
+        filter: None,
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema: TableSchema::from_file_schema(Arc::clone(&table_schema)),
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    };
+
+    let stream = morselizer.open(file)?.await?;
+
+    let format_opts = FormatOptions::new().with_types_info(true);
+    let data = stream.try_collect::<Vec<_>>().await?;
+
+    // Verify the output has columns in table schema order (a, b, c)
+    // not file order (c, b, a)
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+-------+-------+
+    | a     | b     | c     |
+    | Int32 | Int32 | Int32 |
+    +-------+-------+-------+
+    | 100   | 200   | 300   |
+    | 101   | 201   | 301   |
+    | 102   | 202   | 302   |
+    +-------+-------+-------+
+    ");
+
+    Ok(())
+}
+
+#[tokio::test]
+// This test verifies that expression rewriting doesn't fail when there is
+// a nested schema mismatch between the physical file schema and logical
+// table schema.
+async fn test_adapter_logical_physical_struct_mismatch() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+    let file_struct_fields = Fields::from(vec![
+        Field::new("field1", DataType::Utf8, true),
+        Field::new("field2", DataType::Utf8, true),
+    ]);
+    let struct_array = StructArray::new(
+        file_struct_fields.clone(),
+        vec![
+            Arc::new(StringArray::from(vec!["value1", "value2", "value3"])),
+            Arc::new(StringArray::from(vec!["a", "b", "c"])),
+        ],
+        None,
+    );
+    let batch = RecordBatch::try_new(
+        Arc::new(Schema::new(vec![Field::new(
+            "my_struct",
+            DataType::Struct(file_struct_fields),
+            true,
+        )])),
+        vec![Arc::new(struct_array)],
+    )?;
+    let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
+
+    // Table schema has an extra utf8 field.
+    let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new(
+        "my_struct",
+        DataType::Struct(Fields::from(vec![
+            Field::new(
+                "field1",
+                DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+                true,
+            ),
+            Field::new(
+                "field2",
+                DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+                true,
+            ),
+            Field::new("field3", DataType::Utf8, true),
+        ])),
+        true,
+    )])));
+
+    let morselizer = make_morselizer(
+        Arc::clone(&object_store),
+        table_schema.clone(),
+        // expression references my_struct column which has different fields in each
+        // field.
+        Some(logical2physical(
+            &col("my_struct").is_not_null(),
+            table_schema.table_schema(),
+        )),
+    );
+
+    // The morselizer should be able to open the file with a filter on the
+    // struct column.
+    let data = morselizer
+        .open(PartitionedFile::new(file_path.to_string(), data_size))?
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    assert_eq!(data.len(), 1);
+    assert_eq!(data[0].num_rows(), 3);
+
+    Ok(())
+}
+
+#[tokio::test]
+// Minimal reproducing test for the schema projection bug.
+// Before the fix, this would fail with a cast error when the file schema
+// and table schema have different field orders and we project a subset of columns.
+async fn test_projection_bug_minimal_repro() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    // File has columns in order: a, b, c with simple types
+    let batch = record_batch!(
+        ("a", Int32, vec![Some(1)]),
+        ("b", Utf8, vec![Some("test")]),
+        ("c", Int32, vec![Some(2)])
+    )
+    .unwrap();
+    let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
+
+    // Table schema has columns in DIFFERENT order: c, a, b
+    // and different types that require casting (Utf8 -> Dictionary)
+    let table_schema = TableSchema::new(
+        Arc::new(Schema::new(vec![
+            Field::new("c", DataType::Int32, true),
+            Field::new("a", DataType::Int32, true),
+            Field::new(
+                "b",
+                DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+                true,
+            ),
+        ])),
+        vec![],
+    );
+
+    // Project columns [0, 2] from table schema, which should give us: c, b
+    // Before the fix, the schema adapter would get confused about which fields
+    // to select from the file, causing incorrect type mappings.
+    let projection = vec![0, 2];
+
+    let morselizer = VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(&object_store))),
+        projection: ProjectionExprs::from_indices(projection.as_ref(), table_schema.file_schema()),
+        filter: None,
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema: table_schema.clone(),
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    };
+
+    // This should succeed and return the correctly projected and cast data
+    let data = morselizer
+        .open(PartitionedFile::new(file_path.to_string(), data_size))?
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    // Verify the columns are in the right order and have the right values
+    use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
+    let format_opts = FormatOptions::new().with_types_info(true);
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+--------------------------+
+    | c     | b                        |
+    | Int32 | Dictionary(UInt32, Utf8) |
+    +-------+--------------------------+
+    | 2     | test                     |
+    +-------+--------------------------+
+    ");
+
+    Ok(())
+}
+
+fn make_test_batch_with_10_rows() -> RecordBatch {
+    record_batch!(
+        ("a", Int32, (0..=9).map(Some).collect::<Vec<_>>()),
+        (
+            "b",
+            Utf8,
+            (0..=9).map(|i| Some(format!("r{}", i))).collect::<Vec<_>>()
+        )
+    )
+    .unwrap()
+}
+
+fn make_test_morselizer(
+    object_store: Arc<dyn ObjectStore>,
+    schema: SchemaRef,
+    projection: ProjectionExprs,
+) -> VortexMorselizer {
+    VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
+        projection,
+        filter: None,
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema: TableSchema::from_file_schema(schema),
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    }
+}
+
+#[tokio::test]
+// Test that Selection::IncludeByIndex filters to specific row indices.
+async fn test_selection_include_by_index() -> anyhow::Result<()> {
+    use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
+    use vortex::buffer::Buffer;
+    use vortex::scan::selection::Selection;
+
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    let batch = make_test_batch_with_10_rows();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let schema = batch.schema();
+    let mut file = PartitionedFile::new(file_path.to_string(), data_size);
+    file.extensions.insert(
+        VortexAccessPlan::default().with_selection(Selection::IncludeByIndex(Buffer::from_iter(
+            vec![1, 3, 5, 7],
+        ))),
+    );
+
+    let morselizer = make_test_morselizer(
+        Arc::clone(&object_store),
+        Arc::clone(&schema),
+        ProjectionExprs::from_indices(&[0, 1], &schema),
+    );
+
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+    let format_opts = FormatOptions::new().with_types_info(true);
+
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+------+
+    | a     | b    |
+    | Int32 | Utf8 |
+    +-------+------+
+    | 1     | r1   |
+    | 3     | r3   |
+    | 5     | r5   |
+    | 7     | r7   |
+    +-------+------+
+    ");
+
+    Ok(())
+}
+
+#[tokio::test]
+// Test that Selection::ExcludeByIndex excludes specific row indices.
+async fn test_selection_exclude_by_index() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    let batch = make_test_batch_with_10_rows();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let schema = batch.schema();
+    let mut file = PartitionedFile::new(file_path.to_string(), data_size);
+    file.extensions.insert(
+        VortexAccessPlan::default().with_selection(Selection::ExcludeByIndex(Buffer::from_iter(
+            vec![0, 2, 4, 6, 8],
+        ))),
+    );
+
+    let morselizer = make_test_morselizer(
+        Arc::clone(&object_store),
+        Arc::clone(&schema),
+        ProjectionExprs::from_indices(&[0, 1], &schema),
+    );
+
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+    let format_opts = FormatOptions::new().with_types_info(true);
+
+    assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
+    +-------+------+
+    | a     | b    |
+    | Int32 | Utf8 |
+    +-------+------+
+    | 1     | r1   |
+    | 3     | r3   |
+    | 5     | r5   |
+    | 7     | r7   |
+    | 9     | r9   |
+    +-------+------+
+    ");
+
+    Ok(())
+}
+
+#[tokio::test]
+// Test that Selection::All returns all rows.
+async fn test_selection_all() -> anyhow::Result<()> {
+    use vortex::scan::selection::Selection;
+
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    let batch = make_test_batch_with_10_rows();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let schema = batch.schema();
+    let mut file = PartitionedFile::new(file_path.to_string(), data_size);
+    file.extensions
+        .insert(VortexAccessPlan::default().with_selection(Selection::All));
+
+    let morselizer = make_test_morselizer(
+        Arc::clone(&object_store),
+        Arc::clone(&schema),
+        ProjectionExprs::from_indices(&[0], &schema),
+    );
+
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+
+    let total_rows: usize = data.iter().map(|rb| rb.num_rows()).sum();
+    assert_eq!(total_rows, 10);
+
+    Ok(())
+}
+
+#[tokio::test]
+// Test that when no extensions are provided, all rows are returned (backward compatibility).
+async fn test_selection_no_extensions() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    let batch = make_test_batch_with_10_rows();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let schema = batch.schema();
+    let file = PartitionedFile::new(file_path.to_string(), data_size);
+    // file.extensions is None by default
+
+    let morselizer = make_test_morselizer(
+        Arc::clone(&object_store),
+        Arc::clone(&schema),
+        ProjectionExprs::from_indices(&[0], &schema),
+    );
+
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+
+    let total_rows: usize = data.iter().map(|rb| rb.num_rows()).sum();
+    assert_eq!(total_rows, 10);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_projection_expr_pushdown() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+    let file_path = "/path/file.vortex";
+
+    let batch = record_batch!(
+        ("a", Int32, vec![Some(1), Some(2), Some(3)]),
+        ("b", Int32, vec![Some(10), Some(20), Some(30)])
+    )
+    .unwrap();
+    let data_size =
+        write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
+
+    let file_schema = batch.schema();
+    let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));
+
+    // Create a projection that includes an arithmetic expression: a + b * 2
+    let col_a = df_expr::col("a", &file_schema)?;
+    let col_b = df_expr::col("b", &file_schema)?;
+    let two = df_expr::lit(ScalarValue::Int32(Some(2)));
+
+    // b * 2
+    let b_times_2 = df_expr::binary(col_b, Operator::Multiply, two, &file_schema)?;
+    // a + (b * 2)
+    let a_plus_b_times_2 = df_expr::binary(col_a, Operator::Plus, b_times_2, &file_schema)?;
+
+    let projection = ProjectionExprs::new(vec![ProjectionExpr::new(
+        a_plus_b_times_2,
+        "result".to_string(),
+    )]);
+
+    let morselizer = VortexMorselizer {
+        partition: 1,
+        session: SESSION.clone(),
+        vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(&object_store))),
+        projection,
+        filter: None,
+        file_pruning_predicate: None,
+        expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+        table_schema,
+        limit: None,
+        metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
+        df_metrics: ExecutionPlanMetricsSet::new(),
+        layout_readers: Default::default(),
+        natural_split_ranges: Default::default(),
+        has_output_ordering: false,
+        expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
+        file_metadata_cache: None,
+        projection_pushdown: false,
+        scan_concurrency: None,
+    };
+
+    let file = PartitionedFile::new(file_path.to_string(), data_size);
+    let stream = morselizer.open(file)?.await?;
+    let data = stream.try_collect::<Vec<_>>().await?;
+
+    // Expected: a + b * 2
+    // row 0: 1 + 10 * 2 = 21
+    // row 1: 2 + 20 * 2 = 42
+    // row 2: 3 + 30 * 2 = 63
+    assert_snapshot!(pretty_format_batches_with_options(&data, &FormatOptions::new().with_types_info(true))?.to_string(), @r"
+    +--------+
+    | result |
+    | Int32  |
+    +--------+
+    | 21     |
+    | 42     |
+    | 63     |
+    +--------+
+    ");
+
+    Ok(())
+}
+
+/// When a Struct contains Dictionary fields, writing to vortex and reading back
+/// should preserve the Dictionary type.
+#[tokio::test]
+async fn test_struct_with_dictionary_roundtrip() -> anyhow::Result<()> {
+    let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+
+    let struct_fields = Fields::from(vec![
+        Field::new_dictionary("a", DataType::UInt32, DataType::Utf8, true),
+        Field::new_dictionary("b", DataType::UInt32, DataType::Utf8, true),
+    ]);
+    let struct_array = StructArray::new(
+        struct_fields.clone(),
+        vec![
+            Arc::new(DictionaryArray::<UInt32Type>::from_iter(["x", "y", "x"])),
+            Arc::new(DictionaryArray::<UInt32Type>::from_iter(["p", "p", "q"])),
+        ],
+        None,
+    );
+
+    let schema = Arc::new(Schema::new(vec![Field::new(
+        "labels",
+        DataType::Struct(struct_fields.clone()),
+        false,
+    )]));
+    let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(struct_array)])?;
+
+    let file_path = "/test.vortex";
+    let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
+
+    let morselizer = make_test_morselizer(
+        Arc::clone(&object_store),
+        Arc::clone(&schema),
+        ProjectionExprs::from_indices(&[0], &schema),
+    );
+    let data: Vec<_> = morselizer
+        .open(PartitionedFile::new(file_path.to_string(), data_size))?
+        .await?
+        .try_collect()
+        .await?;
+
+    assert_eq!(
+        data[0].schema().field(0).data_type(),
+        &DataType::Struct(struct_fields),
+        "Struct(Dictionary) type should be preserved"
+    );
+    Ok(())
+}
diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs
deleted file mode 100644
index d720382a5e8..00000000000
--- a/vortex-datafusion/src/persistent/opener.rs
+++ /dev/null
@@ -1,1792 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright the Vortex contributors
-
-use std::ops::Range;
-use std::sync::Arc;
-use std::sync::Weak;
-
-use arrow_array::RecordBatchOptions;
-use arrow_schema::Field;
-use arrow_schema::Schema;
-use datafusion_common::DataFusionError;
-use datafusion_common::Result as DFResult;
-use datafusion_common::ScalarValue;
-use datafusion_common::Statistics;
-use datafusion_common::arrow::array::AsArray;
-use datafusion_common::arrow::array::RecordBatch;
-use datafusion_common::exec_datafusion_err;
-use datafusion_datasource::PartitionedFile;
-use datafusion_datasource::TableSchema;
-use datafusion_datasource::file_stream::FileOpenFuture;
-use datafusion_datasource::file_stream::FileOpener;
-use datafusion_execution::cache::cache_manager::CachedFileMetadataEntry;
-use datafusion_execution::cache::cache_manager::FileMetadataCache;
-use datafusion_physical_expr::PhysicalExprRef;
-use datafusion_physical_expr::projection::ProjectionExprs;
-use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;
-use datafusion_physical_expr::split_conjunction;
-use datafusion_physical_expr::utils::collect_columns;
-use datafusion_physical_expr::utils::reassign_expr_columns;
-use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
-use datafusion_physical_expr_adapter::replace_columns_with_literals;
-use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
-use datafusion_physical_plan::metrics::MetricBuilder;
-use datafusion_physical_plan::metrics::MetricCategory;
-use datafusion_pruning::FilePruner;
-use futures::FutureExt;
-use futures::StreamExt;
-use futures::TryStreamExt;
-use futures::stream;
-use object_store::path::Path;
-use tracing::Instrument;
-use vortex::array::VortexSessionExecute;
-use vortex::dtype::FieldMask;
-use vortex::error::VortexError;
-use vortex::error::VortexExpect;
-use vortex::file::OpenOptionsSessionExt;
-use vortex::io::InstrumentedReadAt;
-use vortex::layout::LayoutReader;
-use vortex::layout::scan::scan_builder::ScanBuilder;
-use vortex::layout::scan::split_by::SplitBy;
-use vortex::metrics::Label;
-use vortex::metrics::MetricsRegistry;
-use vortex::session::VortexSession;
-use vortex_arrow::ArrowSessionExt;
-use vortex_utils::aliases::dash_map::DashMap;
-use vortex_utils::aliases::dash_map::Entry;
-
-use crate::VortexAccessPlan;
-use crate::convert::exprs::ExpressionConvertor;
-use crate::convert::exprs::ProcessedProjection;
-use crate::convert::exprs::make_vortex_predicate;
-use crate::convert::schema::calculate_physical_schema;
-use crate::metrics::PARTITION_LABEL;
-use crate::metrics::PATH_LABEL;
-use crate::persistent::cache::CachedVortexMetadata;
-use crate::persistent::reader::VortexReaderFactory;
-use crate::persistent::stream::PrunableStream;
-
-#[derive(Clone)]
-pub(crate) struct VortexOpener {
-    /// The partition this opener is assigned to. Only used for labeling metrics.
-    pub partition: usize,
-    pub session: VortexSession,
-    pub vortex_reader_factory: Arc<dyn VortexReaderFactory>,
-    /// Optional table schema projection. The indices are w.r.t. the `table_schema`, which is
-    /// all fields in the final scan result not including the partition columns.
-    pub projection: ProjectionExprs,
-    /// Filter expression optimized for pushdown into Vortex scan operations.
-    /// This may be a subset of file_pruning_predicate containing only expressions
-    /// that Vortex can efficiently evaluate.
-    pub filter: Option<PhysicalExprRef>,
-    /// Filter expression used by DataFusion's FilePruner to eliminate files based on
-    /// statistics and partition values without opening them.
-    pub file_pruning_predicate: Option<PhysicalExprRef>,
-    pub expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
-    /// This is the table's schema without partition columns. It may contain fields which do
-    /// not exist in the file, and are supplied by the `schema_adapter_factory`.
-    pub table_schema: TableSchema,
-    /// If provided, the scan will not return more than this many rows.
-    pub limit: Option<u64>,
-    /// A metrics object for tracking performance of the scan.
-    pub metrics_registry: Arc<dyn MetricsRegistry>,
-    /// DataFusion-native metrics exposed through `DataSourceExec`.
-    pub df_metrics: ExecutionPlanMetricsSet,
-    /// A shared cache of file readers.
-    ///
-    /// To save on the overhead of reparsing FlatBuffers and rebuilding the layout tree, we cache
-    /// a file reader the first time we read a file.
-    pub layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
-    /// Shared full-file natural splits keyed by file path.
-    pub natural_splits: Arc<DashMap<Path, Arc<NaturalSplits>>>,
-    /// Whether the query has output ordering specified
-    pub has_output_ordering: bool,
-
-    pub expression_convertor: Arc<dyn ExpressionConvertor>,
-    pub file_metadata_cache: Option<Arc<dyn FileMetadataCache>>,
-    /// Whether to enable expression pushdown into the underlying Vortex scan.
-    pub projection_pushdown: bool,
-    pub scan_concurrency: Option<usize>,
-}
-
-impl FileOpener for VortexOpener {
-    fn open(&self, file: PartitionedFile) -> DFResult<FileOpenFuture> {
-        // Calculate the output schema before replacing partition columns with literals so it
-        // retains the table and partition-field metadata declared by the plan.
-        let output_schema = Arc::new(
-            self.projection
-                .project_schema(self.table_schema.table_schema())?,
-        );
-        let session = self.session.clone();
-        let metrics_registry = Arc::clone(&self.metrics_registry);
-        let labels = vec![
-            Label::new(PATH_LABEL, file.path().to_string()),
-            Label::new(PARTITION_LABEL, self.partition.to_string()),
-        ];
-
-        let mut projection = self.projection.clone();
-        let mut filter = self.filter.clone();
-
-        let reader = self.vortex_reader_factory.create_reader(&file, &session)?;
-
-        let reader =
-            InstrumentedReadAt::new_with_labels(reader, metrics_registry.as_ref(), labels.clone());
-
-        let mut file_pruning_predicate = self.file_pruning_predicate.clone();
-        let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);
-        let file_metadata_cache = self.file_metadata_cache.clone();
-
-        let unified_file_schema = Arc::clone(self.table_schema.file_schema());
-        let limit = self.limit;
-        let layout_readers = Arc::clone(&self.layout_readers);
-        let natural_splits = Arc::clone(&self.natural_splits);
-        let has_output_ordering = self.has_output_ordering;
-        let scan_concurrency = self.scan_concurrency;
-
-        let expr_convertor = Arc::clone(&self.expression_convertor);
-        let projection_pushdown = self.projection_pushdown;
-
-        let predicate_creation_errors = MetricBuilder::new(&self.df_metrics)
-            .with_category(MetricCategory::Rows)
-            .global_counter("num_predicate_creation_errors");
-
-        // Replace column access for partition columns with literals
-        #[expect(clippy::disallowed_types)]
-        let literal_value_cols = self
-            .table_schema
-            .table_partition_cols()
-            .iter()
-            .map(|f| f.name())
-            .cloned()
-            .zip(file.partition_values.clone())
-            .collect::<std::collections::HashMap<String, ScalarValue>>();
-
-        let predicate_uses_partition_columns =
-            file_pruning_predicate.as_ref().is_some_and(|predicate| {
-                collect_columns(predicate)
-                    .iter()
-                    .any(|column| literal_value_cols.contains_key(column.name()))
-            });
-
-        if !literal_value_cols.is_empty() {
-            projection = projection.try_map_exprs(|expr| {
-                replace_columns_with_literals(Arc::clone(&expr), &literal_value_cols)
-            })?;
-            filter = filter
-                .map(|p| replace_columns_with_literals(p, &literal_value_cols))
-                .transpose()?;
-            file_pruning_predicate = file_pruning_predicate
-                .map(|p| replace_columns_with_literals(p, &literal_value_cols))
-                .transpose()?;
-        }
-
-        // FilePruner requires a statistics object even when the rewritten predicate
-        // only contains partition literals. Supply unknown file-column statistics in
-        // that case so static and dynamic partition predicates can still prune.
-        let synthetic_statistics = (!file.has_statistics() && predicate_uses_partition_columns)
-            .then(|| {
-                file.clone()
-                    .with_statistics(Arc::new(Statistics::new_unknown(&unified_file_schema)))
-            });
-        let pruning_file = synthetic_statistics.as_ref().unwrap_or(&file);
-
-        let mut file_pruner = file_pruning_predicate
-            .filter(|_| file.has_statistics() || predicate_uses_partition_columns)
-            .and_then(|predicate| {
-                FilePruner::try_new(
-                    Arc::clone(&predicate),
-                    &unified_file_schema,
-                    pruning_file,
-                    predicate_creation_errors,
-                )
-            });
-
-        Ok(async move {
-            // Check if this file should be pruned based on statistics/partition values.
-            // Returns empty stream if file can be skipped entirely.
-            if let Some(file_pruner) = file_pruner.as_mut()
-                && file_pruner.should_prune()?
-            {
-                return Ok(stream::empty().boxed());
-            }
-
-            let mut open_opts = session
-                .open_options()
-                .with_file_size(file.object_meta.size)
-                .with_metrics_registry(Arc::clone(&metrics_registry))
-                .with_labels(labels);
-
-            let cached_footer = file_metadata_cache
-                .as_ref()
-                .and_then(|cache| cache.get(file.path()))
-                .filter(|entry| entry.is_valid_for(&file.object_meta))
-                .and_then(|entry| {
-                    entry
-                        .file_metadata
-                        .as_any()
-                        .downcast_ref::<CachedVortexMetadata>()
-                        .map(|vortex_metadata| vortex_metadata.footer().clone())
-                });
-            let footer_cache_hit = cached_footer.is_some();
-
-            if let Some(footer) = cached_footer {
-                open_opts = open_opts.with_footer(footer);
-            }
-
-            let vxf = open_opts
-                .open_read(reader)
-                .await
-                .map_err(|e| exec_datafusion_err!("Failed to open Vortex file {e}"))?;
-
-            // On a miss, cache the parsed footer so other partitions and later executions
-            // skip the footer fetch and parse. `infer_schema`/`infer_stats` also populate
-            // this cache, but only when planning goes through `VortexFormat`.
-            if !footer_cache_hit && let Some(cache) = &file_metadata_cache {
-                cache.put(
-                    file.path(),
-                    CachedFileMetadataEntry::new(
-                        file.object_meta.clone(),
-                        Arc::new(CachedVortexMetadata::new(&vxf)),
-                    ),
-                );
-            }
-
-            // Check if there are rows in this file. If not, we can save
-            // ourselves some work and return an empty stream.
-            if vxf.row_count() == 0 {
-                return Ok(stream::empty().boxed());
-            }
-
-            // This is the expected arrow types of the actual columns in the file, which might have different types
-            // from the unified logical schema or miss
-            let this_file_schema = Arc::new(calculate_physical_schema(
-                vxf.dtype(),
-                &unified_file_schema,
-                &session.arrow(),
-            )?);
-
-            let expr_adapter = expr_adapter_factory.create(
-                Arc::clone(&unified_file_schema),
-                Arc::clone(&this_file_schema),
-            )?;
-
-            let simplifier = PhysicalExprSimplifier::new(&this_file_schema);
-
-            // The adapter rewrites the expressions to the local file schema, allowing
-            // for schema evolution and divergence between the table's schema and individual files.
-            let filter = filter
-                .map(|filter| {
-                    // Expression might now reference columns that don't exist in the file, so we can give it
-                    // another simplification pass.
-                    simplifier.simplify(expr_adapter.rewrite(filter)?)
-                })
-                .transpose()?;
-            let projection =
-                projection.try_map_exprs(|p| simplifier.simplify(expr_adapter.rewrite(p)?))?;
-
-            let ProcessedProjection {
-                scan_projection,
-                leftover_projection,
-            } = if projection_pushdown {
-                expr_convertor.split_projection(
-                    projection.clone(),
-                    &this_file_schema,
-                    output_schema.as_ref(),
-                )?
-            } else {
-                // When projection pushdown is disabled, read only the required columns
-                // and apply the full projection after the scan.
-                expr_convertor.no_pushdown_projection(projection.clone(), &this_file_schema)?
-            };
-
-            // The schema of the stream returned from the vortex scan.
-            // We use a reference schema for types that don't roundtrip (Dictionary, Utf8, etc.).
-            let scan_dtype = scan_projection.return_dtype(vxf.dtype()).map_err(|_e| {
-                exec_datafusion_err!("Couldn't get the dtype for the underlying Vortex scan")
-            })?;
-
-            // When projection pushdown is enabled, the scan outputs the projected columns.
-            // When disabled, the scan outputs raw columns and the projection is applied after.
-            let scan_reference_schema = if projection_pushdown {
-                (*output_schema).clone()
-            } else {
-                // Build schema from the raw columns being read
-                let column_indices = projection.column_indices();
-                let fields: Vec<_> = column_indices
-                    .into_iter()
-                    .map(|idx| this_file_schema.field(idx).clone())
-                    .collect();
-                Schema::new_with_metadata(fields, this_file_schema.metadata().clone())
-            };
-            let stream_schema =
-                calculate_physical_schema(&scan_dtype, &scan_reference_schema, &session.arrow())?;
-
-            let leftover_projection = leftover_projection
-                .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;
-            let projector = leftover_projection.make_projector(&stream_schema)?;
-
-            // We share our layout readers with others partitions in the scan, so we can only need to read each layout in each file once.
-            let layout_reader = match layout_readers.entry(file.object_meta.location.clone()) {
-                Entry::Occupied(mut occupied_entry) => {
-                    if let Some(reader) = occupied_entry.get().upgrade() {
-                        tracing::trace!("reusing layout reader for {}", occupied_entry.key());
-                        reader
-                    } else {
-                        tracing::trace!("creating layout reader for {}", occupied_entry.key());
-                        let reader = vxf.layout_reader().map_err(|e| {
-                            DataFusionError::Execution(format!(
-                                "Failed to create layout reader: {e}"
-                            ))
-                        })?;
-                        occupied_entry.insert(Arc::downgrade(&reader));
-                        reader
-                    }
-                }
-                Entry::Vacant(vacant_entry) => {
-                    tracing::trace!("creating layout reader for {}", vacant_entry.key());
-                    let reader = vxf.layout_reader().map_err(|e| {
-                        DataFusionError::Execution(format!("Failed to create layout reader: {e}"))
-                    })?;
-                    vacant_entry.insert(Arc::downgrade(&reader));
-
-                    reader
-                }
-            };
-
-            let mut scan_builder = ScanBuilder::new(session.clone(), Arc::clone(&layout_reader));
-
-            if let Some(vortex_plan) = file.extensions.get::<VortexAccessPlan>() {
-                scan_builder = vortex_plan.apply_to_builder(scan_builder);
-            }
-
-            if let Some(file_range) = file.range {
-                let byte_range = Range {
-                    start: u64::try_from(file_range.start)
-                        .map_err(|_| exec_datafusion_err!("Vortex file range start is negative"))?,
-                    end: u64::try_from(file_range.end)
-                        .map_err(|_| exec_datafusion_err!("Vortex file range end is negative"))?,
-                };
-                if byte_range.start != 0 || byte_range.end != file.object_meta.size {
-                    // Full-file scans already cover every natural split. Only translate the
-                    // byte range back into row boundaries when DataFusion has trimmed the file.
-                    let natural_splits = natural_splits_for_file(
-                        natural_splits.as_ref(),
-                        &file.object_meta.location,
-                        &layout_reader,
-                        file.object_meta.size,
-                    )?;
-
-                    let Some(row_range) =
-                        split_aligned_row_range(byte_range, natural_splits.as_ref())
-                    else {
-                        return Ok(stream::empty().boxed());
-                    };
-
-                    scan_builder = scan_builder.with_row_range(row_range);
-                }
-            }
-
-            let filter = filter
-                .and_then(|f| {
-                    // Verify that all filters we've accepted from DataFusion get pushed down.
-                    // This will only fail if the user has not configured a suitable
-                    // PhysicalExprAdapterFactory on the file source to handle rewriting the
-                    // expression to handle missing/reordered columns in the Vortex file.
-                    let (pushed, unpushed): (Vec<PhysicalExprRef>, Vec<PhysicalExprRef>) =
-                        split_conjunction(&f)
-                            .into_iter()
-                            .cloned()
-                            .partition(|expr| {
-                                expr_convertor.can_be_pushed_down(expr, &this_file_schema)
-                            });
-
-                    if !unpushed.is_empty() {
-                        return Some(Err(exec_datafusion_err!(
-                            r#"VortexSource accepted but failed to push {} filters.
-                            This should never happen if you have a properly configured
-                            PhysicalExprAdapterFactory configured on the source.
-
-                            Failed filters:
-
-                            {unpushed:#?}
-                            "#,
-                            unpushed.len()
-                        )));
-                    }
-
-                    make_vortex_predicate(expr_convertor.as_ref(), &pushed).transpose()
-                })
-                .transpose()?;
-
-            if let Some(limit) = limit {
-                scan_builder = scan_builder.with_limit(limit);
-            }
-
-            if let Some(concurrency) = scan_concurrency {
-                scan_builder = scan_builder.with_concurrency(concurrency);
-            }
-
-            let stream_target_field = Field::new_struct("", stream_schema.fields().clone(), false);
-            let file_location = file.object_meta.location.clone();
-            let stream = scan_builder
-                .with_metrics_registry(metrics_registry)
-                .with_projection(scan_projection)
-                .with_some_filter(filter)
-                .with_ordered(has_output_ordering)
-                .into_stream()
-                .map_err(|e| exec_datafusion_err!("Failed to create Vortex stream: {e}"))?
-                // Convert to Arrow inline on the polling thread: DataFusion sources are expected
-                // to do their CPU work inside `poll_next`, and spawning this onto the blocking
-                // pool oversubscribes the CPU.
-                .map(move |chunk| {
-                    let mut ctx = session.create_execution_ctx();
-                    chunk.and_then(|chunk| {
-                        let arrow_session = ctx.session().clone();
-                        let arrow = arrow_session.arrow().execute_arrow(
-                            chunk,
-                            Some(&stream_target_field),
-                            &mut ctx,
-                        )?;
-                        Ok(RecordBatch::from(arrow.as_struct().clone()))
-                    })
-                })
-                .map_err(move |e: VortexError| vortex_file_read_error(&file_location, e))
-                .map(move |batch| -> DFResult<RecordBatch> {
-                    let batch = if projector.projection().as_ref().is_empty() {
-                        batch
-                    } else {
-                        batch.and_then(|b| projector.project_batch(&b))
-                    }?;
-
-                    let (_, columns, row_count) = batch.into_parts();
-                    RecordBatch::try_new_with_options(
-                        Arc::clone(&output_schema),
-                        columns,
-                        &RecordBatchOptions::new().with_row_count(Some(row_count)),
-                    )
-                    .map_err(Into::into)
-                })
-                .boxed();
-
-            if let Some(file_pruner) = file_pruner {
-                Ok(PrunableStream::new(file_pruner, stream).boxed())
-            } else {
-                Ok(stream)
-            }
-        }
-        .in_current_span()
-        .boxed())
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct NaturalSplits {
-    row_boundaries: Arc<[u64]>,
-    assignment_bytes: Box<[u64]>,
-}
-
-impl NaturalSplits {
-    fn new(row_boundaries: Arc<[u64]>, total_size: u64) -> Self {
-        let row_count = row_boundaries.last().copied().unwrap_or_default();
-        let assignment_bytes = if row_count == 0 {
-            Box::default()
-        } else {
-            row_boundaries
-                .windows(2)
-                .enumerate()
-                .map(|(idx, boundaries)| {
-                    split_assignment_byte(
-                        idx,
-                        &(boundaries[0]..boundaries[1]),
-                        row_count,
-                        total_size,
-                    )
-                })
-                .collect()
-        };
-
-        debug_assert!(assignment_bytes.is_sorted());
-        debug_assert_eq!(
-            assignment_bytes.len() + usize::from(!row_boundaries.is_empty()),
-            row_boundaries.len()
-        );
-
-        Self {
-            row_boundaries,
-            assignment_bytes,
-        }
-    }
-}
-
-fn natural_splits_for_file(
-    natural_splits: &DashMap<Path, Arc<NaturalSplits>>,
-    path: &Path,
-    layout_reader: &Arc<dyn LayoutReader>,
-    total_size: u64,
-) -> DFResult<Arc<NaturalSplits>> {
-    if let Some(splits) = natural_splits.get(path) {
-        return Ok(Arc::clone(splits.value()));
-    }
-
-    // Compute while holding the entry so concurrent partitions opening the same file wait
-    // for the winner instead of all walking the layout tree; the redundant walks contend on
-    // the lazily-initialized layout children and dominate the cost of the computation itself.
-    match natural_splits.entry(path.clone()) {
-        Entry::Occupied(entry) => Ok(Arc::clone(entry.get())),
-        Entry::Vacant(entry) => {
-            let splits = compute_natural_splits(layout_reader.as_ref(), total_size)?;
-            entry.insert(Arc::clone(&splits));
-            Ok(splits)
-        }
-    }
-}
-
-fn compute_natural_splits(
-    layout_reader: &dyn LayoutReader,
-    total_size: u64,
-) -> DFResult<Arc<NaturalSplits>> {
-    let row_count = layout_reader.row_count();
-    let row_range = 0..row_count;
-    let row_boundaries = SplitBy::Layout
-        .splits(layout_reader, &row_range, &[FieldMask::All])
-        .map_err(|e| exec_datafusion_err!("Failed to compute Vortex natural splits: {e}"))?;
-
-    Ok(Arc::new(NaturalSplits::new(
-        row_boundaries.into(),
-        total_size,
-    )))
-}
-
-/// Translate a DataFusion byte range to the contiguous natural split ranges it owns.
-/// Most splits are assigned by midpoint, but the leading split stays with the range that owns
-/// byte 0 so a tiny first byte range still claims the first rows.
-fn split_aligned_row_range(
-    byte_range: Range<u64>,
-    natural_splits: &NaturalSplits,
-) -> Option<Range<u64>> {
-    if byte_range.start >= byte_range.end {
-        return None;
-    }
-
-    let first_split = natural_splits
-        .assignment_bytes
-        .partition_point(|&assignment_byte| assignment_byte < byte_range.start);
-    let after_last_split = natural_splits
-        .assignment_bytes
-        .partition_point(|&assignment_byte| assignment_byte < byte_range.end);
-    if first_split == after_last_split {
-        return None;
-    }
-
-    Some(
-        natural_splits.row_boundaries[first_split]..natural_splits.row_boundaries[after_last_split],
-    )
-}
-
-fn split_assignment_byte(
-    idx: usize,
-    split_range: &Range<u64>,
-    row_count: u64,
-    total_size: u64,
-) -> u64 {
-    if idx == 0 && split_range.start == 0 {
-        // Byte 0 is the only stable representative for the leading split. A midpoint can fall
-        // into the next DataFusion byte range and leave the first range with no rows to read.
-        0
-    } else {
-        split_midpoint_to_byte(split_range, row_count, total_size)
-    }
-}
-
-fn split_midpoint_to_byte(split_range: &Range<u64>, row_count: u64, total_size: u64) -> u64 {
-    let midpoint_row = split_range.start + (split_range.end - split_range.start) / 2;
-    let midpoint_byte = (u128::from(midpoint_row) * u128::from(total_size)) / u128::from(row_count);
-
-    u64::try_from(midpoint_byte).vortex_expect("midpoint byte projection should fit into u64")
-}
-
-fn vortex_file_read_error(path: &Path, error: VortexError) -> DataFusionError {
-    DataFusionError::External(Box::new(
-        error.with_context(format!("Failed to read Vortex file: {path}")),
-    ))
-}
-
-#[cfg(test)]
-mod tests {
-    use std::fmt;
-    use std::sync::Arc;
-    use std::sync::LazyLock;
-
-    use arrow_schema::Field;
-    use arrow_schema::Fields;
-    use arrow_schema::SchemaRef;
-    use datafusion::arrow::array::DictionaryArray;
-    use datafusion::arrow::array::Int32Array;
-    use datafusion::arrow::array::RecordBatch;
-    use datafusion::arrow::array::StringArray;
-    use datafusion::arrow::array::StructArray;
-    use datafusion::arrow::datatypes::DataType;
-    use datafusion::arrow::datatypes::Schema;
-    use datafusion::arrow::datatypes::UInt32Type;
-    use datafusion::arrow::util::display::FormatOptions;
-    use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
-    use datafusion::common::record_batch;
-    use datafusion::logical_expr::col;
-    use datafusion::logical_expr::lit;
-    use datafusion::physical_expr::planner::logical2physical;
-    use datafusion::physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
-    use datafusion::scalar::ScalarValue;
-    use datafusion_common::stats::Precision;
-    use datafusion_execution::cache::DefaultFilesMetadataCache;
-    use datafusion_expr::Operator;
-    use datafusion_physical_expr::PhysicalExpr;
-    use datafusion_physical_expr::expressions as df_expr;
-    use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
-    use datafusion_physical_expr::projection::ProjectionExpr;
-    use futures::TryStreamExt;
-    use insta::assert_snapshot;
-    use itertools::Itertools;
-    use object_store::ObjectStore;
-    use object_store::memory::InMemory;
-    use rstest::rstest;
-    use vortex::VortexSessionDefault;
-    use vortex::array::ArrayRef;
-    use vortex::buffer::Buffer;
-    use vortex::file::WriteOptionsSessionExt;
-    use vortex::io::VortexWrite;
-    use vortex::io::object_store::ObjectStoreWrite;
-    use vortex::metrics::DefaultMetricsRegistry;
-    use vortex::scan::selection::Selection;
-    use vortex::session::VortexSession;
-    use vortex_arrow::FromArrowArray;
-
-    use super::*;
-    use crate::VortexAccessPlan;
-    use crate::convert::exprs::DefaultExpressionConvertor;
-    use crate::persistent::reader::DefaultVortexReaderFactory;
-
-    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::default);
-
-    /// Test-only expr used to test error reporting.
-    #[derive(Debug, Eq, Hash, PartialEq)]
-    struct SnapshotErrorExpr;
-
-    impl fmt::Display for SnapshotErrorExpr {
-        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            write!(f, "snapshot_error")
-        }
-    }
-
-    impl PhysicalExpr for SnapshotErrorExpr {
-        fn data_type(&self, _input_schema: &Schema) -> DFResult<DataType> {
-            Ok(DataType::Boolean)
-        }
-
-        fn nullable(&self, _input_schema: &Schema) -> DFResult<bool> {
-            Ok(false)
-        }
-
-        fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            fmt::Display::fmt(self, f)
-        }
-
-        fn evaluate(&self, _batch: &RecordBatch) -> DFResult<datafusion_expr::ColumnarValue> {
-            Err(DataFusionError::Internal(
-                "intentional snapshot error".to_owned(),
-            ))
-        }
-
-        fn children(&self) -> Vec<&PhysicalExprRef> {
-            Vec::new()
-        }
-
-        fn with_new_children(
-            self: Arc<Self>,
-            children: Vec<PhysicalExprRef>,
-        ) -> DFResult<PhysicalExprRef> {
-            assert!(children.is_empty());
-            Ok(self)
-        }
-
-        fn snapshot(&self) -> DFResult<Option<PhysicalExprRef>> {
-            Err(DataFusionError::Internal(
-                "intentional snapshot error".to_owned(),
-            ))
-        }
-    }
-
-    fn natural_splits(total_size: u64, split_ranges: &[Range<u64>]) -> NaturalSplits {
-        let mut row_boundaries = Vec::with_capacity(split_ranges.len() + 1);
-        if let Some(first) = split_ranges.first() {
-            row_boundaries.push(first.start);
-            row_boundaries.extend(split_ranges.iter().map(|range| range.end));
-        }
-        NaturalSplits::new(row_boundaries.into(), total_size)
-    }
-
-    #[rstest]
-    #[case(0..3, 10, vec![0..2, 2..5, 5..10], Some(0..2))]
-    #[case(3..7, 10, vec![0..2, 2..5, 5..10], Some(2..5))]
-    #[case(1..8, 10, vec![0..1, 1..9, 9..10], Some(1..9))]
-    #[case(1..4, 16, vec![0..1, 1..2, 2..3, 3..4], None)]
-    #[case(0..1, 10, vec![0..2, 2..10], Some(0..2))]
-    fn test_split_aligned_row_range(
-        #[case] byte_range: Range<u64>,
-        #[case] total_size: u64,
-        #[case] split_ranges: Vec<Range<u64>>,
-        #[case] expected: Option<Range<u64>>,
-    ) {
-        assert_eq!(
-            split_aligned_row_range(byte_range, &natural_splits(total_size, &split_ranges)),
-            expected
-        );
-    }
-
-    #[test]
-    fn test_split_aligned_ranges_cover_splits_exactly_once() {
-        let split_ranges = vec![0..1, 1..4, 4..10, 10..13];
-        let byte_ranges = [0..4, 4..8, 8..12, 12..16];
-        let natural_splits = natural_splits(16, &split_ranges);
-
-        let assigned = byte_ranges
-            .into_iter()
-            .filter_map(|byte_range| split_aligned_row_range(byte_range, &natural_splits))
-            .collect::<Vec<_>>();
-
-        assert_eq!(assigned, vec![0..4, 4..10, 10..13]);
-        assert_eq!(
-            assigned
-                .iter()
-                .map(|range| range.end - range.start)
-                .sum::<u64>(),
-            13
-        );
-
-        let split_starts = split_ranges
-            .iter()
-            .map(|range| range.start)
-            .collect::<Vec<_>>();
-        let split_ends = split_ranges
-            .iter()
-            .map(|range| range.end)
-            .collect::<Vec<_>>();
-
-        for range in &assigned {
-            assert!(split_starts.contains(&range.start));
-            assert!(split_ends.contains(&range.end));
-        }
-
-        for (left, right) in assigned.iter().tuple_windows() {
-            assert_eq!(left.end, right.start);
-        }
-    }
-
-    #[test]
-    fn test_split_aligned_row_range_keeps_colliding_assignments_together() {
-        let natural_splits = natural_splits(2, &[0..1, 1..2, 2..3, 3..4]);
-
-        assert_eq!(natural_splits.assignment_bytes.as_ref(), [0, 0, 1, 1]);
-        assert_eq!(split_aligned_row_range(0..1, &natural_splits), Some(0..2));
-        assert_eq!(split_aligned_row_range(1..2, &natural_splits), Some(2..4));
-    }
-
-    async fn write_arrow_to_vortex(
-        object_store: Arc<dyn ObjectStore>,
-        path: &str,
-        rb: RecordBatch,
-    ) -> anyhow::Result<u64> {
-        let array = ArrayRef::from_arrow(rb, false)?;
-        let path = Path::parse(path)?;
-
-        let mut write = ObjectStoreWrite::new(object_store, &path).await?;
-        let summary = SESSION
-            .write_options()
-            .write(&mut write, array.to_array_stream())
-            .await?;
-        write.shutdown().await?;
-
-        Ok(summary.size())
-    }
-
-    fn make_opener(
-        object_store: Arc<dyn ObjectStore>,
-        table_schema: TableSchema,
-        filter: Option<PhysicalExprRef>,
-    ) -> VortexOpener {
-        VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
-            projection: ProjectionExprs::from_indices(&[0], table_schema.file_schema()),
-            filter,
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema,
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        }
-    }
-
-    #[tokio::test]
-    async fn test_open() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "part=1/file.vortex";
-        let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let file_schema = batch.schema();
-        let mut file = PartitionedFile::new(file_path.to_string(), data_size);
-        file.partition_values = vec![ScalarValue::Int32(Some(1))];
-
-        let table_schema = TableSchema::new(
-            Arc::clone(&file_schema),
-            vec![Arc::new(Field::new("part", DataType::Int32, false))],
-        );
-
-        // filter matches partition value
-        let filter = col("part").eq(lit(1));
-        let filter = logical2physical(&filter, table_schema.table_schema());
-
-        let opener = make_opener(
-            Arc::clone(&object_store),
-            table_schema.clone(),
-            Some(filter),
-        );
-        let stream = opener.open(file.clone()).unwrap().await.unwrap();
-
-        let data = stream.try_collect::<Vec<_>>().await?;
-        let num_batches = data.len();
-        let num_rows = data.iter().map(|rb| rb.num_rows()).sum::<usize>();
-
-        assert_eq!((num_batches, num_rows), (1, 3));
-
-        // filter doesn't matches partition value
-        let filter = col("part").eq(lit(2));
-        let filter = logical2physical(&filter, table_schema.table_schema());
-
-        let opener = make_opener(
-            Arc::clone(&object_store),
-            table_schema.clone(),
-            Some(filter),
-        );
-        let stream = opener.open(file.clone()).unwrap().await.unwrap();
-
-        let data = stream.try_collect::<Vec<_>>().await?;
-        let num_batches = data.len();
-        let num_rows = data.iter().map(|rb| rb.num_rows()).sum::<usize>();
-        assert_eq!((num_batches, num_rows), (0, 0));
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_open_preserves_declared_schema_metadata() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "part=1/file.vortex";
-        let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)]))?;
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let file_schema = Arc::new(
-            batch.schema().as_ref().clone().with_metadata(
-                [("table".to_string(), "metadata".to_string())]
-                    .into_iter()
-                    .collect(),
-            ),
-        );
-        let table_schema = TableSchema::new(
-            file_schema,
-            vec![Arc::new(
-                Field::new("part", DataType::Int32, false).with_metadata(
-                    [("partition".to_string(), "metadata".to_string())]
-                        .into_iter()
-                        .collect(),
-                ),
-            )],
-        );
-        let projection = ProjectionExprs::from_indices(&[0, 1], table_schema.table_schema());
-        let expected_schema = Arc::new(projection.project_schema(table_schema.table_schema())?);
-
-        assert_eq!(
-            expected_schema.metadata().get("table"),
-            Some(&"metadata".to_string())
-        );
-        assert_eq!(
-            expected_schema.field(1).metadata().get("partition"),
-            Some(&"metadata".to_string())
-        );
-
-        for projection_pushdown in [false, true] {
-            let mut opener = make_opener(Arc::clone(&object_store), table_schema.clone(), None);
-            opener.projection = projection.clone();
-            opener.projection_pushdown = projection_pushdown;
-
-            let mut file = PartitionedFile::new(file_path.to_string(), data_size);
-            file.partition_values = vec![ScalarValue::Int32(Some(1))];
-            let batches = opener.open(file)?.await?.try_collect::<Vec<_>>().await?;
-
-            assert!(!batches.is_empty());
-            for batch in batches {
-                assert_eq!(batch.schema().as_ref(), expected_schema.as_ref());
-            }
-        }
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_open_all_valid_nullable_columns_with_nonnullable_table_schema()
-    -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "nullable/file.vortex";
-        let batch = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),
-            vec![Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)]))],
-        )?;
-        let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
-
-        let expected_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
-        let table_schema = TableSchema::from_file_schema(Arc::clone(&expected_schema));
-
-        for projection_pushdown in [false, true] {
-            let mut opener = make_opener(Arc::clone(&object_store), table_schema.clone(), None);
-            opener.projection_pushdown = projection_pushdown;
-
-            let file = PartitionedFile::new(file_path.to_string(), data_size);
-            let batches = opener.open(file)?.await?.try_collect::<Vec<_>>().await?;
-
-            assert_eq!(batches.len(), 1);
-            assert_eq!(batches[0].schema().as_ref(), expected_schema.as_ref());
-        }
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_file_pruning_replaces_partition_columns_without_file_statistics()
-    -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
-        let table_schema = TableSchema::new(
-            Arc::clone(&file_schema),
-            vec![Arc::new(Field::new("part", DataType::Int32, false))],
-        );
-
-        let partition_column = Arc::new(df_expr::Column::new("part", 1)) as PhysicalExprRef;
-        let predicate = Arc::new(df_expr::BinaryExpr::new(
-            Arc::clone(&partition_column),
-            Operator::Gt,
-            df_expr::lit(ScalarValue::Int32(Some(1))),
-        )) as PhysicalExprRef;
-        let dynamic_predicate = Arc::new(DynamicFilterPhysicalExpr::new(
-            vec![partition_column],
-            predicate,
-        )) as PhysicalExprRef;
-
-        let mut opener = make_opener(object_store, table_schema, None);
-        opener.file_pruning_predicate = Some(dynamic_predicate);
-        let df_metrics = opener.df_metrics.clone();
-
-        // The file does not exist and has no statistics. Replacing `part` with 1
-        // makes the predicate false, so pruning must happen before any file I/O.
-        let mut file = PartitionedFile::new("missing.vortex", 1);
-        file.partition_values = vec![ScalarValue::Int32(Some(1))];
-        let batches = opener.open(file)?.await?.try_collect::<Vec<_>>().await?;
-
-        assert!(batches.is_empty());
-        assert_eq!(
-            df_metrics
-                .clone_inner()
-                .sum_by_name("num_predicate_creation_errors")
-                .map(|metric| metric.as_usize()),
-            Some(0)
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_file_pruning_creation_errors_are_reported() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "metrics/file.vortex";
-        let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-        let mut statistics = Statistics::new_unknown(batch.schema().as_ref());
-        statistics.column_statistics[0].null_count = Precision::Exact(0);
-        let file = PartitionedFile::new(file_path, data_size).with_statistics(Arc::new(statistics));
-
-        let mut opener = make_opener(
-            object_store,
-            TableSchema::from_file_schema(batch.schema()),
-            None,
-        );
-        opener.file_pruning_predicate = Some(Arc::new(SnapshotErrorExpr));
-        let df_metrics = opener.df_metrics.clone();
-
-        let batches = opener.open(file)?.await?.try_collect::<Vec<_>>().await?;
-
-        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
-        assert_eq!(
-            df_metrics
-                .clone_inner()
-                .sum_by_name("num_predicate_creation_errors")
-                .map(|metric| metric.as_usize()),
-            Some(1)
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_open_applies_limit_after_filtering() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "filtered-limit/file.vortex";
-        let batch = record_batch!((
-            "a",
-            Int32,
-            vec![Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)]
-        ))
-        .unwrap();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-        let file = PartitionedFile::new(file_path.to_string(), data_size);
-        let table_schema = TableSchema::from_file_schema(batch.schema());
-        // `a > 3` excludes the first three rows, so a limit applied *before* filtering would take
-        // rows [1, 2, 3] and filter them all out (yielding nothing), whereas a limit applied
-        // *after* filtering yields the first three matching rows [4, 5, 6]. Asserting the values
-        // (not just the count) is what makes this test able to detect a pre-filter regression.
-        let filter = logical2physical(&col("a").gt(lit(3_i32)), table_schema.table_schema());
-
-        let mut opener = make_opener(object_store, table_schema, Some(filter));
-        opener.limit = Some(3);
-
-        let batches = opener.open(file)?.await?.try_collect::<Vec<_>>().await?;
-        let values = batches
-            .iter()
-            .flat_map(|batch| {
-                batch
-                    .column(0)
-                    .as_any()
-                    .downcast_ref::<Int32Array>()
-                    .expect("projected column should be Int32")
-                    .values()
-                    .to_vec()
-            })
-            .collect::<Vec<i32>>();
-
-        assert_eq!(values, [4, 5, 6]);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_open_empty_file() -> anyhow::Result<()> {
-        use futures::TryStreamExt;
-
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let data_batch = record_batch!(("a", Int32, Vec::<i32>::new())).unwrap();
-        let file_path = "part=1/empty.vortex";
-        let file_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, data_batch.clone()).await?;
-
-        let file_schema = data_batch.schema();
-        // Parallel scans may attach a byte range even for empty files; the
-        // opener must return early before attempting split-aligned translation.
-        let file =
-            PartitionedFile::new_with_range(file_path.to_string(), file_size, 0, file_size as i64);
-
-        let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));
-
-        let opener = make_opener(object_store, table_schema, None);
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-
-        assert_eq!(data.len(), 0);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_open_populates_file_metadata_cache() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "cached/file.vortex";
-        let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let file = PartitionedFile::new(file_path.to_string(), data_size);
-        let table_schema = TableSchema::from_file_schema(batch.schema());
-
-        let cache: Arc<dyn FileMetadataCache> =
-            Arc::new(DefaultFilesMetadataCache::new(64 * 1024 * 1024));
-        let mut opener = make_opener(Arc::clone(&object_store), table_schema, None);
-        opener.file_metadata_cache = Some(Arc::clone(&cache));
-
-        // The first open misses the cache and must write the parsed footer back.
-        let stream = opener.open(file.clone())?.await?;
-        stream.try_collect::<Vec<_>>().await?;
-
-        let entry = cache
-            .get(file.path())
-            .ok_or_else(|| anyhow::anyhow!("footer was not cached after open"))?;
-        assert!(entry.is_valid_for(&file.object_meta));
-        assert!(
-            entry
-                .file_metadata
-                .as_any()
-                .downcast_ref::<CachedVortexMetadata>()
-                .is_some()
-        );
-
-        // The second open hits the cache and still returns the same data.
-        let stream = opener.open(file.clone())?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-        assert_eq!(data.iter().map(|rb| rb.num_rows()).sum::<usize>(), 3);
-
-        Ok(())
-    }
-
-    #[rstest]
-    #[tokio::test]
-    async fn test_open_files_different_table_schema() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-
-        let file1 = {
-            let file1_path = "/path/file1.vortex";
-            let batch1 = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
-            let data_size1 =
-                write_arrow_to_vortex(Arc::clone(&object_store), file1_path, batch1).await?;
-            PartitionedFile::new(file1_path.to_string(), data_size1)
-        };
-
-        let file2 = {
-            let file2_path = "/path/file2.vortex";
-            let batch2 = record_batch!(("a", Int16, vec![Some(-1), Some(-2), Some(-3)])).unwrap();
-            let data_size2 =
-                write_arrow_to_vortex(Arc::clone(&object_store), file2_path, batch2).await?;
-            PartitionedFile::new(file2_path.to_string(), data_size2)
-        };
-
-        // Table schema has can accommodate both files
-        let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new(
-            "a",
-            DataType::Int32,
-            true,
-        )])));
-
-        let make_opener = |filter| VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(
-                &object_store,
-            ))),
-            projection: ProjectionExprs::from_indices(&[0], table_schema.file_schema()),
-            filter: Some(filter),
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema: table_schema.clone(),
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        };
-
-        let filter = col("a").lt(lit(100_i32));
-        let filter = logical2physical(&filter, table_schema.table_schema());
-
-        let opener1 = make_opener(Arc::clone(&filter));
-        let stream = opener1.open(file1)?.await?;
-
-        let format_opts = FormatOptions::new().with_types_info(true);
-
-        let data = stream.try_collect::<Vec<_>>().await?;
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+
-        | a     |
-        | Int32 |
-        +-------+
-        | 1     |
-        | 2     |
-        | 3     |
-        +-------+
-        ");
-
-        let opener2 = make_opener(Arc::clone(&filter));
-        let stream = opener2.open(file2)?.await?;
-
-        let data = stream.try_collect::<Vec<_>>().await?;
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+
-        | a     |
-        | Int32 |
-        +-------+
-        | -1    |
-        | -2    |
-        | -3    |
-        +-------+
-        ");
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // This test verifies that files with different column order than the
-    // table schema can be opened without errors. The fix ensures that the
-    // schema mapper is only used for type casting, not for reordering,
-    // since the vortex projection already handles reordering.
-    async fn test_schema_different_column_order() -> anyhow::Result<()> {
-        use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
-
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        // File has columns in order: c, b, a
-        let batch = record_batch!(
-            ("c", Int32, vec![Some(300), Some(301), Some(302)]),
-            ("b", Int32, vec![Some(200), Some(201), Some(202)]),
-            ("a", Int32, vec![Some(100), Some(101), Some(102)])
-        )
-        .unwrap();
-        let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
-        let file = PartitionedFile::new(file_path.to_string(), data_size);
-
-        // Table schema has columns in different order: a, b, c
-        let table_schema = Arc::new(Schema::new(vec![
-            Field::new("a", DataType::Int32, true),
-            Field::new("b", DataType::Int32, true),
-            Field::new("c", DataType::Int32, true),
-        ]));
-
-        let opener = VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
-            projection: ProjectionExprs::from_indices(&[0, 1, 2], &table_schema),
-            filter: None,
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema: TableSchema::from_file_schema(Arc::clone(&table_schema)),
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        };
-
-        let stream = opener.open(file)?.await?;
-
-        let format_opts = FormatOptions::new().with_types_info(true);
-        let data = stream.try_collect::<Vec<_>>().await?;
-
-        // Verify the output has columns in table schema order (a, b, c)
-        // not file order (c, b, a)
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+-------+-------+
-        | a     | b     | c     |
-        | Int32 | Int32 | Int32 |
-        +-------+-------+-------+
-        | 100   | 200   | 300   |
-        | 101   | 201   | 301   |
-        | 102   | 202   | 302   |
-        +-------+-------+-------+
-        ");
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // This test verifies that expression rewriting doesn't fail when there is
-    // a nested schema mismatch between the physical file schema and logical
-    // table schema.
-    async fn test_adapter_logical_physical_struct_mismatch() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-        let file_struct_fields = Fields::from(vec![
-            Field::new("field1", DataType::Utf8, true),
-            Field::new("field2", DataType::Utf8, true),
-        ]);
-        let struct_array = StructArray::new(
-            file_struct_fields.clone(),
-            vec![
-                Arc::new(StringArray::from(vec!["value1", "value2", "value3"])),
-                Arc::new(StringArray::from(vec!["a", "b", "c"])),
-            ],
-            None,
-        );
-        let batch = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new(
-                "my_struct",
-                DataType::Struct(file_struct_fields),
-                true,
-            )])),
-            vec![Arc::new(struct_array)],
-        )?;
-        let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
-
-        // Table schema has an extra utf8 field.
-        let table_schema = TableSchema::from_file_schema(Arc::new(Schema::new(vec![Field::new(
-            "my_struct",
-            DataType::Struct(Fields::from(vec![
-                Field::new(
-                    "field1",
-                    DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
-                    true,
-                ),
-                Field::new(
-                    "field2",
-                    DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
-                    true,
-                ),
-                Field::new("field3", DataType::Utf8, true),
-            ])),
-            true,
-        )])));
-
-        let opener = make_opener(
-            Arc::clone(&object_store),
-            table_schema.clone(),
-            // expression references my_struct column which has different fields in each
-            // field.
-            Some(logical2physical(
-                &col("my_struct").is_not_null(),
-                table_schema.table_schema(),
-            )),
-        );
-
-        // The opener should be able to open the file with a filter on the
-        // struct column.
-        let data = opener
-            .open(PartitionedFile::new(file_path.to_string(), data_size))?
-            .await?
-            .try_collect::<Vec<_>>()
-            .await?;
-
-        assert_eq!(data.len(), 1);
-        assert_eq!(data[0].num_rows(), 3);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // Minimal reproducing test for the schema projection bug.
-    // Before the fix, this would fail with a cast error when the file schema
-    // and table schema have different field orders and we project a subset of columns.
-    async fn test_projection_bug_minimal_repro() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        // File has columns in order: a, b, c with simple types
-        let batch = record_batch!(
-            ("a", Int32, vec![Some(1)]),
-            ("b", Utf8, vec![Some("test")]),
-            ("c", Int32, vec![Some(2)])
-        )
-        .unwrap();
-        let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
-
-        // Table schema has columns in DIFFERENT order: c, a, b
-        // and different types that require casting (Utf8 -> Dictionary)
-        let table_schema = TableSchema::new(
-            Arc::new(Schema::new(vec![
-                Field::new("c", DataType::Int32, true),
-                Field::new("a", DataType::Int32, true),
-                Field::new(
-                    "b",
-                    DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
-                    true,
-                ),
-            ])),
-            vec![],
-        );
-
-        // Project columns [0, 2] from table schema, which should give us: c, b
-        // Before the fix, the schema adapter would get confused about which fields
-        // to select from the file, causing incorrect type mappings.
-        let projection = vec![0, 2];
-
-        let opener = VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(
-                &object_store,
-            ))),
-            projection: ProjectionExprs::from_indices(
-                projection.as_ref(),
-                table_schema.file_schema(),
-            ),
-            filter: None,
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema: table_schema.clone(),
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        };
-
-        // This should succeed and return the correctly projected and cast data
-        let data = opener
-            .open(PartitionedFile::new(file_path.to_string(), data_size))?
-            .await?
-            .try_collect::<Vec<_>>()
-            .await?;
-
-        // Verify the columns are in the right order and have the right values
-        use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
-        let format_opts = FormatOptions::new().with_types_info(true);
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+--------------------------+
-        | c     | b                        |
-        | Int32 | Dictionary(UInt32, Utf8) |
-        +-------+--------------------------+
-        | 2     | test                     |
-        +-------+--------------------------+
-        ");
-
-        Ok(())
-    }
-
-    fn make_test_batch_with_10_rows() -> RecordBatch {
-        record_batch!(
-            ("a", Int32, (0..=9).map(Some).collect::<Vec<_>>()),
-            (
-                "b",
-                Utf8,
-                (0..=9).map(|i| Some(format!("r{}", i))).collect::<Vec<_>>()
-            )
-        )
-        .unwrap()
-    }
-
-    fn make_test_opener(
-        object_store: Arc<dyn ObjectStore>,
-        schema: SchemaRef,
-        projection: ProjectionExprs,
-    ) -> VortexOpener {
-        VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(object_store)),
-            projection,
-            filter: None,
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema: TableSchema::from_file_schema(schema),
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        }
-    }
-
-    #[tokio::test]
-    // Test that Selection::IncludeByIndex filters to specific row indices.
-    async fn test_selection_include_by_index() -> anyhow::Result<()> {
-        use datafusion::arrow::util::pretty::pretty_format_batches_with_options;
-        use vortex::buffer::Buffer;
-        use vortex::scan::selection::Selection;
-
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        let batch = make_test_batch_with_10_rows();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let schema = batch.schema();
-        let mut file = PartitionedFile::new(file_path.to_string(), data_size);
-        file.extensions
-            .insert(
-                VortexAccessPlan::default().with_selection(Selection::IncludeByIndex(
-                    Buffer::from_iter(vec![1, 3, 5, 7]),
-                )),
-            );
-
-        let opener = make_test_opener(
-            Arc::clone(&object_store),
-            Arc::clone(&schema),
-            ProjectionExprs::from_indices(&[0, 1], &schema),
-        );
-
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-        let format_opts = FormatOptions::new().with_types_info(true);
-
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+------+
-        | a     | b    |
-        | Int32 | Utf8 |
-        +-------+------+
-        | 1     | r1   |
-        | 3     | r3   |
-        | 5     | r5   |
-        | 7     | r7   |
-        +-------+------+
-        ");
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // Test that Selection::ExcludeByIndex excludes specific row indices.
-    async fn test_selection_exclude_by_index() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        let batch = make_test_batch_with_10_rows();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let schema = batch.schema();
-        let mut file = PartitionedFile::new(file_path.to_string(), data_size);
-        file.extensions
-            .insert(
-                VortexAccessPlan::default().with_selection(Selection::ExcludeByIndex(
-                    Buffer::from_iter(vec![0, 2, 4, 6, 8]),
-                )),
-            );
-
-        let opener = make_test_opener(
-            Arc::clone(&object_store),
-            Arc::clone(&schema),
-            ProjectionExprs::from_indices(&[0, 1], &schema),
-        );
-
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-        let format_opts = FormatOptions::new().with_types_info(true);
-
-        assert_snapshot!(pretty_format_batches_with_options(&data, &format_opts)?.to_string(), @r"
-        +-------+------+
-        | a     | b    |
-        | Int32 | Utf8 |
-        +-------+------+
-        | 1     | r1   |
-        | 3     | r3   |
-        | 5     | r5   |
-        | 7     | r7   |
-        | 9     | r9   |
-        +-------+------+
-        ");
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // Test that Selection::All returns all rows.
-    async fn test_selection_all() -> anyhow::Result<()> {
-        use vortex::scan::selection::Selection;
-
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        let batch = make_test_batch_with_10_rows();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let schema = batch.schema();
-        let mut file = PartitionedFile::new(file_path.to_string(), data_size);
-        file.extensions
-            .insert(VortexAccessPlan::default().with_selection(Selection::All));
-
-        let opener = make_test_opener(
-            Arc::clone(&object_store),
-            Arc::clone(&schema),
-            ProjectionExprs::from_indices(&[0], &schema),
-        );
-
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-
-        let total_rows: usize = data.iter().map(|rb| rb.num_rows()).sum();
-        assert_eq!(total_rows, 10);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    // Test that when no extensions are provided, all rows are returned (backward compatibility).
-    async fn test_selection_no_extensions() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        let batch = make_test_batch_with_10_rows();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let schema = batch.schema();
-        let file = PartitionedFile::new(file_path.to_string(), data_size);
-        // file.extensions is None by default
-
-        let opener = make_test_opener(
-            Arc::clone(&object_store),
-            Arc::clone(&schema),
-            ProjectionExprs::from_indices(&[0], &schema),
-        );
-
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-
-        let total_rows: usize = data.iter().map(|rb| rb.num_rows()).sum();
-        assert_eq!(total_rows, 10);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_projection_expr_pushdown() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-        let file_path = "/path/file.vortex";
-
-        let batch = record_batch!(
-            ("a", Int32, vec![Some(1), Some(2), Some(3)]),
-            ("b", Int32, vec![Some(10), Some(20), Some(30)])
-        )
-        .unwrap();
-        let data_size =
-            write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch.clone()).await?;
-
-        let file_schema = batch.schema();
-        let table_schema = TableSchema::from_file_schema(Arc::clone(&file_schema));
-
-        // Create a projection that includes an arithmetic expression: a + b * 2
-        let col_a = df_expr::col("a", &file_schema)?;
-        let col_b = df_expr::col("b", &file_schema)?;
-        let two = df_expr::lit(ScalarValue::Int32(Some(2)));
-
-        // b * 2
-        let b_times_2 = df_expr::binary(col_b, Operator::Multiply, two, &file_schema)?;
-        // a + (b * 2)
-        let a_plus_b_times_2 = df_expr::binary(col_a, Operator::Plus, b_times_2, &file_schema)?;
-
-        let projection = ProjectionExprs::new(vec![ProjectionExpr::new(
-            a_plus_b_times_2,
-            "result".to_string(),
-        )]);
-
-        let opener = VortexOpener {
-            partition: 1,
-            session: SESSION.clone(),
-            vortex_reader_factory: Arc::new(DefaultVortexReaderFactory::new(Arc::clone(
-                &object_store,
-            ))),
-            projection,
-            filter: None,
-            file_pruning_predicate: None,
-            expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
-            table_schema,
-            limit: None,
-            metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
-            df_metrics: ExecutionPlanMetricsSet::new(),
-            layout_readers: Default::default(),
-            natural_splits: Default::default(),
-            has_output_ordering: false,
-            expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
-            file_metadata_cache: None,
-            projection_pushdown: false,
-            scan_concurrency: None,
-        };
-
-        let file = PartitionedFile::new(file_path.to_string(), data_size);
-        let stream = opener.open(file)?.await?;
-        let data = stream.try_collect::<Vec<_>>().await?;
-
-        // Expected: a + b * 2
-        // row 0: 1 + 10 * 2 = 21
-        // row 1: 2 + 20 * 2 = 42
-        // row 2: 3 + 30 * 2 = 63
-        assert_snapshot!(pretty_format_batches_with_options(&data, &FormatOptions::new().with_types_info(true))?.to_string(), @r"
-        +--------+
-        | result |
-        | Int32  |
-        +--------+
-        | 21     |
-        | 42     |
-        | 63     |
-        +--------+
-        ");
-
-        Ok(())
-    }
-
-    /// When a Struct contains Dictionary fields, writing to vortex and reading back
-    /// should preserve the Dictionary type.
-    #[tokio::test]
-    async fn test_struct_with_dictionary_roundtrip() -> anyhow::Result<()> {
-        let object_store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
-
-        let struct_fields = Fields::from(vec![
-            Field::new_dictionary("a", DataType::UInt32, DataType::Utf8, true),
-            Field::new_dictionary("b", DataType::UInt32, DataType::Utf8, true),
-        ]);
-        let struct_array = StructArray::new(
-            struct_fields.clone(),
-            vec![
-                Arc::new(DictionaryArray::<UInt32Type>::from_iter(["x", "y", "x"])),
-                Arc::new(DictionaryArray::<UInt32Type>::from_iter(["p", "p", "q"])),
-            ],
-            None,
-        );
-
-        let schema = Arc::new(Schema::new(vec![Field::new(
-            "labels",
-            DataType::Struct(struct_fields.clone()),
-            false,
-        )]));
-        let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(struct_array)])?;
-
-        let file_path = "/test.vortex";
-        let data_size = write_arrow_to_vortex(Arc::clone(&object_store), file_path, batch).await?;
-
-        let opener = make_test_opener(
-            Arc::clone(&object_store),
-            Arc::clone(&schema),
-            ProjectionExprs::from_indices(&[0], &schema),
-        );
-        let data: Vec<_> = opener
-            .open(PartitionedFile::new(file_path.to_string(), data_size))?
-            .await?
-            .try_collect()
-            .await?;
-
-        assert_eq!(
-            data[0].schema().field(0).data_type(),
-            &DataType::Struct(struct_fields),
-            "Struct(Dictionary) type should be preserved"
-        );
-        Ok(())
-    }
-}
diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs
index d24a77ba13e..a0945c06131 100644
--- a/vortex-datafusion/src/persistent/source.rs
+++ b/vortex-datafusion/src/persistent/source.rs
@@ -7,6 +7,7 @@ use std::sync::Weak;
 
 use datafusion_common::Result as DFResult;
 use datafusion_common::config::ConfigOptions;
+use datafusion_common::not_impl_err;
 use datafusion_datasource::TableSchema;
 use datafusion_datasource::file::FileSource;
 use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -35,9 +36,8 @@ use vortex::metrics::MetricsRegistry;
 use vortex::session::VortexSession;
 use vortex_utils::aliases::dash_map::DashMap;
 
+use super::morsel::NaturalSplitCache;
 use super::morsel::VortexMorselizer;
-use super::opener::NaturalSplits;
-use super::opener::VortexOpener;
 use crate::VortexTableOptions;
 use crate::convert::exprs::DefaultExpressionConvertor;
 use crate::convert::exprs::ExpressionConvertor;
@@ -162,10 +162,10 @@ use crate::persistent::reader::VortexReaderFactory;
 ///
 /// 1. DataFusion calls `DataSourceExec`, which delegates file opening to
 ///    `VortexSource`.
-/// 2. `VortexSource` creates a `VortexOpener` configured with the current
+/// 2. `VortexSource` creates a `VortexMorselizer` configured with the current
 ///    projection, predicate, options, and metrics.
-/// 3. The opener adapts filters and schema for the specific file, applies any
-///    [`VortexAccessPlan`], and builds a Vortex scan.
+/// 3. The morsel planner adapts filters and schema for the specific file,
+///    applies any [`VortexAccessPlan`], and builds a Vortex scan.
 /// 4. Scan results are converted into Arrow `RecordBatch` values for
 ///    DataFusion.
 ///
@@ -190,14 +190,15 @@ pub struct VortexSource {
     /// Subset of predicates that can be pushed down into Vortex scan operations.
     /// These are expressions that Vortex can efficiently evaluate during scanning.
     pub(crate) vortex_predicate: Option<PhysicalExprRef>,
+    /// Desired row count for record batches returned from the scan.
     /// DataFusion-native metrics exposed through `DataSourceExec`.
     df_metrics: ExecutionPlanMetricsSet,
     /// Shared layout readers, the source only lives as long as one scan.
     ///
     /// Sharing the readers allows us to only read every layout once from the file, even across partitions.
     layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
-    /// Shared full-file natural splits keyed by path.
-    natural_splits: Arc<DashMap<Path, Arc<NaturalSplits>>>,
+    /// Shared full-file natural split ranges keyed by path.
+    natural_split_ranges: Arc<NaturalSplitCache>,
     expression_convertor: Arc<dyn ExpressionConvertor>,
     pub(crate) vortex_reader_factory: Option<Arc<dyn VortexReaderFactory>>,
     pub(crate) ordered: bool,
@@ -230,7 +231,7 @@ impl VortexSource {
             vortex_predicate: None,
             df_metrics: Default::default(),
             layout_readers: Arc::new(DashMap::default()),
-            natural_splits: Arc::new(DashMap::default()),
+            natural_split_ranges: Arc::new(DashMap::default()),
             expression_convertor,
             vortex_reader_factory: None,
             vx_metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
@@ -328,12 +329,12 @@ impl VortexSource {
         self.vortex_predicate.as_ref()
     }
 
-    fn create_vortex_opener(
+    fn create_vortex_morselizer(
         &self,
         object_store: Arc<dyn ObjectStore>,
         base_config: &FileScanConfig,
         partition: usize,
-    ) -> DFResult<VortexOpener> {
+    ) -> DFResult<VortexMorselizer> {
         let expr_adapter_factory = base_config
             .expr_adapter_factory
             .clone()
@@ -344,7 +345,7 @@ impl VortexSource {
             .clone()
             .unwrap_or_else(|| Arc::new(DefaultVortexReaderFactory::new(object_store)));
 
-        let opener = VortexOpener {
+        Ok(VortexMorselizer {
             partition,
             session: self.session.clone(),
             vortex_reader_factory,
@@ -357,34 +358,28 @@ impl VortexSource {
             metrics_registry: Arc::clone(&self.vx_metrics_registry),
             df_metrics: self.df_metrics.clone(),
             layout_readers: Arc::clone(&self.layout_readers),
-            natural_splits: Arc::clone(&self.natural_splits),
+            natural_split_ranges: Arc::clone(&self.natural_split_ranges),
             has_output_ordering: !base_config.output_ordering.is_empty() || self.ordered,
             expression_convertor: Arc::clone(&self.expression_convertor),
             file_metadata_cache: self.file_metadata_cache.clone(),
             projection_pushdown: self.options.projection_pushdown,
             scan_concurrency: self.options.scan_concurrency,
-        };
-
-        Ok(opener)
+        })
     }
 }
 
 impl FileSource for VortexSource {
     fn create_file_opener(
         &self,
-        object_store: Arc<dyn ObjectStore>,
-        base_config: &FileScanConfig,
-        partition: usize,
+        _object_store: Arc<dyn ObjectStore>,
+        _base_config: &FileScanConfig,
+        _partition: usize,
     ) -> DFResult<Arc<dyn FileOpener>> {
-        Ok(Arc::new(self.create_vortex_opener(
-            object_store,
-            base_config,
-            partition,
-        )?))
+        not_impl_err!("VortexSource only supports morsel-driven I/O")
     }
 
     fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {
-        // DataSourceExec applies BatchSplitStream after the FileSource stream.
+        // DataSourceExec applies BatchSplitStream after the morsel stream.
         Arc::new(self.clone())
     }
 
@@ -394,28 +389,11 @@ impl FileSource for VortexSource {
         base_config: &FileScanConfig,
         partition: usize,
     ) -> DFResult<Box<dyn datafusion_datasource::morsel::Morselizer>> {
-        let opener = self.create_vortex_opener(object_store, base_config, partition)?;
-
-        Ok(Box::new(VortexMorselizer {
-            partition: opener.partition,
-            session: opener.session,
-            vortex_reader_factory: opener.vortex_reader_factory,
-            projection: opener.projection,
-            filter: opener.filter,
-            file_pruning_predicate: opener.file_pruning_predicate,
-            expr_adapter_factory: opener.expr_adapter_factory,
-            table_schema: opener.table_schema,
-            limit: opener.limit,
-            metrics_registry: opener.metrics_registry,
-            df_metrics: opener.df_metrics,
-            layout_readers: opener.layout_readers,
-            natural_split_ranges: opener.natural_split_ranges,
-            has_output_ordering: opener.has_output_ordering,
-            expression_convertor: opener.expression_convertor,
-            file_metadata_cache: opener.file_metadata_cache,
-            projection_pushdown: opener.projection_pushdown,
-            scan_concurrency: opener.scan_concurrency,
-        }))
+        Ok(Box::new(self.create_vortex_morselizer(
+            object_store,
+            base_config,
+            partition,
+        )?))
     }
 
     fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {
@@ -685,7 +663,7 @@ mod tests {
     }
 
     #[test]
-    fn create_vortex_opener_preserves_expression_convertor() -> anyhow::Result<()> {
+    fn create_vortex_morselizer_preserves_configuration() -> anyhow::Result<()> {
         let file_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
         let expression_convertor = Arc::new(TrackingExpressionConvertor {
             inner: DefaultExpressionConvertor::default(),
@@ -703,14 +681,14 @@ mod tests {
         )
         .build();
 
-        let opener = source.create_vortex_opener(
+        let morselizer = source.create_vortex_morselizer(
             Arc::new(InMemory::new()) as Arc<dyn ObjectStore>,
             &config,
             0,
         )?;
 
         assert!(Arc::ptr_eq(
-            &opener.expression_convertor,
+            &morselizer.expression_convertor,
             &expression_convertor
         ));
         Ok(())

From c273ef137f207fa94bd8c4e589b028c7a9b73db1 Mon Sep 17 00:00:00 2001
From: Adam Gutglick <adam@spiraldb.com>
Date: Thu, 30 Jul 2026 11:43:20 +0100
Subject: [PATCH 3/4] simpler

Signed-off-by: Adam Gutglick <adam@spiraldb.com>
---
 vortex-datafusion/src/persistent/morsel/mod.rs | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/vortex-datafusion/src/persistent/morsel/mod.rs b/vortex-datafusion/src/persistent/morsel/mod.rs
index d43a8403e4f..349177d7052 100644
--- a/vortex-datafusion/src/persistent/morsel/mod.rs
+++ b/vortex-datafusion/src/persistent/morsel/mod.rs
@@ -587,14 +587,13 @@ impl MorselPlanner for VortexMorselPlanner {
                 split_ranges,
             } => {
                 let total_size = state.file.object_meta.size;
-                let compute_layout_reader = Arc::clone(&layout_reader);
 
                 Ok(Some(
                     MorselPlan::new().with_pending_planner(
                         async move {
                             let split_ranges = split_ranges
-                                .get_or_try_init(|| {
-                                    compute_natural_split_ranges_async(compute_layout_reader)
+                                .get_or_try_init(|| async {
+                                    compute_natural_split_ranges(layout_reader.as_ref())
                                 })
                                 .await?;
 
@@ -913,16 +912,6 @@ fn natural_split_cell_for_file(
     }
 }
 
-async fn compute_natural_split_ranges_async(
-    layout_reader: Arc<dyn LayoutReader>,
-) -> DFResult<NaturalSplitRanges> {
-    tokio::task::spawn_blocking(move || compute_natural_split_ranges(layout_reader.as_ref()))
-        .await
-        .map_err(|error| {
-            exec_datafusion_err!("Vortex natural split calculation task failed: {error}")
-        })?
-}
-
 fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<NaturalSplitRanges> {
     let row_count = layout_reader.row_count();
     let row_range = 0..row_count;

From 030ed74bd6e64a50e9078991c6aad52e7982f863 Mon Sep 17 00:00:00 2001
From: Adam Gutglick <adam@spiraldb.com>
Date: Fri, 31 Jul 2026 14:46:19 +0100
Subject: [PATCH 4/4] Reuse precomputed natural splits in the Morselizer

The Morselizer carried a copy of the natural-split helpers from the opener it
replaced, which predated the precomputed-assignment form on the base branch.
Port that forward: cache a NaturalSplits (row boundaries plus each split's
assignment byte) instead of raw ranges, and translate a byte range with two
binary searches instead of scanning every split. The async OnceCell cache the
Morselizer introduced is kept.

The boundaries are used only for byte-to-row partition alignment; the scan
derives its own splits from the referenced columns' field masks, matching the
base branch's fix for the all-column split inflation.

Signed-off-by: Adam Gutglick <adam@spiraldb.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../src/persistent/morsel/mod.rs              | 134 +++++++++++-------
 .../src/persistent/morsel/tests.rs            |  43 ++++--
 vortex-datafusion/src/persistent/source.rs    |   6 +-
 3 files changed, 121 insertions(+), 62 deletions(-)

diff --git a/vortex-datafusion/src/persistent/morsel/mod.rs b/vortex-datafusion/src/persistent/morsel/mod.rs
index 349177d7052..d281fa4bdc6 100644
--- a/vortex-datafusion/src/persistent/morsel/mod.rs
+++ b/vortex-datafusion/src/persistent/morsel/mod.rs
@@ -44,7 +44,6 @@ use datafusion_pruning::FilePruner;
 use futures::StreamExt;
 use futures::TryStreamExt;
 use futures::stream::BoxStream;
-use itertools::Itertools;
 use object_store::path::Path;
 use tokio::sync::OnceCell;
 use tracing::Instrument;
@@ -79,8 +78,52 @@ use crate::persistent::cache::CachedVortexMetadata;
 use crate::persistent::stream::PrunableStream;
 use crate::reader::VortexReaderFactory;
 
-pub(crate) type NaturalSplitRanges = Arc<[Range<u64>]>;
-pub(crate) type NaturalSplitCache = DashMap<Path, Arc<OnceCell<NaturalSplitRanges>>>;
+pub(crate) type NaturalSplitCache = DashMap<Path, Arc<OnceCell<Arc<NaturalSplits>>>>;
+
+/// A file's natural split boundaries, with each split's byte assignment precomputed.
+///
+/// `row_boundaries` is the sorted, deduplicated split point list covering the whole file.
+/// `assignment_bytes` holds the byte that owns each split (see [`split_assignment_byte`]) and is
+/// sorted, so translating a DataFusion byte range into rows is two binary searches rather than a
+/// scan over every split.
+#[derive(Debug)]
+pub(crate) struct NaturalSplits {
+    row_boundaries: Arc<[u64]>,
+    assignment_bytes: Box<[u64]>,
+}
+
+impl NaturalSplits {
+    fn new(row_boundaries: Arc<[u64]>, total_size: u64) -> Self {
+        let row_count = row_boundaries.last().copied().unwrap_or_default();
+        let assignment_bytes = if row_count == 0 {
+            Box::default()
+        } else {
+            row_boundaries
+                .windows(2)
+                .enumerate()
+                .map(|(idx, boundaries)| {
+                    split_assignment_byte(
+                        idx,
+                        &(boundaries[0]..boundaries[1]),
+                        row_count,
+                        total_size,
+                    )
+                })
+                .collect()
+        };
+
+        debug_assert!(assignment_bytes.is_sorted());
+        debug_assert_eq!(
+            assignment_bytes.len() + usize::from(!row_boundaries.is_empty()),
+            row_boundaries.len()
+        );
+
+        Self {
+            row_boundaries,
+            assignment_bytes,
+        }
+    }
+}
 
 /// Creates morsel planners for Vortex files.
 pub struct VortexMorselizer {
@@ -114,7 +157,7 @@ pub struct VortexMorselizer {
     /// a file reader the first time we read a file.
     pub layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
     /// Shared full-file natural split ranges keyed by file path.
-    pub natural_split_ranges: Arc<NaturalSplitCache>,
+    pub(crate) natural_splits: Arc<NaturalSplitCache>,
     /// Whether the query has output ordering specified
     pub has_output_ordering: bool,
 
@@ -139,7 +182,7 @@ impl std::fmt::Debug for VortexMorselizer {
             .field("limit", &self.limit)
             .field("df_metrics", &self.df_metrics)
             .field("layout_readers", &self.layout_readers)
-            .field("natural_split_ranges", &self.natural_split_ranges)
+            .field("natural_splits", &self.natural_splits)
             .field("has_output_ordering", &self.has_output_ordering)
             .field("file_metadata_cache", &self.file_metadata_cache)
             .field("projection_pushdown", &self.projection_pushdown)
@@ -194,7 +237,7 @@ impl VortexMorselPlanner {
         let unified_file_schema = Arc::clone(morselizer.table_schema.file_schema());
         let limit = morselizer.limit;
         let layout_readers = Arc::clone(&morselizer.layout_readers);
-        let natural_split_ranges = Arc::clone(&morselizer.natural_split_ranges);
+        let natural_splits = Arc::clone(&morselizer.natural_splits);
         let has_output_ordering = morselizer.has_output_ordering;
         let scan_concurrency = morselizer.scan_concurrency;
 
@@ -273,7 +316,7 @@ impl VortexMorselPlanner {
                     unified_file_schema,
                     limit,
                     layout_readers,
-                    natural_split_ranges,
+                    natural_splits,
                     has_output_ordering,
                     expr_convertor,
                     projection_pushdown,
@@ -299,7 +342,7 @@ struct FileOpenState {
     unified_file_schema: SchemaRef,
     limit: Option<u64>,
     layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
-    natural_split_ranges: Arc<NaturalSplitCache>,
+    natural_splits: Arc<NaturalSplitCache>,
     has_output_ordering: bool,
     expr_convertor: Arc<dyn ExpressionConvertor>,
     projection_pushdown: bool,
@@ -326,7 +369,7 @@ enum State {
         vxf: VortexFile,
         layout_reader: Arc<dyn LayoutReader>,
         byte_range: Range<u64>,
-        split_ranges: Arc<OnceCell<NaturalSplitRanges>>,
+        split_ranges: Arc<OnceCell<Arc<NaturalSplits>>>,
     },
     PrepareScan {
         state: FileOpenState,
@@ -457,7 +500,7 @@ impl std::fmt::Debug for FileOpenState {
             .field("unified_file_schema", &self.unified_file_schema)
             .field("limit", &self.limit)
             .field("layout_readers", &self.layout_readers)
-            .field("natural_split_ranges", &self.natural_split_ranges)
+            .field("natural_splits", &self.natural_splits)
             .field("has_output_ordering", &self.has_output_ordering)
             .field("expr_convertor", &"<expr_convertor>")
             .field("projection_pushdown", &self.projection_pushdown)
@@ -563,7 +606,7 @@ impl MorselPlanner for VortexMorselPlanner {
                 };
 
                 let split_ranges = natural_split_cell_for_file(
-                    state.natural_split_ranges.as_ref(),
+                    state.natural_splits.as_ref(),
                     &state.file.object_meta.location,
                 );
 
@@ -591,16 +634,17 @@ impl MorselPlanner for VortexMorselPlanner {
                 Ok(Some(
                     MorselPlan::new().with_pending_planner(
                         async move {
-                            let split_ranges = split_ranges
-                                .get_or_try_init(|| async {
-                                    compute_natural_split_ranges(layout_reader.as_ref())
-                                })
-                                .await?;
+                            let natural_splits = Arc::clone(
+                                split_ranges
+                                    .get_or_try_init(|| async {
+                                        compute_natural_splits(layout_reader.as_ref(), total_size)
+                                    })
+                                    .await?,
+                            );
 
                             let new_state = match split_aligned_row_range(
                                 byte_range,
-                                total_size,
-                                split_ranges.as_ref(),
+                                natural_splits.as_ref(),
                             ) {
                                 Some(row_range) => State::PrepareScan {
                                     state,
@@ -899,10 +943,10 @@ fn partial_file_byte_range(file: &PartitionedFile) -> DFResult<Option<Range<u64>
 }
 
 fn natural_split_cell_for_file(
-    natural_split_ranges: &NaturalSplitCache,
+    natural_splits: &NaturalSplitCache,
     path: &Path,
-) -> Arc<OnceCell<NaturalSplitRanges>> {
-    match natural_split_ranges.entry(path.clone()) {
+) -> Arc<OnceCell<Arc<NaturalSplits>>> {
+    match natural_splits.entry(path.clone()) {
         Entry::Occupied(entry) => Arc::clone(entry.get()),
         Entry::Vacant(entry) => {
             let split_ranges = Arc::new(OnceCell::new());
@@ -912,18 +956,20 @@ fn natural_split_cell_for_file(
     }
 }
 
-fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<NaturalSplitRanges> {
+fn compute_natural_splits(
+    layout_reader: &dyn LayoutReader,
+    total_size: u64,
+) -> DFResult<Arc<NaturalSplits>> {
     let row_count = layout_reader.row_count();
     let row_range = 0..row_count;
-    let split_points: Vec<_> = SplitBy::Layout
+    let row_boundaries = SplitBy::Layout
         .splits(layout_reader, &row_range, &[FieldMask::All])
-        .map_err(|e| exec_datafusion_err!("Failed to compute Vortex natural splits: {e}"))?
-        .into_iter()
-        .tuple_windows()
-        .map(|(s, e)| s..e)
-        .collect::<Vec<_>>();
+        .map_err(|e| exec_datafusion_err!("Failed to compute Vortex natural splits: {e}"))?;
 
-    Ok(split_points.into())
+    Ok(Arc::new(NaturalSplits::new(
+        row_boundaries.into(),
+        total_size,
+    )))
 }
 
 /// Translate a DataFusion byte range to the contiguous natural split ranges it owns.
@@ -931,33 +977,25 @@ fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult<Na
 /// byte 0 so a tiny first byte range still claims the first rows.
 fn split_aligned_row_range(
     byte_range: Range<u64>,
-    total_size: u64,
-    split_ranges: &[Range<u64>],
+    natural_splits: &NaturalSplits,
 ) -> Option<Range<u64>> {
     if byte_range.start >= byte_range.end {
         return None;
     }
 
-    let row_count = split_ranges.last().map(|split| split.end)?;
-    if row_count == 0 {
+    let first_split = natural_splits
+        .assignment_bytes
+        .partition_point(|&assignment_byte| assignment_byte < byte_range.start);
+    let after_last_split = natural_splits
+        .assignment_bytes
+        .partition_point(|&assignment_byte| assignment_byte < byte_range.end);
+    if first_split == after_last_split {
         return None;
     }
 
-    let mut owned_splits = split_ranges
-        .iter()
-        .enumerate()
-        .filter_map(|(idx, split_range)| {
-            let assignment_byte = split_assignment_byte(idx, split_range, row_count, total_size);
-            byte_range.contains(&assignment_byte).then_some(split_range)
-        });
-
-    let first_split = owned_splits.next()?;
-    let mut row_range = first_split.start..first_split.end;
-    for split_range in owned_splits {
-        row_range.end = split_range.end;
-    }
-
-    Some(row_range)
+    Some(
+        natural_splits.row_boundaries[first_split]..natural_splits.row_boundaries[after_last_split],
+    )
 }
 
 fn split_assignment_byte(
diff --git a/vortex-datafusion/src/persistent/morsel/tests.rs b/vortex-datafusion/src/persistent/morsel/tests.rs
index 0f533579fb4..6ac05082d38 100644
--- a/vortex-datafusion/src/persistent/morsel/tests.rs
+++ b/vortex-datafusion/src/persistent/morsel/tests.rs
@@ -143,6 +143,16 @@ impl PhysicalExpr for SnapshotErrorExpr {
     }
 }
 
+/// Build [`NaturalSplits`] from contiguous split ranges, the shape the layout walk produces.
+fn natural_splits(total_size: u64, split_ranges: &[Range<u64>]) -> NaturalSplits {
+    let mut row_boundaries = Vec::with_capacity(split_ranges.len() + 1);
+    if let Some(first) = split_ranges.first() {
+        row_boundaries.push(first.start);
+        row_boundaries.extend(split_ranges.iter().map(|range| range.end));
+    }
+    NaturalSplits::new(row_boundaries.into(), total_size)
+}
+
 #[rstest]
 #[case(0..3, 10, vec![0..2, 2..5, 5..10], Some(0..2))]
 #[case(3..7, 10, vec![0..2, 2..5, 5..10], Some(2..5))]
@@ -156,7 +166,7 @@ fn test_split_aligned_row_range(
     #[case] expected: Option<Range<u64>>,
 ) {
     assert_eq!(
-        split_aligned_row_range(byte_range, total_size, &split_ranges),
+        split_aligned_row_range(byte_range, &natural_splits(total_size, &split_ranges)),
         expected
     );
 }
@@ -165,10 +175,11 @@ fn test_split_aligned_row_range(
 fn test_split_aligned_ranges_cover_splits_exactly_once() {
     let split_ranges = vec![0..1, 1..4, 4..10, 10..13];
     let byte_ranges = [0..4, 4..8, 8..12, 12..16];
+    let natural_splits = natural_splits(16, &split_ranges);
 
     let assigned = byte_ranges
         .into_iter()
-        .filter_map(|byte_range| split_aligned_row_range(byte_range, 16, &split_ranges))
+        .filter_map(|byte_range| split_aligned_row_range(byte_range, &natural_splits))
         .collect::<Vec<_>>();
 
     assert_eq!(assigned, vec![0..4, 4..10, 10..13]);
@@ -199,6 +210,16 @@ fn test_split_aligned_ranges_cover_splits_exactly_once() {
     }
 }
 
+/// Splits whose assignment bytes collide must stay with a single byte range.
+#[test]
+fn test_split_aligned_row_range_keeps_colliding_assignments_together() {
+    let natural_splits = natural_splits(2, &[0..1, 1..2, 2..3, 3..4]);
+
+    assert_eq!(natural_splits.assignment_bytes.as_ref(), [0, 0, 1, 1]);
+    assert_eq!(split_aligned_row_range(0..1, &natural_splits), Some(0..2));
+    assert_eq!(split_aligned_row_range(1..2, &natural_splits), Some(2..4));
+}
+
 #[tokio::test]
 async fn test_natural_split_cell_is_shared_by_file_path() {
     let cache = NaturalSplitCache::default();
@@ -217,17 +238,17 @@ async fn test_natural_split_cell_is_shared_by_file_path() {
         first_cell.get_or_init(|| async move {
             first_initializations.fetch_add(1, Ordering::Relaxed);
             tokio::task::yield_now().await;
-            Arc::from([0..5, 5..10])
+            Arc::new(natural_splits(10, &[0..5, 5..10]))
         }),
         second_cell.get_or_init(|| async move {
             second_initializations.fetch_add(1, Ordering::Relaxed);
-            Arc::from([10..15, 15..20])
+            Arc::new(natural_splits(20, &[10..15, 15..20]))
         }),
     );
 
     assert_eq!(initializations.load(Ordering::Relaxed), 1);
     assert!(Arc::ptr_eq(first_ranges, second_ranges));
-    assert_eq!(first_ranges.as_ref(), [0..5, 5..10]);
+    assert_eq!(first_ranges.row_boundaries.as_ref(), [0, 5, 10]);
 }
 
 async fn write_arrow_to_vortex(
@@ -266,7 +287,7 @@ fn make_morselizer(
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
@@ -656,7 +677,7 @@ async fn test_open_files_different_table_schema() -> anyhow::Result<()> {
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
@@ -743,7 +764,7 @@ async fn test_schema_different_column_order() -> anyhow::Result<()> {
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
@@ -895,7 +916,7 @@ async fn test_projection_bug_minimal_repro() -> anyhow::Result<()> {
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
@@ -955,7 +976,7 @@ fn make_test_morselizer(
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
@@ -1160,7 +1181,7 @@ async fn test_projection_expr_pushdown() -> anyhow::Result<()> {
         metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
         df_metrics: ExecutionPlanMetricsSet::new(),
         layout_readers: Default::default(),
-        natural_split_ranges: Default::default(),
+        natural_splits: Default::default(),
         has_output_ordering: false,
         expression_convertor: Arc::new(DefaultExpressionConvertor::default()),
         file_metadata_cache: None,
diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs
index a0945c06131..8ab7625fc2e 100644
--- a/vortex-datafusion/src/persistent/source.rs
+++ b/vortex-datafusion/src/persistent/source.rs
@@ -198,7 +198,7 @@ pub struct VortexSource {
     /// Sharing the readers allows us to only read every layout once from the file, even across partitions.
     layout_readers: Arc<DashMap<Path, Weak<dyn LayoutReader>>>,
     /// Shared full-file natural split ranges keyed by path.
-    natural_split_ranges: Arc<NaturalSplitCache>,
+    natural_splits: Arc<NaturalSplitCache>,
     expression_convertor: Arc<dyn ExpressionConvertor>,
     pub(crate) vortex_reader_factory: Option<Arc<dyn VortexReaderFactory>>,
     pub(crate) ordered: bool,
@@ -231,7 +231,7 @@ impl VortexSource {
             vortex_predicate: None,
             df_metrics: Default::default(),
             layout_readers: Arc::new(DashMap::default()),
-            natural_split_ranges: Arc::new(DashMap::default()),
+            natural_splits: Arc::new(DashMap::default()),
             expression_convertor,
             vortex_reader_factory: None,
             vx_metrics_registry: Arc::new(DefaultMetricsRegistry::default()),
@@ -358,7 +358,7 @@ impl VortexSource {
             metrics_registry: Arc::clone(&self.vx_metrics_registry),
             df_metrics: self.df_metrics.clone(),
             layout_readers: Arc::clone(&self.layout_readers),
-            natural_split_ranges: Arc::clone(&self.natural_split_ranges),
+            natural_splits: Arc::clone(&self.natural_splits),
             has_output_ordering: !base_config.output_ordering.is_empty() || self.ordered,
             expression_convertor: Arc::clone(&self.expression_convertor),
             file_metadata_cache: self.file_metadata_cache.clone(),