Skip to content

Commit 9f89821

Browse files
morristaiclaude
andauthored
feat: allow downcasting ParquetFileReaderFactory (apache#26635)
## Which issue does this PR close? - Closes apache#26633. ## Rationale for this change There is no way to tell which reader factory a `ParquetSource` uses. `parquet_file_reader_factory()` returns `&Arc<dyn ParquetFileReaderFactory>`, and the trait has no downcast path. ## What changes are included in this PR? `ParquetFileReaderFactory` gets `Any` as a supertrait, and `dyn ParquetFileReaderFactory` gets `is` and `downcast_ref`, like `dyn FileSource` in apache#21576. I also added a 57.0.0 upgrade guide entry, like the ones for `MemoryPool` (apache#21803) and `QueryPlanner` (apache#22241). ## What is the testing strategy for this PR? A unit test in `source.rs` sets each built-in factory on a `ParquetSource` and checks that `parquet_file_reader_factory()` downcasts to it. ## Are there any user-facing changes? Two new methods on `dyn ParquetFileReaderFactory`. The semver check will flag the new supertrait, but the trait already required `'static`, so existing implementations compile unchanged. Drafted with Claude. I reviewed the change and this text. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent a8c4feb commit 9f89821

3 files changed

Lines changed: 129 additions & 1 deletion

File tree

‎datafusion/datasource-parquet/src/reader.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ pub trait ParquetFileSchemaProvider: Debug + Send + Sync + 'static {
5555
/// such as pre-cached metadata, I/O coalescing, etc.
5656
///
5757
/// See [`DefaultParquetFileReaderFactory`] for a simple implementation.
58-
pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
58+
pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync {
5959
/// Provides an `AsyncFileReader` for reading data from a parquet file specified
6060
///
6161
/// # Notes
@@ -79,6 +79,18 @@ pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
7979
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>>;
8080
}
8181

82+
impl dyn ParquetFileReaderFactory {
83+
/// Returns `true` if this factory is of type `T`.
84+
pub fn is<T: ParquetFileReaderFactory>(&self) -> bool {
85+
(self as &dyn Any).is::<T>()
86+
}
87+
88+
/// Attempts to downcast this factory to a concrete type `T`.
89+
pub fn downcast_ref<T: ParquetFileReaderFactory>(&self) -> Option<&T> {
90+
(self as &dyn Any).downcast_ref()
91+
}
92+
}
93+
8294
/// Default implementation of [`ParquetFileReaderFactory`]
8395
///
8496
/// This implementation:

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1518,6 +1518,107 @@ mod tests {
15181518
assert!(source.filter().is_some());
15191519
}
15201520

1521+
#[test]
1522+
fn parquet_file_reader_factory_downcasts_to_concrete_type() {
1523+
use crate::CachedParquetFileReaderFactory;
1524+
use datafusion_datasource::PartitionedFile;
1525+
use datafusion_execution::cache::cache_manager::FileMetadataCache;
1526+
use datafusion_execution::cache::default_cache::DefaultCache;
1527+
use object_store::memory::InMemory;
1528+
use parquet::arrow::async_reader::AsyncFileReader;
1529+
1530+
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
1531+
let source = ParquetSource::new(Arc::new(Schema::empty()))
1532+
.with_parquet_file_reader_factory(Arc::new(
1533+
DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
1534+
));
1535+
let factory = source.parquet_file_reader_factory().unwrap();
1536+
assert!(factory.is::<DefaultParquetFileReaderFactory>());
1537+
assert!(
1538+
factory
1539+
.downcast_ref::<DefaultParquetFileReaderFactory>()
1540+
.is_some()
1541+
);
1542+
assert!(
1543+
factory
1544+
.downcast_ref::<CachedParquetFileReaderFactory>()
1545+
.is_none()
1546+
);
1547+
1548+
let cache: Arc<FileMetadataCache> = Arc::new(DefaultCache::new(1024));
1549+
let source = ParquetSource::new(Arc::new(Schema::empty()))
1550+
.with_parquet_file_reader_factory(Arc::new(
1551+
CachedParquetFileReaderFactory::new(Arc::clone(&store), cache),
1552+
));
1553+
let factory = source.parquet_file_reader_factory().unwrap();
1554+
assert!(factory.is::<CachedParquetFileReaderFactory>());
1555+
assert!(
1556+
factory
1557+
.downcast_ref::<CachedParquetFileReaderFactory>()
1558+
.is_some()
1559+
);
1560+
assert!(
1561+
factory
1562+
.downcast_ref::<DefaultParquetFileReaderFactory>()
1563+
.is_none()
1564+
);
1565+
1566+
// A custom factory whose Debug output resembles a built-in factory.
1567+
// Verifies that concrete-type downcasting avoids false-positive matches
1568+
// from Debug substring checks (issue #26633).
1569+
struct CustomParquetFileReaderFactory {
1570+
store: Arc<dyn ObjectStore>,
1571+
}
1572+
1573+
impl Debug for CustomParquetFileReaderFactory {
1574+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1575+
f.debug_struct("DefaultParquetFileReaderFactory")
1576+
.field("store", &self.store)
1577+
.finish()
1578+
}
1579+
}
1580+
1581+
impl ParquetFileReaderFactory for CustomParquetFileReaderFactory {
1582+
fn create_reader(
1583+
&self,
1584+
_partition_index: usize,
1585+
_partitioned_file: PartitionedFile,
1586+
_metadata_size_hint: Option<usize>,
1587+
_metrics: &ExecutionPlanMetricsSet,
1588+
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {
1589+
unimplemented!()
1590+
}
1591+
}
1592+
1593+
let custom_factory = Arc::new(CustomParquetFileReaderFactory { store });
1594+
assert!(
1595+
format!("{custom_factory:?}")
1596+
.starts_with("DefaultParquetFileReaderFactory { store:")
1597+
);
1598+
1599+
let source = ParquetSource::new(Arc::new(Schema::empty()))
1600+
.with_parquet_file_reader_factory(custom_factory);
1601+
let factory = source.parquet_file_reader_factory().unwrap();
1602+
assert!(!factory.is::<DefaultParquetFileReaderFactory>());
1603+
assert!(!factory.is::<CachedParquetFileReaderFactory>());
1604+
assert!(
1605+
factory
1606+
.downcast_ref::<DefaultParquetFileReaderFactory>()
1607+
.is_none()
1608+
);
1609+
assert!(
1610+
factory
1611+
.downcast_ref::<CachedParquetFileReaderFactory>()
1612+
.is_none()
1613+
);
1614+
assert!(factory.is::<CustomParquetFileReaderFactory>());
1615+
assert!(
1616+
factory
1617+
.downcast_ref::<CustomParquetFileReaderFactory>()
1618+
.is_some()
1619+
);
1620+
}
1621+
15211622
/// Render a `ParquetSource`'s `fmt_extra` output as a `String` for
15221623
/// inspection in tests.
15231624
fn render_fmt_extra(source: &ParquetSource, t: DisplayFormatType) -> String {

‎docs/source/library-user-guide/upgrading/57.0.0.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,18 @@ available without the Spark function library:
6262
```sql
6363
SET datafusion.sql_parser.dialect = 'spark';
6464
```
65+
66+
### `ParquetFileReaderFactory` adds `Any` as a supertrait
67+
68+
To enable downcasting of `dyn ParquetFileReaderFactory` to concrete factory
69+
types (via `is::<T>()` / `downcast_ref::<T>()`), the
70+
`ParquetFileReaderFactory` trait now has `Any` as a supertrait:
71+
72+
```diff
73+
- pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static
74+
+ pub trait ParquetFileReaderFactory: Any + Debug + Send + Sync
75+
```
76+
77+
The trait already required `'static`, which `Any` implies, so existing
78+
implementations need no changes. `ParquetSource::parquet_file_reader_factory`
79+
returns the trait object to downcast.

0 commit comments

Comments
 (0)