Skip to content

Commit 74881dc

Browse files
timsaucerclaude
authored andcommitted
feat: Implement FFI_QueryPlanner (apache#24028)
## Which issue does this PR close? - Unblocks apache/datafusion-python#1612 ## Rationale for this change This is the last in a series of PRs that would enable FFI `Session` to support a `QueryPlanner`. The prior work was in - apache#23649 - apache#23703 - apache#23842 With those changes in place we now have the dependencies correct that we can expose a `FFI_QueryPlanner` on a `FFI_Session`. With this we can enable foreign libraries such as `datafusion-distributed` and `ballista` to provide a query planner in Python and connect it directly to a `datafusion-python`'s `SessionContext`. ## What changes are included in this PR? Addition only. Adds these functions to `FFI_Session` and their supporting structures: - `query_planner()` - `optimize()` - `physical_optimizers()` ## Are these changes tested? Unit and integration tests are provided. ## Are there any user-facing changes? This is addition, but it does break the FFI ABI, which is already evolving in DF55. --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
1 parent efe6abd commit 74881dc

16 files changed

Lines changed: 1299 additions & 40 deletions

File tree

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ async fn get_file_decryption_properties(
297297
}
298298

299299
#[cfg(not(feature = "parquet_encryption"))]
300+
#[expect(clippy::unused_async)]
300301
async fn get_file_decryption_properties(
301302
_state: &dyn Session,
302303
_options: &TableParquetOptions,

‎datafusion/datasource-parquet/src/opener/encryption.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ impl EncryptionContext {
7676

7777
#[cfg(not(feature = "parquet_encryption"))]
7878
#[expect(dead_code)]
79+
#[expect(clippy::unused_async)]
7980
impl EncryptionContext {
8081
pub(super) async fn get_file_decryption_properties(
8182
&self,

‎datafusion/datasource-parquet/src/sink.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ async fn set_writer_encryption_properties(
243243
}
244244

245245
#[cfg(not(feature = "parquet_encryption"))]
246+
#[expect(clippy::unused_async)]
246247
async fn set_writer_encryption_properties(
247248
builder: WriterPropertiesBuilder,
248249
_runtime: &Arc<RuntimeEnv>,

‎datafusion/ffi/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ pub mod physical_optimizer;
3939
pub mod placement;
4040
pub mod plan_properties;
4141
pub mod proto;
42+
pub mod query_planner;
4243
pub mod record_batch_stream;
4344
pub mod schema_provider;
4445
pub mod session;

‎datafusion/ffi/src/proto/logical_extension_codec.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ pub struct FFI_LogicalExtensionCodec {
9999
try_encode_udwf:
100100
unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFI_Result<SVec<u8>>,
101101

102-
pub task_ctx_provider: FFI_TaskContextProvider,
102+
pub(crate) task_ctx_provider: FFI_TaskContextProvider,
103103

104104
/// Used to create a clone on the provider of the execution plan. This should
105105
/// only need to be called by the receiver of the plan.
@@ -295,7 +295,7 @@ impl Drop for FFI_LogicalExtensionCodec {
295295
impl FFI_LogicalExtensionCodec {
296296
/// Creates a new [`FFI_LogicalExtensionCodec`].
297297
pub fn new(
298-
codec: Arc<dyn LogicalExtensionCodec + Send>,
298+
codec: Arc<dyn LogicalExtensionCodec>,
299299
runtime: Option<Handle>,
300300
task_ctx_provider: impl Into<FFI_TaskContextProvider>,
301301
) -> Self {
@@ -712,14 +712,12 @@ mod tests {
712712

713713
#[test]
714714
fn ffi_logical_extension_codec_local_bypass() {
715-
let codec =
716-
Arc::new(TestExtensionCodec {}) as Arc<dyn LogicalExtensionCodec + Send>;
715+
let codec = Arc::new(TestExtensionCodec {}) as Arc<dyn LogicalExtensionCodec>;
717716
let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();
718717

719718
let mut ffi_codec =
720719
FFI_LogicalExtensionCodec::new(Arc::clone(&codec), None, task_ctx_provider);
721720

722-
let codec = codec as Arc<dyn LogicalExtensionCodec>;
723721
// Verify local libraries can be downcast to their original
724722
let foreign_codec: Arc<dyn LogicalExtensionCodec> = (&ffi_codec).into();
725723
assert!(arc_ptr_eq(&foreign_codec, &codec));

‎datafusion/ffi/src/proto/physical_extension_codec.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ pub struct FFI_PhysicalExtensionCodec {
9292
unsafe extern "C" fn(&Self, node: FFI_WindowUDF) -> FFI_Result<SVec<u8>>,
9393

9494
/// Access the current [`TaskContext`].
95-
task_ctx_provider: FFI_TaskContextProvider,
95+
pub(crate) task_ctx_provider: FFI_TaskContextProvider,
9696

9797
/// Used to create a clone on the provider of the execution plan. This should
9898
/// only need to be called by the receiver of the plan.
@@ -281,7 +281,7 @@ impl Drop for FFI_PhysicalExtensionCodec {
281281
impl FFI_PhysicalExtensionCodec {
282282
/// Creates a new [`FFI_PhysicalExtensionCodec`].
283283
pub fn new(
284-
codec: Arc<dyn PhysicalExtensionCodec + Send>,
284+
codec: Arc<dyn PhysicalExtensionCodec>,
285285
runtime: Option<Handle>,
286286
task_ctx_provider: impl Into<FFI_TaskContextProvider>,
287287
) -> Self {
@@ -695,14 +695,12 @@ pub(crate) mod tests {
695695

696696
#[test]
697697
fn ffi_physical_extension_codec_local_bypass() {
698-
let codec =
699-
Arc::new(TestExtensionCodec {}) as Arc<dyn PhysicalExtensionCodec + Send>;
698+
let codec = Arc::new(TestExtensionCodec {}) as Arc<dyn PhysicalExtensionCodec>;
700699
let (_ctx, task_ctx_provider) = crate::util::tests::test_session_and_ctx();
701700

702701
let mut ffi_codec =
703702
FFI_PhysicalExtensionCodec::new(Arc::clone(&codec), None, task_ctx_provider);
704703

705-
let codec = codec as Arc<dyn PhysicalExtensionCodec>;
706704
// Verify local libraries can be downcast to their original
707705
let foreign_codec: Arc<dyn PhysicalExtensionCodec> = (&ffi_codec).into();
708706
assert!(arc_ptr_eq(&foreign_codec, &codec));

0 commit comments

Comments
 (0)