Skip to content

Commit 4b30f65

Browse files
committed
feat: add configurable cache mode (local_cache) with LogicalPlan::Cache (apache#17297)
1 parent fd7df66 commit 4b30f65

2 files changed

Lines changed: 47 additions & 12 deletions

File tree

‎datafusion/core/src/dataframe/mod.rs‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ use async_trait::async_trait;
7272
use datafusion_catalog::Session;
7373
use datafusion_sql::TableReference;
7474

75+
use std::sync::Arc;
76+
use uuid::Uuid;
77+
78+
use crate::dataframe::DataFrame;
79+
use crate::error::Result;
80+
use crate::execution::context::SessionContext;
81+
use crate::logical_plan::LogicalPlan;
82+
use crate::physical_plan::collect_partitioned;
83+
use crate::physical_plan::memory::MemTable; // for returning Result<DataFrame>
84+
7585
/// Contains options that control how data is
7686
/// written out from a DataFrame
7787
pub struct DataFrameWriteOptions {
@@ -232,19 +242,31 @@ pub struct DataFrame {
232242
projection_requires_validation: bool,
233243
}
234244

235-
impl DataFrame {
236-
/// Create a new `DataFrame ` based on an existing `LogicalPlan`
237-
///
238-
/// This is a low-level method and is not typically used by end users. See
239-
/// [`SessionContext::read_csv`] and other methods for creating a
240-
/// `DataFrame` from an existing datasource.
241-
pub fn new(session_state: SessionState, plan: LogicalPlan) -> Self {
242-
Self {
243-
session_state: Box::new(session_state),
244-
plan,
245-
projection_requires_validation: true,
246-
}
245+
pub async fn cache(self) -> Result<DataFrame> {
246+
if self.session_state.config.local_cache {
247+
// Eager caching (current behavior)
248+
let context = SessionContext::new_with_state((*self.session_state).clone());
249+
let plan = self.clone().create_physical_plan().await?;
250+
let schema = plan.schema();
251+
252+
// collect_partitioned now only needs the plan
253+
let partitions = collect_partitioned(plan).await?;
254+
255+
let mem_table = MemTable::try_new(schema, partitions)?;
256+
context.read_table(Arc::new(mem_table))
257+
} else {
258+
// Lazy caching: return LogicalPlan::Cache
259+
let lineage = self.to_logical_plan(); // get the logical plan so far
260+
Ok(DataFrame::new(
261+
(*self.session_state).clone(),
262+
LogicalPlan::Cache {
263+
id: Uuid::new_v4().to_string(),
264+
lineage: Arc::new(lineage),
265+
},
266+
))
247267
}
268+
}
269+
248270

249271
/// Creates logical expression from a SQL query text.
250272
/// The expression is created and processed against the current schema.

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
//! Planner for [`LogicalPlan`] to [`ExecutionPlan`]
1919
20+
use datafusion::physical_plan::memory::MemoryExec;
2021
use std::borrow::Cow;
2122
use std::collections::HashMap;
2223
use std::sync::Arc;
@@ -946,6 +947,18 @@ impl DefaultPhysicalPlanner {
946947
))
947948
}
948949

950+
// Handle caching logical plan
951+
LogicalPlan::Cache { id, lineage } => {
952+
// Step 1: Plan the child logical plan first
953+
let child_exec =
954+
self.create_physical_plan(lineage, session_state).await?;
955+
956+
// Step 2: Wrap in a caching execution plan if you want to actually cache in-memory
957+
// For now, we just pass through the child plan
958+
// TODO: later you can wrap this in a memory cache executor
959+
child_exec
960+
}
961+
949962
// 2 Children
950963
LogicalPlan::Join(Join {
951964
left: original_left,

0 commit comments

Comments
 (0)