@@ -72,6 +72,16 @@ use async_trait::async_trait;
7272use datafusion_catalog:: Session ;
7373use datafusion_sql:: TableReference ;
7474
75+ use std:: sync:: Arc ;
76+ use uuid:: Uuid ;
77+
78+ use crate :: dataframe:: DataFrame ;
79+ use crate :: error:: Result ;
80+ use crate :: execution:: context:: SessionContext ;
81+ use crate :: logical_plan:: LogicalPlan ;
82+ use crate :: physical_plan:: collect_partitioned;
83+ use crate :: physical_plan:: memory:: MemTable ; // for returning Result<DataFrame>
84+
7585/// Contains options that control how data is
7686/// written out from a DataFrame
7787pub struct DataFrameWriteOptions {
@@ -232,19 +242,31 @@ pub struct DataFrame {
232242 projection_requires_validation : bool ,
233243}
234244
235- impl DataFrame {
236- /// Create a new `DataFrame ` based on an existing `LogicalPlan`
237- ///
238- /// This is a low-level method and is not typically used by end users. See
239- /// [`SessionContext::read_csv`] and other methods for creating a
240- /// `DataFrame` from an existing datasource.
241- pub fn new ( session_state : SessionState , plan : LogicalPlan ) -> Self {
242- Self {
243- session_state : Box :: new ( session_state) ,
244- plan,
245- projection_requires_validation : true ,
246- }
245+ pub async fn cache ( self ) -> Result < DataFrame > {
246+ if self . session_state . config . local_cache {
247+ // Eager caching (current behavior)
248+ let context = SessionContext :: new_with_state ( ( * self . session_state ) . clone ( ) ) ;
249+ let plan = self . clone ( ) . create_physical_plan ( ) . await ?;
250+ let schema = plan. schema ( ) ;
251+
252+ // collect_partitioned now only needs the plan
253+ let partitions = collect_partitioned ( plan) . await ?;
254+
255+ let mem_table = MemTable :: try_new ( schema, partitions) ?;
256+ context. read_table ( Arc :: new ( mem_table) )
257+ } else {
258+ // Lazy caching: return LogicalPlan::Cache
259+ let lineage = self . to_logical_plan ( ) ; // get the logical plan so far
260+ Ok ( DataFrame :: new (
261+ ( * self . session_state ) . clone ( ) ,
262+ LogicalPlan :: Cache {
263+ id : Uuid :: new_v4 ( ) . to_string ( ) ,
264+ lineage : Arc :: new ( lineage) ,
265+ } ,
266+ ) )
247267 }
268+ }
269+
248270
249271 /// Creates logical expression from a SQL query text.
250272 /// The expression is created and processed against the current schema.
0 commit comments