Skip to content

Commit b65ff6b

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent e126357 commit b65ff6b

5 files changed

Lines changed: 302 additions & 305 deletions

File tree

‎datafusion-examples/examples/custom_file_format.rs‎

Lines changed: 121 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ use arrow::{
2121
array::{AsArray, RecordBatch, StringArray, UInt8Array},
2222
datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type},
2323
};
24-
use datafusion::physical_expr::LexRequirement;
24+
use datafusion::{logical_expr::statistics::TableStatistics, physical_expr::LexRequirement};
2525
use datafusion::physical_expr::PhysicalExpr;
2626
use datafusion::{
2727
catalog::Session,
28-
common::{GetExt, Statistics},
28+
common::GetExt,
2929
};
3030
use datafusion::{
3131
datasource::physical_plan::FileSource, execution::session_state::SessionStateBuilder,
@@ -67,74 +67,74 @@ impl TSVFileFormat {
6767
}
6868
}
6969

70-
// #[async_trait::async_trait]
71-
// impl FileFormat for TSVFileFormat {
72-
// fn as_any(&self) -> &dyn Any {
73-
// self
74-
// }
75-
76-
// fn get_ext(&self) -> String {
77-
// "tsv".to_string()
78-
// }
79-
80-
// fn get_ext_with_compression(&self, c: &FileCompressionType) -> Result<String> {
81-
// if c == &FileCompressionType::UNCOMPRESSED {
82-
// Ok("tsv".to_string())
83-
// } else {
84-
// todo!("Compression not supported")
85-
// }
86-
// }
87-
88-
// async fn infer_schema(
89-
// &self,
90-
// state: &dyn Session,
91-
// store: &Arc<dyn ObjectStore>,
92-
// objects: &[ObjectMeta],
93-
// ) -> Result<SchemaRef> {
94-
// self.csv_file_format
95-
// .infer_schema(state, store, objects)
96-
// .await
97-
// }
98-
99-
// async fn infer_stats(
100-
// &self,
101-
// state: &dyn Session,
102-
// store: &Arc<dyn ObjectStore>,
103-
// table_schema: SchemaRef,
104-
// object: &ObjectMeta,
105-
// ) -> Result<Statistics> {
106-
// self.csv_file_format
107-
// .infer_stats(state, store, table_schema, object)
108-
// .await
109-
// }
110-
111-
// async fn create_physical_plan(
112-
// &self,
113-
// state: &dyn Session,
114-
// conf: FileScanConfig,
115-
// filters: Option<&Arc<dyn PhysicalExpr>>,
116-
// ) -> Result<Arc<dyn ExecutionPlan>> {
117-
// self.csv_file_format
118-
// .create_physical_plan(state, conf, filters)
119-
// .await
120-
// }
121-
122-
// async fn create_writer_physical_plan(
123-
// &self,
124-
// input: Arc<dyn ExecutionPlan>,
125-
// state: &dyn Session,
126-
// conf: FileSinkConfig,
127-
// order_requirements: Option<LexRequirement>,
128-
// ) -> Result<Arc<dyn ExecutionPlan>> {
129-
// self.csv_file_format
130-
// .create_writer_physical_plan(input, state, conf, order_requirements)
131-
// .await
132-
// }
133-
134-
// fn file_source(&self) -> Arc<dyn FileSource> {
135-
// self.csv_file_format.file_source()
136-
// }
137-
// }
70+
#[async_trait::async_trait]
71+
impl FileFormat for TSVFileFormat {
72+
fn as_any(&self) -> &dyn Any {
73+
self
74+
}
75+
76+
fn get_ext(&self) -> String {
77+
"tsv".to_string()
78+
}
79+
80+
fn get_ext_with_compression(&self, c: &FileCompressionType) -> Result<String> {
81+
if c == &FileCompressionType::UNCOMPRESSED {
82+
Ok("tsv".to_string())
83+
} else {
84+
todo!("Compression not supported")
85+
}
86+
}
87+
88+
async fn infer_schema(
89+
&self,
90+
state: &dyn Session,
91+
store: &Arc<dyn ObjectStore>,
92+
objects: &[ObjectMeta],
93+
) -> Result<SchemaRef> {
94+
self.csv_file_format
95+
.infer_schema(state, store, objects)
96+
.await
97+
}
98+
99+
async fn infer_stats(
100+
&self,
101+
state: &dyn Session,
102+
store: &Arc<dyn ObjectStore>,
103+
table_schema: SchemaRef,
104+
object: &ObjectMeta,
105+
) -> Result<TableStatistics> {
106+
self.csv_file_format
107+
.infer_stats(state, store, table_schema, object)
108+
.await
109+
}
110+
111+
async fn create_physical_plan(
112+
&self,
113+
state: &dyn Session,
114+
conf: FileScanConfig,
115+
filters: Option<&Arc<dyn PhysicalExpr>>,
116+
) -> Result<Arc<dyn ExecutionPlan>> {
117+
self.csv_file_format
118+
.create_physical_plan(state, conf, filters)
119+
.await
120+
}
121+
122+
async fn create_writer_physical_plan(
123+
&self,
124+
input: Arc<dyn ExecutionPlan>,
125+
state: &dyn Session,
126+
conf: FileSinkConfig,
127+
order_requirements: Option<LexRequirement>,
128+
) -> Result<Arc<dyn ExecutionPlan>> {
129+
self.csv_file_format
130+
.create_writer_physical_plan(input, state, conf, order_requirements)
131+
.await
132+
}
133+
134+
fn file_source(&self) -> Arc<dyn FileSource> {
135+
self.csv_file_format.file_source()
136+
}
137+
}
138138

139139
#[derive(Default, Debug)]
140140
/// Factory for creating TSV file formats
@@ -153,29 +153,29 @@ impl TSVFileFactory {
153153
}
154154
}
155155

156-
// impl FileFormatFactory for TSVFileFactory {
157-
// fn create(
158-
// &self,
159-
// state: &dyn Session,
160-
// format_options: &std::collections::HashMap<String, String>,
161-
// ) -> Result<Arc<dyn FileFormat>> {
162-
// let mut new_options = format_options.clone();
163-
// new_options.insert("format.delimiter".to_string(), "\t".to_string());
156+
impl FileFormatFactory for TSVFileFactory {
157+
fn create(
158+
&self,
159+
state: &dyn Session,
160+
format_options: &std::collections::HashMap<String, String>,
161+
) -> Result<Arc<dyn FileFormat>> {
162+
let mut new_options = format_options.clone();
163+
new_options.insert("format.delimiter".to_string(), "\t".to_string());
164164

165-
// let csv_file_format = self.csv_file_factory.create(state, &new_options)?;
166-
// let tsv_file_format = Arc::new(TSVFileFormat::new(csv_file_format));
165+
let csv_file_format = self.csv_file_factory.create(state, &new_options)?;
166+
let tsv_file_format = Arc::new(TSVFileFormat::new(csv_file_format));
167167

168-
// Ok(tsv_file_format)
169-
// }
168+
Ok(tsv_file_format)
169+
}
170170

171-
// fn default(&self) -> Arc<dyn FileFormat> {
172-
// todo!()
173-
// }
171+
fn default(&self) -> Arc<dyn FileFormat> {
172+
todo!()
173+
}
174174

175-
// fn as_any(&self) -> &dyn Any {
176-
// self
177-
// }
178-
// }
175+
fn as_any(&self) -> &dyn Any {
176+
self
177+
}
178+
}
179179

180180
impl GetExt for TSVFileFactory {
181181
fn get_ext(&self) -> String {
@@ -185,38 +185,38 @@ impl GetExt for TSVFileFactory {
185185

186186
#[tokio::main]
187187
async fn main() -> Result<()> {
188-
// // Create a new context with the default configuration
189-
// let mut state = SessionStateBuilder::new().with_default_features().build();
190-
191-
// // Register the custom file format
192-
// let file_format = Arc::new(TSVFileFactory::new());
193-
// state.register_file_format(file_format, true).unwrap();
194-
195-
// // Create a new context with the custom file format
196-
// let ctx = SessionContext::new_with_state(state);
197-
198-
// let mem_table = create_mem_table();
199-
// ctx.register_table("mem_table", mem_table).unwrap();
200-
201-
// let temp_dir = tempdir().unwrap();
202-
// let table_save_path = temp_dir.path().join("mem_table.tsv");
203-
204-
// let d = ctx
205-
// .sql(&format!(
206-
// "COPY mem_table TO '{}' STORED AS TSV;",
207-
// table_save_path.display(),
208-
// ))
209-
// .await?;
210-
211-
// let results = d.collect().await?;
212-
// println!(
213-
// "Number of inserted rows: {:?}",
214-
// (results[0]
215-
// .column_by_name("count")
216-
// .unwrap()
217-
// .as_primitive::<UInt64Type>()
218-
// .value(0))
219-
// );
188+
// Create a new context with the default configuration
189+
let mut state = SessionStateBuilder::new().with_default_features().build();
190+
191+
// Register the custom file format
192+
let file_format = Arc::new(TSVFileFactory::new());
193+
state.register_file_format(file_format, true).unwrap();
194+
195+
// Create a new context with the custom file format
196+
let ctx = SessionContext::new_with_state(state);
197+
198+
let mem_table = create_mem_table();
199+
ctx.register_table("mem_table", mem_table).unwrap();
200+
201+
let temp_dir = tempdir().unwrap();
202+
let table_save_path = temp_dir.path().join("mem_table.tsv");
203+
204+
let d = ctx
205+
.sql(&format!(
206+
"COPY mem_table TO '{}' STORED AS TSV;",
207+
table_save_path.display(),
208+
))
209+
.await?;
210+
211+
let results = d.collect().await?;
212+
println!(
213+
"Number of inserted rows: {:?}",
214+
(results[0]
215+
.column_by_name("count")
216+
.unwrap()
217+
.as_primitive::<UInt64Type>()
218+
.value(0))
219+
);
220220

221221
Ok(())
222222
}

0 commit comments

Comments
 (0)