Skip to content

Commit 739cadc

Browse files
author
Sergey Zhukov
committed
Statistics: Migrate to Distribution from Precision (apache#14896)
1 parent 9d9e912 commit 739cadc

12 files changed

Lines changed: 431 additions & 359 deletions

File tree

‎datafusion-examples/examples/advanced_parquet_index.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ impl TableProvider for IndexTableProvider {
498498
// provide the factory to create parquet reader without re-reading metadata
499499
.with_parquet_file_reader_factory(Arc::new(reader_factory)),
500500
);
501-
let file_scan_config = FileScanConfig::new(object_store_url, schema, file_source)
501+
let file_scan_config = FileScanConfig::new(object_store_url, schema, file_source)?
502502
.with_limit(limit)
503503
.with_projection(projection.cloned())
504504
.with_file(partitioned_file);

‎datafusion-examples/examples/csv_json_opener.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ async fn csv_opener() -> Result<()> {
6060
ObjectStoreUrl::local_filesystem(),
6161
Arc::clone(&schema),
6262
Arc::new(CsvSource::default()),
63-
)
63+
)?
6464
.with_projection(Some(vec![12, 0]))
6565
.with_limit(Some(5))
6666
.with_file(PartitionedFile::new(path.display().to_string(), 10));
@@ -125,7 +125,7 @@ async fn json_opener() -> Result<()> {
125125
ObjectStoreUrl::local_filesystem(),
126126
schema,
127127
Arc::new(JsonSource::default()),
128-
)
128+
)?
129129
.with_projection(Some(vec![1, 0]))
130130
.with_limit(Some(5))
131131
.with_file(PartitionedFile::new(path.to_string(), 10));

‎datafusion-examples/examples/custom_file_format.rs‎

Lines changed: 119 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -67,74 +67,74 @@ impl TSVFileFormat {
6767
}
6868
}
6969

70-
#[async_trait::async_trait]
71-
impl FileFormat for TSVFileFormat {
72-
fn as_any(&self) -> &dyn Any {
73-
self
74-
}
75-
76-
fn get_ext(&self) -> String {
77-
"tsv".to_string()
78-
}
79-
80-
fn get_ext_with_compression(&self, c: &FileCompressionType) -> Result<String> {
81-
if c == &FileCompressionType::UNCOMPRESSED {
82-
Ok("tsv".to_string())
83-
} else {
84-
todo!("Compression not supported")
85-
}
86-
}
87-
88-
async fn infer_schema(
89-
&self,
90-
state: &dyn Session,
91-
store: &Arc<dyn ObjectStore>,
92-
objects: &[ObjectMeta],
93-
) -> Result<SchemaRef> {
94-
self.csv_file_format
95-
.infer_schema(state, store, objects)
96-
.await
97-
}
98-
99-
async fn infer_stats(
100-
&self,
101-
state: &dyn Session,
102-
store: &Arc<dyn ObjectStore>,
103-
table_schema: SchemaRef,
104-
object: &ObjectMeta,
105-
) -> Result<Statistics> {
106-
self.csv_file_format
107-
.infer_stats(state, store, table_schema, object)
108-
.await
109-
}
110-
111-
async fn create_physical_plan(
112-
&self,
113-
state: &dyn Session,
114-
conf: FileScanConfig,
115-
filters: Option<&Arc<dyn PhysicalExpr>>,
116-
) -> Result<Arc<dyn ExecutionPlan>> {
117-
self.csv_file_format
118-
.create_physical_plan(state, conf, filters)
119-
.await
120-
}
121-
122-
async fn create_writer_physical_plan(
123-
&self,
124-
input: Arc<dyn ExecutionPlan>,
125-
state: &dyn Session,
126-
conf: FileSinkConfig,
127-
order_requirements: Option<LexRequirement>,
128-
) -> Result<Arc<dyn ExecutionPlan>> {
129-
self.csv_file_format
130-
.create_writer_physical_plan(input, state, conf, order_requirements)
131-
.await
132-
}
133-
134-
fn file_source(&self) -> Arc<dyn FileSource> {
135-
self.csv_file_format.file_source()
136-
}
137-
}
70+
// #[async_trait::async_trait]
71+
// impl FileFormat for TSVFileFormat {
72+
// fn as_any(&self) -> &dyn Any {
73+
// self
74+
// }
75+
76+
// fn get_ext(&self) -> String {
77+
// "tsv".to_string()
78+
// }
79+
80+
// fn get_ext_with_compression(&self, c: &FileCompressionType) -> Result<String> {
81+
// if c == &FileCompressionType::UNCOMPRESSED {
82+
// Ok("tsv".to_string())
83+
// } else {
84+
// todo!("Compression not supported")
85+
// }
86+
// }
87+
88+
// async fn infer_schema(
89+
// &self,
90+
// state: &dyn Session,
91+
// store: &Arc<dyn ObjectStore>,
92+
// objects: &[ObjectMeta],
93+
// ) -> Result<SchemaRef> {
94+
// self.csv_file_format
95+
// .infer_schema(state, store, objects)
96+
// .await
97+
// }
98+
99+
// async fn infer_stats(
100+
// &self,
101+
// state: &dyn Session,
102+
// store: &Arc<dyn ObjectStore>,
103+
// table_schema: SchemaRef,
104+
// object: &ObjectMeta,
105+
// ) -> Result<Statistics> {
106+
// self.csv_file_format
107+
// .infer_stats(state, store, table_schema, object)
108+
// .await
109+
// }
110+
111+
// async fn create_physical_plan(
112+
// &self,
113+
// state: &dyn Session,
114+
// conf: FileScanConfig,
115+
// filters: Option<&Arc<dyn PhysicalExpr>>,
116+
// ) -> Result<Arc<dyn ExecutionPlan>> {
117+
// self.csv_file_format
118+
// .create_physical_plan(state, conf, filters)
119+
// .await
120+
// }
121+
122+
// async fn create_writer_physical_plan(
123+
// &self,
124+
// input: Arc<dyn ExecutionPlan>,
125+
// state: &dyn Session,
126+
// conf: FileSinkConfig,
127+
// order_requirements: Option<LexRequirement>,
128+
// ) -> Result<Arc<dyn ExecutionPlan>> {
129+
// self.csv_file_format
130+
// .create_writer_physical_plan(input, state, conf, order_requirements)
131+
// .await
132+
// }
133+
134+
// fn file_source(&self) -> Arc<dyn FileSource> {
135+
// self.csv_file_format.file_source()
136+
// }
137+
// }
138138

139139
#[derive(Default, Debug)]
140140
/// Factory for creating TSV file formats
@@ -153,29 +153,29 @@ impl TSVFileFactory {
153153
}
154154
}
155155

156-
impl FileFormatFactory for TSVFileFactory {
157-
fn create(
158-
&self,
159-
state: &dyn Session,
160-
format_options: &std::collections::HashMap<String, String>,
161-
) -> Result<Arc<dyn FileFormat>> {
162-
let mut new_options = format_options.clone();
163-
new_options.insert("format.delimiter".to_string(), "\t".to_string());
156+
// impl FileFormatFactory for TSVFileFactory {
157+
// fn create(
158+
// &self,
159+
// state: &dyn Session,
160+
// format_options: &std::collections::HashMap<String, String>,
161+
// ) -> Result<Arc<dyn FileFormat>> {
162+
// let mut new_options = format_options.clone();
163+
// new_options.insert("format.delimiter".to_string(), "\t".to_string());
164164

165-
let csv_file_format = self.csv_file_factory.create(state, &new_options)?;
166-
let tsv_file_format = Arc::new(TSVFileFormat::new(csv_file_format));
165+
// let csv_file_format = self.csv_file_factory.create(state, &new_options)?;
166+
// let tsv_file_format = Arc::new(TSVFileFormat::new(csv_file_format));
167167

168-
Ok(tsv_file_format)
169-
}
168+
// Ok(tsv_file_format)
169+
// }
170170

171-
fn default(&self) -> Arc<dyn FileFormat> {
172-
todo!()
173-
}
171+
// fn default(&self) -> Arc<dyn FileFormat> {
172+
// todo!()
173+
// }
174174

175-
fn as_any(&self) -> &dyn Any {
176-
self
177-
}
178-
}
175+
// fn as_any(&self) -> &dyn Any {
176+
// self
177+
// }
178+
// }
179179

180180
impl GetExt for TSVFileFactory {
181181
fn get_ext(&self) -> String {
@@ -185,38 +185,38 @@ impl GetExt for TSVFileFactory {
185185

186186
#[tokio::main]
187187
async fn main() -> Result<()> {
188-
// Create a new context with the default configuration
189-
let mut state = SessionStateBuilder::new().with_default_features().build();
190-
191-
// Register the custom file format
192-
let file_format = Arc::new(TSVFileFactory::new());
193-
state.register_file_format(file_format, true).unwrap();
194-
195-
// Create a new context with the custom file format
196-
let ctx = SessionContext::new_with_state(state);
197-
198-
let mem_table = create_mem_table();
199-
ctx.register_table("mem_table", mem_table).unwrap();
200-
201-
let temp_dir = tempdir().unwrap();
202-
let table_save_path = temp_dir.path().join("mem_table.tsv");
203-
204-
let d = ctx
205-
.sql(&format!(
206-
"COPY mem_table TO '{}' STORED AS TSV;",
207-
table_save_path.display(),
208-
))
209-
.await?;
210-
211-
let results = d.collect().await?;
212-
println!(
213-
"Number of inserted rows: {:?}",
214-
(results[0]
215-
.column_by_name("count")
216-
.unwrap()
217-
.as_primitive::<UInt64Type>()
218-
.value(0))
219-
);
188+
// // Create a new context with the default configuration
189+
// let mut state = SessionStateBuilder::new().with_default_features().build();
190+
191+
// // Register the custom file format
192+
// let file_format = Arc::new(TSVFileFactory::new());
193+
// state.register_file_format(file_format, true).unwrap();
194+
195+
// // Create a new context with the custom file format
196+
// let ctx = SessionContext::new_with_state(state);
197+
198+
// let mem_table = create_mem_table();
199+
// ctx.register_table("mem_table", mem_table).unwrap();
200+
201+
// let temp_dir = tempdir().unwrap();
202+
// let table_save_path = temp_dir.path().join("mem_table.tsv");
203+
204+
// let d = ctx
205+
// .sql(&format!(
206+
// "COPY mem_table TO '{}' STORED AS TSV;",
207+
// table_save_path.display(),
208+
// ))
209+
// .await?;
210+
211+
// let results = d.collect().await?;
212+
// println!(
213+
// "Number of inserted rows: {:?}",
214+
// (results[0]
215+
// .column_by_name("count")
216+
// .unwrap()
217+
// .as_primitive::<UInt64Type>()
218+
// .value(0))
219+
// );
220220

221221
Ok(())
222222
}

0 commit comments

Comments
 (0)