@@ -21,11 +21,11 @@ use arrow::{
2121 array:: { AsArray , RecordBatch , StringArray , UInt8Array } ,
2222 datatypes:: { DataType , Field , Schema , SchemaRef , UInt64Type } ,
2323} ;
24- use datafusion:: physical_expr:: LexRequirement ;
24+ use datafusion:: { logical_expr :: statistics :: TableStatistics , physical_expr:: LexRequirement } ;
2525use datafusion:: physical_expr:: PhysicalExpr ;
2626use datafusion:: {
2727 catalog:: Session ,
28- common:: { GetExt , Statistics } ,
28+ common:: GetExt ,
2929} ;
3030use datafusion:: {
3131 datasource:: physical_plan:: FileSource , execution:: session_state:: SessionStateBuilder ,
@@ -67,74 +67,74 @@ impl TSVFileFormat {
6767 }
6868}
6969
70- // #[async_trait::async_trait]
71- // impl FileFormat for TSVFileFormat {
72- // fn as_any(&self) -> &dyn Any {
73- // self
74- // }
75-
76- // fn get_ext(&self) -> String {
77- // "tsv".to_string()
78- // }
79-
80- // fn get_ext_with_compression(&self, c: &FileCompressionType) -> Result<String> {
81- // if c == &FileCompressionType::UNCOMPRESSED {
82- // Ok("tsv".to_string())
83- // } else {
84- // todo!("Compression not supported")
85- // }
86- // }
87-
88- // async fn infer_schema(
89- // &self,
90- // state: &dyn Session,
91- // store: &Arc<dyn ObjectStore>,
92- // objects: &[ObjectMeta],
93- // ) -> Result<SchemaRef> {
94- // self.csv_file_format
95- // .infer_schema(state, store, objects)
96- // .await
97- // }
98-
99- // async fn infer_stats(
100- // &self,
101- // state: &dyn Session,
102- // store: &Arc<dyn ObjectStore>,
103- // table_schema: SchemaRef,
104- // object: &ObjectMeta,
105- // ) -> Result<Statistics > {
106- // self.csv_file_format
107- // .infer_stats(state, store, table_schema, object)
108- // .await
109- // }
110-
111- // async fn create_physical_plan(
112- // &self,
113- // state: &dyn Session,
114- // conf: FileScanConfig,
115- // filters: Option<&Arc<dyn PhysicalExpr>>,
116- // ) -> Result<Arc<dyn ExecutionPlan>> {
117- // self.csv_file_format
118- // .create_physical_plan(state, conf, filters)
119- // .await
120- // }
121-
122- // async fn create_writer_physical_plan(
123- // &self,
124- // input: Arc<dyn ExecutionPlan>,
125- // state: &dyn Session,
126- // conf: FileSinkConfig,
127- // order_requirements: Option<LexRequirement>,
128- // ) -> Result<Arc<dyn ExecutionPlan>> {
129- // self.csv_file_format
130- // .create_writer_physical_plan(input, state, conf, order_requirements)
131- // .await
132- // }
133-
134- // fn file_source(&self) -> Arc<dyn FileSource> {
135- // self.csv_file_format.file_source()
136- // }
137- // }
70+ #[ async_trait:: async_trait]
71+ impl FileFormat for TSVFileFormat {
72+ fn as_any ( & self ) -> & dyn Any {
73+ self
74+ }
75+
76+ fn get_ext ( & self ) -> String {
77+ "tsv" . to_string ( )
78+ }
79+
80+ fn get_ext_with_compression ( & self , c : & FileCompressionType ) -> Result < String > {
81+ if c == & FileCompressionType :: UNCOMPRESSED {
82+ Ok ( "tsv" . to_string ( ) )
83+ } else {
84+ todo ! ( "Compression not supported" )
85+ }
86+ }
87+
88+ async fn infer_schema (
89+ & self ,
90+ state : & dyn Session ,
91+ store : & Arc < dyn ObjectStore > ,
92+ objects : & [ ObjectMeta ] ,
93+ ) -> Result < SchemaRef > {
94+ self . csv_file_format
95+ . infer_schema ( state, store, objects)
96+ . await
97+ }
98+
99+ async fn infer_stats (
100+ & self ,
101+ state : & dyn Session ,
102+ store : & Arc < dyn ObjectStore > ,
103+ table_schema : SchemaRef ,
104+ object : & ObjectMeta ,
105+ ) -> Result < TableStatistics > {
106+ self . csv_file_format
107+ . infer_stats ( state, store, table_schema, object)
108+ . await
109+ }
110+
111+ async fn create_physical_plan (
112+ & self ,
113+ state : & dyn Session ,
114+ conf : FileScanConfig ,
115+ filters : Option < & Arc < dyn PhysicalExpr > > ,
116+ ) -> Result < Arc < dyn ExecutionPlan > > {
117+ self . csv_file_format
118+ . create_physical_plan ( state, conf, filters)
119+ . await
120+ }
121+
122+ async fn create_writer_physical_plan (
123+ & self ,
124+ input : Arc < dyn ExecutionPlan > ,
125+ state : & dyn Session ,
126+ conf : FileSinkConfig ,
127+ order_requirements : Option < LexRequirement > ,
128+ ) -> Result < Arc < dyn ExecutionPlan > > {
129+ self . csv_file_format
130+ . create_writer_physical_plan ( input, state, conf, order_requirements)
131+ . await
132+ }
133+
134+ fn file_source ( & self ) -> Arc < dyn FileSource > {
135+ self . csv_file_format . file_source ( )
136+ }
137+ }
138138
139139#[ derive( Default , Debug ) ]
140140/// Factory for creating TSV file formats
@@ -153,29 +153,29 @@ impl TSVFileFactory {
153153 }
154154}
155155
156- // impl FileFormatFactory for TSVFileFactory {
157- // fn create(
158- // &self,
159- // state: &dyn Session,
160- // format_options: &std::collections::HashMap<String, String>,
161- // ) -> Result<Arc<dyn FileFormat>> {
162- // let mut new_options = format_options.clone();
163- // new_options.insert("format.delimiter".to_string(), "\t".to_string());
156+ impl FileFormatFactory for TSVFileFactory {
157+ fn create (
158+ & self ,
159+ state : & dyn Session ,
160+ format_options : & std:: collections:: HashMap < String , String > ,
161+ ) -> Result < Arc < dyn FileFormat > > {
162+ let mut new_options = format_options. clone ( ) ;
163+ new_options. insert ( "format.delimiter" . to_string ( ) , "\t " . to_string ( ) ) ;
164164
165- // let csv_file_format = self.csv_file_factory.create(state, &new_options)?;
166- // let tsv_file_format = Arc::new(TSVFileFormat::new(csv_file_format));
165+ let csv_file_format = self . csv_file_factory . create ( state, & new_options) ?;
166+ let tsv_file_format = Arc :: new ( TSVFileFormat :: new ( csv_file_format) ) ;
167167
168- // Ok(tsv_file_format)
169- // }
168+ Ok ( tsv_file_format)
169+ }
170170
171- // fn default(&self) -> Arc<dyn FileFormat> {
172- // todo!()
173- // }
171+ fn default ( & self ) -> Arc < dyn FileFormat > {
172+ todo ! ( )
173+ }
174174
175- // fn as_any(&self) -> &dyn Any {
176- // self
177- // }
178- // }
175+ fn as_any ( & self ) -> & dyn Any {
176+ self
177+ }
178+ }
179179
180180impl GetExt for TSVFileFactory {
181181 fn get_ext ( & self ) -> String {
@@ -185,38 +185,38 @@ impl GetExt for TSVFileFactory {
185185
186186#[ tokio:: main]
187187async fn main ( ) -> Result < ( ) > {
188- // // Create a new context with the default configuration
189- // let mut state = SessionStateBuilder::new().with_default_features().build();
190-
191- // // Register the custom file format
192- // let file_format = Arc::new(TSVFileFactory::new());
193- // state.register_file_format(file_format, true).unwrap();
194-
195- // // Create a new context with the custom file format
196- // let ctx = SessionContext::new_with_state(state);
197-
198- // let mem_table = create_mem_table();
199- // ctx.register_table("mem_table", mem_table).unwrap();
200-
201- // let temp_dir = tempdir().unwrap();
202- // let table_save_path = temp_dir.path().join("mem_table.tsv");
203-
204- // let d = ctx
205- // .sql(&format!(
206- // "COPY mem_table TO '{}' STORED AS TSV;",
207- // table_save_path.display(),
208- // ))
209- // .await?;
210-
211- // let results = d.collect().await?;
212- // println!(
213- // "Number of inserted rows: {:?}",
214- // (results[0]
215- // .column_by_name("count")
216- // .unwrap()
217- // .as_primitive::<UInt64Type>()
218- // .value(0))
219- // );
188+ // Create a new context with the default configuration
189+ let mut state = SessionStateBuilder :: new ( ) . with_default_features ( ) . build ( ) ;
190+
191+ // Register the custom file format
192+ let file_format = Arc :: new ( TSVFileFactory :: new ( ) ) ;
193+ state. register_file_format ( file_format, true ) . unwrap ( ) ;
194+
195+ // Create a new context with the custom file format
196+ let ctx = SessionContext :: new_with_state ( state) ;
197+
198+ let mem_table = create_mem_table ( ) ;
199+ ctx. register_table ( "mem_table" , mem_table) . unwrap ( ) ;
200+
201+ let temp_dir = tempdir ( ) . unwrap ( ) ;
202+ let table_save_path = temp_dir. path ( ) . join ( "mem_table.tsv" ) ;
203+
204+ let d = ctx
205+ . sql ( & format ! (
206+ "COPY mem_table TO '{}' STORED AS TSV;" ,
207+ table_save_path. display( ) ,
208+ ) )
209+ . await ?;
210+
211+ let results = d. collect ( ) . await ?;
212+ println ! (
213+ "Number of inserted rows: {:?}" ,
214+ ( results[ 0 ]
215+ . column_by_name( "count" )
216+ . unwrap( )
217+ . as_primitive:: <UInt64Type >( )
218+ . value( 0 ) )
219+ ) ;
220220
221221 Ok ( ( ) )
222222}
0 commit comments