1717
1818use std:: collections:: { HashMap , HashSet } ;
1919use std:: path:: PathBuf ;
20+ use std:: str:: FromStr ;
2021use std:: sync:: Arc ;
2122
2223use object_store:: ObjectStore ;
@@ -40,6 +41,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future};
4041use datafusion:: arrow:: datatypes:: { DataType , Schema } ;
4142use datafusion:: arrow:: pyarrow:: PyArrowType ;
4243use datafusion:: arrow:: record_batch:: RecordBatch ;
44+ use datafusion:: datasource:: file_format:: file_type:: FileCompressionType ;
4345use datafusion:: datasource:: MemTable ;
4446use datafusion:: datasource:: TableProvider ;
4547use datafusion:: execution:: context:: { SessionConfig , SessionContext , TaskContext } ;
@@ -469,7 +471,8 @@ impl PySessionContext {
469471 has_header=true ,
470472 delimiter="," ,
471473 schema_infer_max_records=1000 ,
472- file_extension=".csv" ) ) ]
474+ file_extension=".csv" ,
475+ file_compression_type=None ) ) ]
473476 fn register_csv (
474477 & mut self ,
475478 name : & str ,
@@ -479,6 +482,7 @@ impl PySessionContext {
479482 delimiter : & str ,
480483 schema_infer_max_records : usize ,
481484 file_extension : & str ,
485+ file_compression_type : Option < String > ,
482486 py : Python ,
483487 ) -> PyResult < ( ) > {
484488 let path = path
@@ -495,7 +499,8 @@ impl PySessionContext {
495499 . has_header ( has_header)
496500 . delimiter ( delimiter[ 0 ] )
497501 . schema_infer_max_records ( schema_infer_max_records)
498- . file_extension ( file_extension) ;
502+ . file_extension ( file_extension)
503+ . file_compression_type ( parse_file_compression_type ( file_compression_type) ?) ;
499504 options. schema = schema. as_ref ( ) . map ( |x| & x. 0 ) ;
500505
501506 let result = self . ctx . register_csv ( name, path, options) ;
@@ -559,21 +564,23 @@ impl PySessionContext {
559564 }
560565
561566 #[ allow( clippy:: too_many_arguments) ]
562- #[ pyo3( signature = ( path, schema=None , schema_infer_max_records=1000 , file_extension=".json" , table_partition_cols=vec![ ] ) ) ]
567+ #[ pyo3( signature = ( path, schema=None , schema_infer_max_records=1000 , file_extension=".json" , table_partition_cols=vec![ ] , file_compression_type= None ) ) ]
563568 fn read_json (
564569 & mut self ,
565570 path : PathBuf ,
566571 schema : Option < PyArrowType < Schema > > ,
567572 schema_infer_max_records : usize ,
568573 file_extension : & str ,
569574 table_partition_cols : Vec < ( String , String ) > ,
575+ file_compression_type : Option < String > ,
570576 py : Python ,
571577 ) -> PyResult < PyDataFrame > {
572578 let path = path
573579 . to_str ( )
574580 . ok_or_else ( || PyValueError :: new_err ( "Unable to convert path to a string" ) ) ?;
575581 let mut options = NdJsonReadOptions :: default ( )
576- . table_partition_cols ( convert_table_partition_cols ( table_partition_cols) ?) ;
582+ . table_partition_cols ( convert_table_partition_cols ( table_partition_cols) ?)
583+ . file_compression_type ( parse_file_compression_type ( file_compression_type) ?) ;
577584 options. schema_infer_max_records = schema_infer_max_records;
578585 options. file_extension = file_extension;
579586 let df = if let Some ( schema) = schema {
@@ -595,7 +602,8 @@ impl PySessionContext {
595602 delimiter="," ,
596603 schema_infer_max_records=1000 ,
597604 file_extension=".csv" ,
598- table_partition_cols=vec![ ] ) ) ]
605+ table_partition_cols=vec![ ] ,
606+ file_compression_type=None ) ) ]
599607 fn read_csv (
600608 & self ,
601609 path : PathBuf ,
@@ -605,6 +613,7 @@ impl PySessionContext {
605613 schema_infer_max_records : usize ,
606614 file_extension : & str ,
607615 table_partition_cols : Vec < ( String , String ) > ,
616+ file_compression_type : Option < String > ,
608617 py : Python ,
609618 ) -> PyResult < PyDataFrame > {
610619 let path = path
@@ -623,7 +632,8 @@ impl PySessionContext {
623632 . delimiter ( delimiter[ 0 ] )
624633 . schema_infer_max_records ( schema_infer_max_records)
625634 . file_extension ( file_extension)
626- . table_partition_cols ( convert_table_partition_cols ( table_partition_cols) ?) ;
635+ . table_partition_cols ( convert_table_partition_cols ( table_partition_cols) ?)
636+ . file_compression_type ( parse_file_compression_type ( file_compression_type) ?) ;
627637
628638 if let Some ( py_schema) = schema {
629639 options. schema = Some ( & py_schema. 0 ) ;
@@ -743,6 +753,15 @@ fn convert_table_partition_cols(
743753 . collect :: < Result < Vec < _ > , _ > > ( )
744754}
745755
756+ fn parse_file_compression_type (
757+ file_compression_type : Option < String > ,
758+ ) -> Result < FileCompressionType , PyErr > {
759+ FileCompressionType :: from_str ( & * file_compression_type. unwrap_or ( "" . to_string ( ) ) . as_str ( ) )
760+ . map_err ( |_| {
761+ PyValueError :: new_err ( "file_compression_type must one of: gzip, bz2, xz, zstd" )
762+ } )
763+ }
764+
746765impl From < PySessionContext > for SessionContext {
747766 fn from ( ctx : PySessionContext ) -> SessionContext {
748767 ctx. ctx
0 commit comments