Skip to content

Commit e1b3740

Browse files
authored
feat: add compression options (apache#456)
1 parent 37c91f4 commit e1b3740

3 files changed

Lines changed: 83 additions & 9 deletions

File tree

‎datafusion/tests/test_context.py‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
17+
import gzip
1818
import os
1919

2020
import pyarrow as pa
@@ -336,11 +336,47 @@ def test_read_json(ctx):
336336
assert result[0].column(1) == pa.array([1, 2, 3])
337337

338338

339+
def test_read_json_compressed(ctx, tmp_path):
340+
path = os.path.dirname(os.path.abspath(__file__))
341+
test_data_path = os.path.join(path, "data_test_context", "data.json")
342+
343+
# File compression type
344+
gzip_path = tmp_path / "data.json.gz"
345+
346+
with open(test_data_path, "rb") as csv_file:
347+
with gzip.open(gzip_path, "wb") as gzipped_file:
348+
gzipped_file.writelines(csv_file)
349+
350+
df = ctx.read_json(
351+
gzip_path, file_extension=".gz", file_compression_type="gz"
352+
)
353+
result = df.collect()
354+
355+
assert result[0].column(0) == pa.array(["a", "b", "c"])
356+
assert result[0].column(1) == pa.array([1, 2, 3])
357+
358+
339359
def test_read_csv(ctx):
340360
csv_df = ctx.read_csv(path="testing/data/csv/aggregate_test_100.csv")
341361
csv_df.select(column("c1")).show()
342362

343363

364+
def test_read_csv_compressed(ctx, tmp_path):
365+
test_data_path = "testing/data/csv/aggregate_test_100.csv"
366+
367+
# File compression type
368+
gzip_path = tmp_path / "aggregate_test_100.csv.gz"
369+
370+
with open(test_data_path, "rb") as csv_file:
371+
with gzip.open(gzip_path, "wb") as gzipped_file:
372+
gzipped_file.writelines(csv_file)
373+
374+
csv_df = ctx.read_csv(
375+
gzip_path, file_extension=".gz", file_compression_type="gz"
376+
)
377+
csv_df.select(column("c1")).show()
378+
379+
344380
def test_read_parquet(ctx):
345381
csv_df = ctx.read_parquet(path="parquet/data/alltypes_plain.parquet")
346382
csv_df.show()

‎datafusion/tests/test_sql.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import pyarrow as pa
2020
import pyarrow.dataset as ds
2121
import pytest
22+
import gzip
2223

2324
from datafusion import udf
2425

@@ -32,6 +33,7 @@ def test_no_table(ctx):
3233

3334
def test_register_csv(ctx, tmp_path):
3435
path = tmp_path / "test.csv"
36+
gzip_path = tmp_path / "test.csv.gz"
3537

3638
table = pa.Table.from_arrays(
3739
[
@@ -43,6 +45,10 @@ def test_register_csv(ctx, tmp_path):
4345
)
4446
pa.csv.write_csv(table, path)
4547

48+
with open(path, "rb") as csv_file:
49+
with gzip.open(gzip_path, "wb") as gzipped_file:
50+
gzipped_file.writelines(csv_file)
51+
4652
ctx.register_csv("csv", path)
4753
ctx.register_csv("csv1", str(path))
4854
ctx.register_csv(
@@ -52,6 +58,13 @@ def test_register_csv(ctx, tmp_path):
5258
delimiter=",",
5359
schema_infer_max_records=10,
5460
)
61+
ctx.register_csv(
62+
"csv_gzip",
63+
gzip_path,
64+
file_extension="gz",
65+
file_compression_type="gzip",
66+
)
67+
5568
alternative_schema = pa.schema(
5669
[
5770
("some_int", pa.int16()),
@@ -61,9 +74,9 @@ def test_register_csv(ctx, tmp_path):
6174
)
6275
ctx.register_csv("csv3", path, schema=alternative_schema)
6376

64-
assert ctx.tables() == {"csv", "csv1", "csv2", "csv3"}
77+
assert ctx.tables() == {"csv", "csv1", "csv2", "csv3", "csv_gzip"}
6578

66-
for table in ["csv", "csv1", "csv2"]:
79+
for table in ["csv", "csv1", "csv2", "csv_gzip"]:
6780
result = ctx.sql(f"SELECT COUNT(int) AS cnt FROM {table}").collect()
6881
result = pa.Table.from_batches(result)
6982
assert result.to_pydict() == {"cnt": [4]}
@@ -77,6 +90,12 @@ def test_register_csv(ctx, tmp_path):
7790
):
7891
ctx.register_csv("csv4", path, delimiter="wrong")
7992

93+
with pytest.raises(
94+
ValueError,
95+
match="file_compression_type must one of: gzip, bz2, xz, zstd",
96+
):
97+
ctx.register_csv("csv4", path, file_compression_type="rar")
98+
8099

81100
def test_register_parquet(ctx, tmp_path):
82101
path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data())

‎src/context.rs‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
use std::collections::{HashMap, HashSet};
1919
use std::path::PathBuf;
20+
use std::str::FromStr;
2021
use std::sync::Arc;
2122

2223
use object_store::ObjectStore;
@@ -40,6 +41,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future};
4041
use datafusion::arrow::datatypes::{DataType, Schema};
4142
use datafusion::arrow::pyarrow::PyArrowType;
4243
use datafusion::arrow::record_batch::RecordBatch;
44+
use datafusion::datasource::file_format::file_type::FileCompressionType;
4345
use datafusion::datasource::MemTable;
4446
use datafusion::datasource::TableProvider;
4547
use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext};
@@ -469,7 +471,8 @@ impl PySessionContext {
469471
has_header=true,
470472
delimiter=",",
471473
schema_infer_max_records=1000,
472-
file_extension=".csv"))]
474+
file_extension=".csv",
475+
file_compression_type=None))]
473476
fn register_csv(
474477
&mut self,
475478
name: &str,
@@ -479,6 +482,7 @@ impl PySessionContext {
479482
delimiter: &str,
480483
schema_infer_max_records: usize,
481484
file_extension: &str,
485+
file_compression_type: Option<String>,
482486
py: Python,
483487
) -> PyResult<()> {
484488
let path = path
@@ -495,7 +499,8 @@ impl PySessionContext {
495499
.has_header(has_header)
496500
.delimiter(delimiter[0])
497501
.schema_infer_max_records(schema_infer_max_records)
498-
.file_extension(file_extension);
502+
.file_extension(file_extension)
503+
.file_compression_type(parse_file_compression_type(file_compression_type)?);
499504
options.schema = schema.as_ref().map(|x| &x.0);
500505

501506
let result = self.ctx.register_csv(name, path, options);
@@ -559,21 +564,23 @@ impl PySessionContext {
559564
}
560565

561566
#[allow(clippy::too_many_arguments)]
562-
#[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![]))]
567+
#[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![], file_compression_type=None))]
563568
fn read_json(
564569
&mut self,
565570
path: PathBuf,
566571
schema: Option<PyArrowType<Schema>>,
567572
schema_infer_max_records: usize,
568573
file_extension: &str,
569574
table_partition_cols: Vec<(String, String)>,
575+
file_compression_type: Option<String>,
570576
py: Python,
571577
) -> PyResult<PyDataFrame> {
572578
let path = path
573579
.to_str()
574580
.ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?;
575581
let mut options = NdJsonReadOptions::default()
576-
.table_partition_cols(convert_table_partition_cols(table_partition_cols)?);
582+
.table_partition_cols(convert_table_partition_cols(table_partition_cols)?)
583+
.file_compression_type(parse_file_compression_type(file_compression_type)?);
577584
options.schema_infer_max_records = schema_infer_max_records;
578585
options.file_extension = file_extension;
579586
let df = if let Some(schema) = schema {
@@ -595,7 +602,8 @@ impl PySessionContext {
595602
delimiter=",",
596603
schema_infer_max_records=1000,
597604
file_extension=".csv",
598-
table_partition_cols=vec![]))]
605+
table_partition_cols=vec![],
606+
file_compression_type=None))]
599607
fn read_csv(
600608
&self,
601609
path: PathBuf,
@@ -605,6 +613,7 @@ impl PySessionContext {
605613
schema_infer_max_records: usize,
606614
file_extension: &str,
607615
table_partition_cols: Vec<(String, String)>,
616+
file_compression_type: Option<String>,
608617
py: Python,
609618
) -> PyResult<PyDataFrame> {
610619
let path = path
@@ -623,7 +632,8 @@ impl PySessionContext {
623632
.delimiter(delimiter[0])
624633
.schema_infer_max_records(schema_infer_max_records)
625634
.file_extension(file_extension)
626-
.table_partition_cols(convert_table_partition_cols(table_partition_cols)?);
635+
.table_partition_cols(convert_table_partition_cols(table_partition_cols)?)
636+
.file_compression_type(parse_file_compression_type(file_compression_type)?);
627637

628638
if let Some(py_schema) = schema {
629639
options.schema = Some(&py_schema.0);
@@ -743,6 +753,15 @@ fn convert_table_partition_cols(
743753
.collect::<Result<Vec<_>, _>>()
744754
}
745755

756+
fn parse_file_compression_type(
757+
file_compression_type: Option<String>,
758+
) -> Result<FileCompressionType, PyErr> {
759+
FileCompressionType::from_str(&*file_compression_type.unwrap_or("".to_string()).as_str())
760+
.map_err(|_| {
761+
PyValueError::new_err("file_compression_type must one of: gzip, bz2, xz, zstd")
762+
})
763+
}
764+
746765
impl From<PySessionContext> for SessionContext {
747766
fn from(ctx: PySessionContext) -> SessionContext {
748767
ctx.ctx

0 commit comments

Comments
 (0)