Skip to content

Commit d4329aa

Browse files
bors[bot]jdroenner
andauthored
Merge #775
775: disable arrow compression r=michaelmattig a=jdroenner - [ ] I added an entry to [`CHANGELOG.md`](CHANGELOG.md) if knowledge of this change could be valuable to users. --- Here is a brief summary of what I did: I disabled the compression in raster streams because pyarrow (using Arrow C++) has this issue: apache/arrow#15102 Co-authored-by: Johannes Drönner <[email protected]> Co-authored-by: Johannes Drönner <[email protected]>
2 parents 7a5bad6 + 994b3da commit d4329aa

2 files changed

Lines changed: 35 additions & 6 deletions

File tree

‎datatypes/src/raster/arrow_conversion.rs‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,7 @@ use arrow::{
66
Field, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, Schema,
77
UInt16Type, UInt32Type, UInt64Type, UInt8Type,
88
},
9-
ipc::{
10-
writer::{FileWriter, IpcWriteOptions},
11-
CompressionType,
12-
},
9+
ipc::writer::{FileWriter, IpcWriteOptions},
1310
record_batch::RecordBatch,
1411
};
1512
use std::{collections::HashMap, sync::Arc};
@@ -30,7 +27,7 @@ pub fn raster_tile_2d_to_arrow_ipc_file<P: Pixel>(
3027
let mut file_writer = FileWriter::try_new_with_options(
3128
Vec::new(),
3229
&record_batch.schema(),
33-
IpcWriteOptions::default().try_with_compression(Some(CompressionType::LZ4_FRAME))?,
30+
IpcWriteOptions::default(), //.try_with_compression(Some(CompressionType::LZ4_FRAME))?, // TODO: enable compression when pyarrow >= 12. Also: make this configurable.
3431
)?;
3532
file_writer.write(&record_batch)?;
3633
file_writer.finish()?;
@@ -199,6 +196,37 @@ mod tests {
199196

200197
assert_eq!(array.len(), 16);
201198
assert_eq!(array.null_count(), 3);
199+
assert_eq!(*array.data_type(), arrow_data_type::<u8>());
200+
assert_eq!(*array.data_type(), arrow::datatypes::DataType::UInt8);
201+
}
202+
203+
#[test]
204+
fn test_grid_2d_to_arrow_array_u16() {
205+
let values = Grid2D::new_filled([4, 4].into(), 3);
206+
let validity = Grid2D::new_filled([4, 4].into(), true);
207+
208+
let masked_grid = MaskedGrid2D::new(values.clone(), validity).unwrap();
209+
210+
let array = grid_2d_to_arrow_array::<u16>(masked_grid.into());
211+
212+
assert_eq!(array.len(), 16);
213+
assert_eq!(array.null_count(), 0);
214+
215+
// with nulls
216+
217+
let mut validity = Grid2D::new_filled([4, 4].into(), true);
218+
validity.set_at_grid_index([1, 1], false).unwrap();
219+
validity.set_at_grid_index([2, 2], false).unwrap();
220+
validity.set_at_grid_index([3, 3], false).unwrap();
221+
222+
let masked_grid = MaskedGrid2D::new(values, validity).unwrap();
223+
224+
let array = grid_2d_to_arrow_array::<u16>(masked_grid.into());
225+
226+
assert_eq!(array.len(), 16);
227+
assert_eq!(array.null_count(), 3);
228+
assert_eq!(*array.data_type(), arrow_data_type::<u16>());
229+
assert_eq!(*array.data_type(), arrow::datatypes::DataType::UInt16);
202230
}
203231

204232
#[test]

‎services/src/workflows/raster_stream.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,8 @@ mod tests {
226226

227227
let bytes = websocket_context.next().await.unwrap().unwrap();
228228

229-
let bytes = &bytes[4..]; // the first four bytes are WS op bytes
229+
let pos = bytes.windows(6).position(|w| w == b"ARROW1").unwrap();
230+
let bytes = &bytes[pos..]; // the first "pos" bytes are WS op bytes
230231

231232
let record_batches = arrow_ipc_file_to_record_batches(bytes).unwrap();
232233
assert_eq!(record_batches.len(), 1);

0 commit comments

Comments
 (0)