Skip to content

Commit 6644732

Browse files
fix: render binary columns as hex in DataFrame::describe() (apache#21728)
## Which issue does this PR close? - Closes apache#21496. ## Rationale for this change `DataFrame::describe()` is the standard way to get a statistical summary of a DataFrame (count, null_count, mean, std, min, max, median per column). Today it handles binary-like columns poorly: - For `Binary`, an exclusion filter in `min`/`max` aggregations caused both to be reported as `null`, losing useful information for columns that hold hashes, UUIDs, fingerprints, or other content-addressed identifiers. - For `LargeBinary`, `BinaryView`, and `FixedSizeBinary`, the filter did not apply, so `min`/`max` ran successfully but then the display step tried to `cast(column, Utf8)`, which Arrow correctly rejects, producing an `ArrowError::CastError` that bubbled up and failed the whole `describe()` call. The fix in this PR is aligned with what the issue proposes: stop filtering `Binary` from the aggregations and render binary outputs as lowercase hex (matching Arrow's default display of binary arrays). ## What changes are included in this PR? - `datafusion/core/src/dataframe/mod.rs`: - Drop `DataType::Binary` from the `min`/`max` exclusion filter (now only `Boolean` is excluded, which is still meaningful for a statistical summary). - Add a dedicated display branch for `Binary`, `LargeBinary`, `BinaryView`, and `FixedSizeBinary` that uses `arrow::util::display::ArrayFormatter` with default options, which renders bytes as lowercase hex. - Tidy a now-stale comment that referenced the previous binary filter. - Drive-by: use the newly imported `FormatOptions` unqualified in `DataFrame::to_string()` for consistency. ## Are these changes tested? Yes, a new integration test `describe_binary_columns` in `datafusion/core/tests/dataframe/describe.rs` builds an in-memory `RecordBatch` with one column per binary-like type and asserts the full `describe()` output via an inline `insta` snapshot. The test covers non-null values and a null row per column, so it exercises both `null_count` and the hex rendering path for `min`/`max`. All existing `describe` tests continue to pass unchanged. ## Are there any user-facing changes? Yes — this is a visible behavior change for `DataFrame::describe()`: - Before: `min`/`max` on `Binary` columns were `null`; other binary-like types caused a cast error. - After: `min`/`max` on all binary-like types render as lowercase hex strings (e.g. `"0001"`, `"ffee"`). No public API changes. --------- Co-authored-by: Jeffrey Vo <[email protected]>
1 parent 7621299 commit 6644732

2 files changed

Lines changed: 88 additions & 10 deletions

File tree

‎datafusion/core/src/dataframe/mod.rs‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ use std::sync::Arc;
4848
use arrow::array::{Array, ArrayRef, Int64Array, StringArray};
4949
use arrow::compute::{cast, concat};
5050
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
51+
use arrow::util::display::{ArrayFormatter, FormatOptions};
5152
use arrow_schema::FieldRef;
5253
use datafusion_common::config::{CsvOptions, JsonOptions};
5354
use datafusion_common::{
@@ -979,8 +980,13 @@ impl DataFrame {
979980

980981
/// Return a new `DataFrame` that has statistics for a DataFrame.
981982
///
982-
/// Only summarizes numeric datatypes at the moment and returns nulls for
983-
/// non numeric datatypes. The output format is modeled after pandas
983+
/// The summary contains the `count`, `null_count`, `mean`, `std`, `min`,
984+
/// `max`, and `median` of each column. `count` and `null_count` are
985+
/// computed for every column; `min` and `max` for every column except
986+
/// `Boolean`; and `mean`, `std`, and `median` only for numeric columns
987+
/// (other columns report `null` for these). `min`/`max` of binary columns
988+
/// (`Binary`, `LargeBinary`, `BinaryView`, `FixedSizeBinary`) are rendered
989+
/// as lowercase hex. The output format is modeled after pandas
984990
///
985991
/// # Example
986992
/// ```
@@ -1074,9 +1080,7 @@ impl DataFrame {
10741080
vec![],
10751081
original_schema_fields
10761082
.clone()
1077-
.filter(|f| {
1078-
!matches!(f.data_type(), DataType::Binary | DataType::Boolean)
1079-
})
1083+
.filter(|f| !matches!(f.data_type(), DataType::Boolean))
10801084
.map(|f| min(ident(f.name())).alias(f.name()))
10811085
.collect::<Vec<_>>(),
10821086
),
@@ -1085,9 +1089,7 @@ impl DataFrame {
10851089
vec![],
10861090
original_schema_fields
10871091
.clone()
1088-
.filter(|f| {
1089-
!matches!(f.data_type(), DataType::Binary | DataType::Boolean)
1090-
})
1092+
.filter(|f| !matches!(f.data_type(), DataType::Boolean))
10911093
.map(|f| max(ident(f.name())).alias(f.name()))
10921094
.collect::<Vec<_>>(),
10931095
),
@@ -1126,14 +1128,31 @@ impl DataFrame {
11261128
Arc::new(StringArray::from(vec!["null"]))
11271129
} else if field.data_type().is_numeric() {
11281130
cast(column, &DataType::Float64)?
1131+
} else if field.data_type().is_binary() {
1132+
let formatter = ArrayFormatter::try_new(
1133+
column.as_ref(),
1134+
&FormatOptions::default(),
1135+
)?;
1136+
let values: Vec<Option<String>> = (0..column.len())
1137+
.map(|i| {
1138+
if column.is_null(i) {
1139+
None
1140+
} else {
1141+
let value = formatter.value(i);
1142+
Some(value.to_string())
1143+
}
1144+
})
1145+
.collect();
1146+
Arc::new(StringArray::from(values))
11291147
} else {
11301148
cast(column, &DataType::Utf8)?
11311149
}
11321150
}
11331151
_ => Arc::new(StringArray::from(vec!["null"])),
11341152
}
11351153
}
1136-
//Handling error when only boolean/binary column, and in other cases
1154+
// Handles the case where all columns were filtered out
1155+
// (e.g. only boolean columns for mean/std/min/max/median)
11371156
Err(err)
11381157
if err.to_string().contains(
11391158
"Error during planning: \
@@ -1517,7 +1536,7 @@ impl DataFrame {
15171536
/// # }
15181537
pub async fn to_string(self) -> Result<String> {
15191538
let options = self.session_state.config().options().format.clone();
1520-
let arrow_options: arrow::util::display::FormatOptions = (&options).try_into()?;
1539+
let arrow_options: FormatOptions = (&options).try_into()?;
15211540

15221541
let registry = self.session_state.extension_type_registry();
15231542
let formatter_factory = DFArrayFormatterFactory::new(Arc::clone(registry));

‎datafusion/core/tests/dataframe/describe.rs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use std::sync::Arc;
19+
20+
use arrow::array::{
21+
BinaryArray, BinaryViewArray, FixedSizeBinaryArray, LargeBinaryArray, RecordBatch,
22+
};
23+
use arrow::datatypes::{DataType, Field, Schema};
1824
use datafusion::prelude::{ParquetReadOptions, SessionContext};
1925
use datafusion_common::test_util::batches_to_string;
2026
use datafusion_common::{Result, test_util::parquet_test_data};
@@ -112,6 +118,59 @@ async fn describe_null() -> Result<()> {
112118
Ok(())
113119
}
114120

121+
#[tokio::test]
122+
async fn describe_binary_columns() -> Result<()> {
123+
let schema = Arc::new(Schema::new(vec![
124+
Field::new("bin", DataType::Binary, true),
125+
Field::new("lbin", DataType::LargeBinary, true),
126+
Field::new("vbin", DataType::BinaryView, true),
127+
Field::new("fbin", DataType::FixedSizeBinary(2), true),
128+
]));
129+
130+
let bin: BinaryArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]
131+
.into_iter()
132+
.collect();
133+
let lbin: LargeBinaryArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]
134+
.into_iter()
135+
.collect();
136+
let vbin: BinaryViewArray = vec![Some([0x00u8, 0x01]), Some([0xff, 0xee]), None]
137+
.into_iter()
138+
.collect();
139+
let fbin = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
140+
[Some([0x00u8, 0x01]), Some([0xff, 0xee]), None].into_iter(),
141+
2,
142+
)?;
143+
let batch = RecordBatch::try_new(
144+
schema,
145+
vec![
146+
Arc::new(bin),
147+
Arc::new(lbin),
148+
Arc::new(vbin),
149+
Arc::new(fbin),
150+
],
151+
)?;
152+
let ctx = SessionContext::new();
153+
ctx.register_batch("t", batch)?;
154+
let result = ctx.table("t").await?.describe().await?.collect().await?;
155+
156+
assert_snapshot!(batches_to_string(&result),
157+
@r"
158+
+------------+------+------+------+------+
159+
| describe | bin | lbin | vbin | fbin |
160+
+------------+------+------+------+------+
161+
| count | 2 | 2 | 2 | 2 |
162+
| null_count | 1 | 1 | 1 | 1 |
163+
| mean | null | null | null | null |
164+
| std | null | null | null | null |
165+
| min | 0001 | 0001 | 0001 | 0001 |
166+
| max | ffee | ffee | ffee | ffee |
167+
| median | null | null | null | null |
168+
+------------+------+------+------+------+
169+
");
170+
171+
Ok(())
172+
}
173+
115174
/// Return a SessionContext with parquet file registered
116175
async fn parquet_context() -> SessionContext {
117176
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)