Skip to content

Commit 5c10375

Browse files
authored
Merge branch 'main' into feat/adaptive-partial-agg-cost
2 parents 2b49798 + bdf8a6d commit 5c10375

30 files changed

Lines changed: 886 additions & 123 deletions

File tree

‎datafusion/catalog/src/information_schema.rs‎

Lines changed: 121 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -967,18 +967,34 @@ struct InformationSchemata {
967967
config: InformationSchemaConfig,
968968
}
969969

970+
/// The Arrow schema of [`information_schema.schemata`] rows.
971+
///
972+
/// Useful for downstream catalog implementations that want to declare a
973+
/// `TableProvider` for `schemata` before populating any rows via
974+
/// [`InformationSchemataBuilder`].
975+
///
976+
/// Columns and nullability match
977+
/// <https://www.postgresql.org/docs/current/infoschema-schemata.html>.
978+
///
979+
/// [`information_schema.schemata`]: https://www.postgresql.org/docs/current/infoschema-schemata.html
980+
pub fn schemata_schema() -> SchemaRef {
981+
Arc::new(Schema::new(vec![
982+
Field::new("catalog_name", DataType::Utf8, false),
983+
Field::new("schema_name", DataType::Utf8, false),
984+
Field::new("schema_owner", DataType::Utf8, true),
985+
Field::new("default_character_set_catalog", DataType::Utf8, true),
986+
Field::new("default_character_set_schema", DataType::Utf8, true),
987+
Field::new("default_character_set_name", DataType::Utf8, true),
988+
Field::new("sql_path", DataType::Utf8, true),
989+
]))
990+
}
991+
970992
impl InformationSchemata {
971993
fn new(config: InformationSchemaConfig) -> Self {
972-
let schema = Arc::new(Schema::new(vec![
973-
Field::new("catalog_name", DataType::Utf8, false),
974-
Field::new("schema_name", DataType::Utf8, false),
975-
Field::new("schema_owner", DataType::Utf8, true),
976-
Field::new("default_character_set_catalog", DataType::Utf8, true),
977-
Field::new("default_character_set_schema", DataType::Utf8, true),
978-
Field::new("default_character_set_name", DataType::Utf8, true),
979-
Field::new("sql_path", DataType::Utf8, true),
980-
]));
981-
Self { schema, config }
994+
Self {
995+
schema: schemata_schema(),
996+
config,
997+
}
982998
}
983999

9841000
fn builder(&self) -> InformationSchemataBuilder {
@@ -995,7 +1011,16 @@ impl InformationSchemata {
9951011
}
9961012
}
9971013

998-
struct InformationSchemataBuilder {
1014+
/// Builder that produces [`RecordBatch`] values matching the schema of
1015+
/// `information_schema.schemata` (see [`schemata_schema`]).
1016+
///
1017+
/// Intended for downstream catalog implementations that need to emit
1018+
/// `schemata` rows from their own metadata source rather than going
1019+
/// through DataFusion's `InformationSchemaProvider`, which enumerates
1020+
/// schemas synchronously via `CatalogProviderList` and so is unsuitable
1021+
/// for catalog backends that resolve asynchronously.
1022+
#[derive(Debug)]
1023+
pub struct InformationSchemataBuilder {
9991024
schema: SchemaRef,
10001025
catalog_name: StringBuilder,
10011026
schema_name: StringBuilder,
@@ -1006,8 +1031,32 @@ struct InformationSchemataBuilder {
10061031
sql_path: StringBuilder,
10071032
}
10081033

1034+
impl Default for InformationSchemataBuilder {
1035+
fn default() -> Self {
1036+
Self::new()
1037+
}
1038+
}
1039+
10091040
impl InformationSchemataBuilder {
1010-
fn add_schemata(
1041+
/// Construct an empty builder.
1042+
pub fn new() -> Self {
1043+
Self {
1044+
schema: schemata_schema(),
1045+
catalog_name: StringBuilder::new(),
1046+
schema_name: StringBuilder::new(),
1047+
schema_owner: StringBuilder::new(),
1048+
default_character_set_catalog: StringBuilder::new(),
1049+
default_character_set_schema: StringBuilder::new(),
1050+
default_character_set_name: StringBuilder::new(),
1051+
sql_path: StringBuilder::new(),
1052+
}
1053+
}
1054+
1055+
/// Append one row to the builder. `schema_owner` is the optional SQL
1056+
/// schema owner; the three `default_character_set_*` columns and
1057+
/// `sql_path` are written as null (DataFusion does not model those
1058+
/// concepts; see the PostgreSQL docs link on [`schemata_schema`]).
1059+
pub fn add_schemata(
10111060
&mut self,
10121061
catalog_name: &str,
10131062
schema_name: &str,
@@ -1019,15 +1068,19 @@ impl InformationSchemataBuilder {
10191068
Some(owner) => self.schema_owner.append_value(owner),
10201069
None => self.schema_owner.append_null(),
10211070
}
1022-
// refer to https://www.postgresql.org/docs/current/infoschema-schemata.html,
1023-
// these rows apply to a feature that is not implemented in DataFusion
10241071
self.default_character_set_catalog.append_null();
10251072
self.default_character_set_schema.append_null();
10261073
self.default_character_set_name.append_null();
10271074
self.sql_path.append_null();
10281075
}
10291076

1030-
fn finish(&mut self) -> RecordBatch {
1077+
/// Finalize the builder into a [`RecordBatch`].
1078+
///
1079+
/// Returns an error only if Arrow buffer construction fails, which
1080+
/// the builder's column-count and type invariants make unreachable
1081+
/// under normal use. The `Result` return type preserves room to add
1082+
/// validation in the future without a breaking API change.
1083+
pub fn finish(&mut self) -> Result<RecordBatch> {
10311084
RecordBatch::try_new(
10321085
Arc::clone(&self.schema),
10331086
vec![
@@ -1040,7 +1093,7 @@ impl InformationSchemataBuilder {
10401093
Arc::new(self.sql_path.finish()),
10411094
],
10421095
)
1043-
.unwrap()
1096+
.map_err(DataFusionError::from)
10441097
}
10451098
}
10461099

@@ -1057,7 +1110,7 @@ impl PartitionStream for InformationSchemata {
10571110
// TODO: Stream this
10581111
futures::stream::once(async move {
10591112
config.make_schemata(&mut builder).await;
1060-
Ok(builder.finish())
1113+
builder.finish()
10611114
}),
10621115
))
10631116
}
@@ -1413,6 +1466,57 @@ impl PartitionStream for InformationSchemaParameters {
14131466
mod tests {
14141467
use super::*;
14151468
use crate::CatalogProvider;
1469+
use arrow::array::Array;
1470+
1471+
#[test]
1472+
fn schemata_builder_emits_canonical_schema_and_rows() {
1473+
// Construct via `Default` so the test exercises both `new()` (via
1474+
// the `Default` impl) and the public column-layout contract.
1475+
let mut builder = InformationSchemataBuilder::default();
1476+
builder.add_schemata("cat", "schema_one", Some("alice"));
1477+
builder.add_schemata("cat", "schema_two", None);
1478+
let batch = builder.finish().expect("finish should not fail");
1479+
1480+
assert_eq!(batch.schema(), schemata_schema());
1481+
assert_eq!(batch.num_rows(), 2);
1482+
1483+
let col = |name: &str| {
1484+
batch
1485+
.column_by_name(name)
1486+
.unwrap_or_else(|| panic!("missing column {name}"))
1487+
};
1488+
let string_col = |name: &str| {
1489+
col(name)
1490+
.as_any()
1491+
.downcast_ref::<arrow::array::StringArray>()
1492+
.unwrap_or_else(|| panic!("{name} should be a StringArray"))
1493+
};
1494+
1495+
let catalog = string_col("catalog_name");
1496+
assert_eq!(catalog.value(0), "cat");
1497+
assert_eq!(catalog.value(1), "cat");
1498+
1499+
let schema = string_col("schema_name");
1500+
assert_eq!(schema.value(0), "schema_one");
1501+
assert_eq!(schema.value(1), "schema_two");
1502+
1503+
let owner = string_col("schema_owner");
1504+
assert_eq!(owner.value(0), "alice");
1505+
assert!(owner.is_null(1));
1506+
1507+
// The three character-set columns and sql_path are unconditionally
1508+
// null — they exist for SQL-standard column-layout compatibility.
1509+
for name in [
1510+
"default_character_set_catalog",
1511+
"default_character_set_schema",
1512+
"default_character_set_name",
1513+
"sql_path",
1514+
] {
1515+
let c = string_col(name);
1516+
assert!(c.is_null(0), "{name} row 0 should be null");
1517+
assert!(c.is_null(1), "{name} row 1 should be null");
1518+
}
1519+
}
14161520

14171521
#[tokio::test]
14181522
async fn make_tables_uses_table_type() {

‎datafusion/common/src/scalar/mod.rs‎

Lines changed: 110 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5343,6 +5343,35 @@ macro_rules! format_option {
53435343
}};
53445344
}
53455345

5346+
macro_rules! format_decimal {
5347+
($F:expr, $TYPE:ty, $VALUE:expr, $PRECISION:expr, $SCALE:expr) => {{
5348+
match $VALUE {
5349+
Some(value) => write!(
5350+
$F,
5351+
"{}",
5352+
<$TYPE>::format_decimal(*value, *$PRECISION, *$SCALE)
5353+
),
5354+
None => write!($F, "NULL"),
5355+
}
5356+
}};
5357+
}
5358+
5359+
macro_rules! format_decimal_debug {
5360+
($F:expr, $TYPE_NAME:literal, $TYPE:ty, $VALUE:expr, $PRECISION:expr, $SCALE:expr) => {{
5361+
match $VALUE {
5362+
Some(value) => write!(
5363+
$F,
5364+
"{}({},{},{})",
5365+
$TYPE_NAME,
5366+
<$TYPE>::format_decimal(*value, *$PRECISION, *$SCALE),
5367+
$PRECISION,
5368+
$SCALE
5369+
),
5370+
None => write!($F, "{}(NULL,{},{})", $TYPE_NAME, $PRECISION, $SCALE),
5371+
}
5372+
}};
5373+
}
5374+
53465375
// Implement Display trait for ScalarValue
53475376
//
53485377
// # Panics
@@ -5352,16 +5381,16 @@ impl fmt::Display for ScalarValue {
53525381
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
53535382
match self {
53545383
ScalarValue::Decimal32(v, p, s) => {
5355-
write!(f, "{v:?},{p:?},{s:?}")?;
5384+
format_decimal!(f, Decimal32Type, v, p, s)?
53565385
}
53575386
ScalarValue::Decimal64(v, p, s) => {
5358-
write!(f, "{v:?},{p:?},{s:?}")?;
5387+
format_decimal!(f, Decimal64Type, v, p, s)?
53595388
}
53605389
ScalarValue::Decimal128(v, p, s) => {
5361-
write!(f, "{v:?},{p:?},{s:?}")?;
5390+
format_decimal!(f, Decimal128Type, v, p, s)?
53625391
}
53635392
ScalarValue::Decimal256(v, p, s) => {
5364-
write!(f, "{v:?},{p:?},{s:?}")?;
5393+
format_decimal!(f, Decimal256Type, v, p, s)?
53655394
}
53665395
ScalarValue::Boolean(e) => format_option!(f, e)?,
53675396
ScalarValue::Float16(e) => format_option!(f, e)?,
@@ -5535,8 +5564,9 @@ fn fmt_list(arr: &dyn Array, f: &mut fmt::Formatter) -> fmt::Result {
55355564
write!(f, "{value_formatter}")
55365565
}
55375566

5538-
/// writes a byte array to formatter. `[1, 2, 3]` ==> `"1,2,3"`
5539-
fn fmt_binary(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result {
5567+
/// Writes a byte array for ScalarValue Debug formatting.
5568+
/// `[1, 2, 3]` -> `"1,2,3"`
5569+
fn fmt_binary_debug(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result {
55405570
let mut iter = data.iter();
55415571
if let Some(b) = iter.next() {
55425572
write!(f, "{b}")?;
@@ -5550,10 +5580,46 @@ fn fmt_binary(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result {
55505580
impl fmt::Debug for ScalarValue {
55515581
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
55525582
match self {
5553-
ScalarValue::Decimal32(_, _, _) => write!(f, "Decimal32({self})"),
5554-
ScalarValue::Decimal64(_, _, _) => write!(f, "Decimal64({self})"),
5555-
ScalarValue::Decimal128(_, _, _) => write!(f, "Decimal128({self})"),
5556-
ScalarValue::Decimal256(_, _, _) => write!(f, "Decimal256({self})"),
5583+
ScalarValue::Decimal32(value, precision, scale) => {
5584+
format_decimal_debug!(
5585+
f,
5586+
"Decimal32",
5587+
Decimal32Type,
5588+
value,
5589+
precision,
5590+
scale
5591+
)
5592+
}
5593+
ScalarValue::Decimal64(value, precision, scale) => {
5594+
format_decimal_debug!(
5595+
f,
5596+
"Decimal64",
5597+
Decimal64Type,
5598+
value,
5599+
precision,
5600+
scale
5601+
)
5602+
}
5603+
ScalarValue::Decimal128(value, precision, scale) => {
5604+
format_decimal_debug!(
5605+
f,
5606+
"Decimal128",
5607+
Decimal128Type,
5608+
value,
5609+
precision,
5610+
scale
5611+
)
5612+
}
5613+
ScalarValue::Decimal256(value, precision, scale) => {
5614+
format_decimal_debug!(
5615+
f,
5616+
"Decimal256",
5617+
Decimal256Type,
5618+
value,
5619+
precision,
5620+
scale
5621+
)
5622+
}
55575623
ScalarValue::Boolean(_) => write!(f, "Boolean({self})"),
55585624
ScalarValue::Float16(_) => write!(f, "Float16({self})"),
55595625
ScalarValue::Float32(_) => write!(f, "Float32({self})"),
@@ -5587,27 +5653,27 @@ impl fmt::Debug for ScalarValue {
55875653
ScalarValue::Binary(None) => write!(f, "Binary({self})"),
55885654
ScalarValue::Binary(Some(b)) => {
55895655
write!(f, "Binary(\"")?;
5590-
fmt_binary(b.as_slice(), f)?;
5656+
fmt_binary_debug(b.as_slice(), f)?;
55915657
write!(f, "\")")
55925658
}
55935659
ScalarValue::BinaryView(None) => write!(f, "BinaryView({self})"),
55945660
ScalarValue::BinaryView(Some(b)) => {
55955661
write!(f, "BinaryView(\"")?;
5596-
fmt_binary(b.as_slice(), f)?;
5662+
fmt_binary_debug(b.as_slice(), f)?;
55975663
write!(f, "\")")
55985664
}
55995665
ScalarValue::FixedSizeBinary(size, None) => {
56005666
write!(f, "FixedSizeBinary({size}, {self})")
56015667
}
56025668
ScalarValue::FixedSizeBinary(size, Some(b)) => {
56035669
write!(f, "FixedSizeBinary({size}, \"")?;
5604-
fmt_binary(b.as_slice(), f)?;
5670+
fmt_binary_debug(b.as_slice(), f)?;
56055671
write!(f, "\")")
56065672
}
56075673
ScalarValue::LargeBinary(None) => write!(f, "LargeBinary({self})"),
56085674
ScalarValue::LargeBinary(Some(b)) => {
56095675
write!(f, "LargeBinary(\"")?;
5610-
fmt_binary(b.as_slice(), f)?;
5676+
fmt_binary_debug(b.as_slice(), f)?;
56115677
write!(f, "\")")
56125678
}
56135679
ScalarValue::FixedSizeList(_) => write!(f, "FixedSizeList({self})"),
@@ -9600,6 +9666,36 @@ mod tests {
96009666
);
96019667
}
96029668

9669+
#[test]
9670+
fn test_decimal_display_and_debug() {
9671+
let decimal32 = ScalarValue::Decimal32(Some(123), 3, 2);
9672+
assert_eq!(decimal32.to_string(), "1.23");
9673+
assert_eq!(format!("{decimal32:?}"), "Decimal32(1.23,3,2)");
9674+
9675+
let decimal64 = ScalarValue::Decimal64(Some(-12345), 5, 3);
9676+
assert_eq!(decimal64.to_string(), "-12.345");
9677+
assert_eq!(format!("{decimal64:?}"), "Decimal64(-12.345,5,3)");
9678+
9679+
let decimal128 = ScalarValue::Decimal128(Some(1), 1, 1);
9680+
assert_eq!(decimal128.to_string(), "0.1");
9681+
assert_eq!(format!("{decimal128:?}"), "Decimal128(0.1,1,1)");
9682+
9683+
let decimal128_trailing_zero = ScalarValue::Decimal128(Some(120), 3, 2);
9684+
assert_eq!(decimal128_trailing_zero.to_string(), "1.20");
9685+
assert_eq!(
9686+
format!("{decimal128_trailing_zero:?}"),
9687+
"Decimal128(1.20,3,2)"
9688+
);
9689+
9690+
let decimal256 = ScalarValue::Decimal256(Some(i256::from(100123)), 28, 3);
9691+
assert_eq!(decimal256.to_string(), "100.123");
9692+
assert_eq!(format!("{decimal256:?}"), "Decimal256(100.123,28,3)");
9693+
9694+
let null_decimal = ScalarValue::Decimal128(None, 10, 2);
9695+
assert_eq!(null_decimal.to_string(), "NULL");
9696+
assert_eq!(format!("{null_decimal:?}"), "Decimal128(NULL,10,2)");
9697+
}
9698+
96039699
#[test]
96049700
fn test_struct_display_null() {
96059701
let fields = vec![Field::new("a", DataType::Int32, false)];

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3345,7 +3345,11 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(
33453345

33463346
// To be able to remove user specific paths from the plan, for stable assertions
33473347
let testdata_clean = Path::new(&testdata).canonicalize()?.display().to_string();
3348-
let testdata_clean = testdata_clean.strip_prefix("/").unwrap_or(&testdata_clean);
3348+
let testdata_clean = testdata_clean.replace("\\", "/");
3349+
let testdata_clean = testdata_clean
3350+
.strip_prefix("//?/")
3351+
.or_else(|| testdata_clean.strip_prefix("/"))
3352+
.unwrap_or(&testdata_clean);
33493353

33503354
// Use displayable() rather than explain().collect() to avoid table formatting issues. We need
33513355
// to replace machine-specific paths with variable lengths, which breaks table alignment and

0 commit comments

Comments
 (0)