Skip to content

Commit b1ff201

Browse files
committed
[fix](regression) Align Paimon CHAR tests with MAP key validation
### What problem does this PR solve? Related PR: #69101, #69083 Problem Summary: The Paimon CHAR-padding suite assumes STRUCT MAP keys reach the writer, but SQL analysis now rejects complex MAP keys. Assert that rejection for all four complex-key groups and verify the tables remain empty. Keep primitive-key padding/collision checks and the remaining write/DML coverage. Direct complex-key normalization remains covered by PaimonWriteSchemaTest. ### Release note None ### Check List (For Author) - Test: Reproduced the original complete suite failure using the failing CI runtime. Regenerated expected output with the regression runner; an independent normal run passed the complete CHAR-padding and Variant-error suites with zero failures or skips. - Behavior changed: No product behavior changes. - Does this need documentation: No.
1 parent 86cd873 commit b1ff201

2 files changed

Lines changed: 46 additions & 55 deletions

File tree

‎regression-test/data/external_table_p0/paimon/write/test_paimon_write_char_padding.out‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@
1212
2 2 [k |kk] 1|2
1313
3 2 [k|k ] 1|2
1414

15-
-- !map_row_key_padded --
16-
1 1 [k ] 7
15+
-- !map_row_key_rejected --
16+
0
1717

18-
-- !map_nested_map_key_padded --
19-
2 1 [k ] 10
18+
-- !map_nested_map_key_rejected --
19+
0
2020

21-
-- !map_nan_key_padded --
22-
2 1 [k ] NaN 10
21+
-- !map_nan_key_rejected --
22+
0
2323

24-
-- !map_nested_nan_key_padded --
25-
2 2 [k ]|[k ] 10|20
24+
-- !map_nested_nan_key_rejected --
25+
0
2626

2727
-- !partition_padded --
2828
1 [x ]

‎regression-test/suites/external_table_p0/paimon/write/test_paimon_write_char_padding.groovy‎

Lines changed: 38 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -265,41 +265,37 @@ suite("test_paimon_write_char_padding", "p0,external,paimon") {
265265
FROM t_char_map_avro
266266
"""
267267

268-
// The writer takes a map key from the Arrow batch BE sends, where a ROW key holding a ROW is a columnar row
269-
// that cannot be hashed, so keys are compared by their bytes: such a key is padded, and two keys that
270-
// padding makes equal fail the row as above.
271-
sql """INSERT INTO t_char_map_row_key VALUES (1, map(named_struct('c', 'k', 'r', named_struct('i', 1)), 7))"""
268+
// SQL rejects complex MAP keys before the Paimon writer runs, even when padding would keep them distinct.
269+
// PaimonWriteSchemaTest covers the writer's complex-key normalization directly; these statements check
270+
// the SQL boundary and that rejected writes leave each table empty.
271+
test {
272+
sql """INSERT INTO t_char_map_row_key VALUES (1, map(named_struct('c', 'k', 'r', named_struct('i', 1)), 7))"""
273+
exception "MAP key type must be a primitive type"
274+
}
272275
test {
273276
sql """
274277
INSERT INTO t_char_map_row_key VALUES (2, map(named_struct('c', 'k', 'r', named_struct('i', 1)), 1,
275278
named_struct('c', 'k ', 'r', named_struct('i', 1)), 2))
276279
"""
277-
exception "into the same key"
280+
exception "MAP key type must be a primitive type"
278281
}
279-
order_qt_map_row_key_padded """
280-
SELECT id, map_size(m), concat('[', struct_element(map_keys(m)[1], 'c'), ']'), map_values(m)[1]
281-
FROM t_char_map_row_key
282-
"""
282+
order_qt_map_row_key_rejected """SELECT count(*) FROM t_char_map_row_key"""
283283

284-
// Paimon compares maps regardless of the order of their entries, and its Avro reader keeps one of two keys
285-
// whose nested maps differ only in that order, so such keys that padding makes equal fail the row too. A
286-
// key holding a map is padded.
284+
// A STRUCT containing a MAP is still a complex key, regardless of the nested entry order.
287285
test {
288286
sql """
289287
INSERT INTO t_char_map_nested_map_key VALUES (1, map(named_struct('c', 'k', 'm', map('a', 1, 'b', 2)),
290288
10, named_struct('c', 'k ', 'm', map('b', 2, 'a', 1)), 20))
291289
"""
292-
exception "into the same key"
290+
exception "MAP key type must be a primitive type"
293291
}
294-
sql """INSERT INTO t_char_map_nested_map_key VALUES (2, map(named_struct('c', 'k', 'm', map('a', 1)), 10))"""
295-
order_qt_map_nested_map_key_padded """
296-
SELECT id, map_size(m), concat('[', struct_element(map_keys(m)[1], 'c'), ']'), map_values(m)[1]
297-
FROM t_char_map_nested_map_key
298-
"""
292+
test {
293+
sql """INSERT INTO t_char_map_nested_map_key VALUES (2, map(named_struct('c', 'k', 'm', map('a', 1)), 10))"""
294+
exception "MAP key type must be a primitive type"
295+
}
296+
order_qt_map_nested_map_key_rejected """SELECT count(*) FROM t_char_map_nested_map_key"""
299297

300-
// Paimon compares a DOUBLE regardless of the bits of a NaN, and its Avro reader keeps one of two keys that
301-
// differ only so, while a NaN cast at run time keeps its sign. So the keys (k, NaN) and (k , -NaN), which
302-
// padding makes equal, fail the row too. A key holding a NaN is padded.
298+
// Runtime NaN values do not bypass the key-type check in INSERT SELECT.
303299
sql """INSERT INTO internal.${dbName}.char_nan_source VALUES ('NaN', '-NaN')"""
304300
test {
305301
sql """
@@ -308,42 +304,37 @@ suite("test_paimon_write_char_padding", "p0,external,paimon") {
308304
named_struct('c', 'k ', 'd', cast(neg AS DOUBLE)), 20)
309305
FROM internal.${dbName}.char_nan_source
310306
"""
311-
exception "into the same key"
307+
exception "MAP key type must be a primitive type"
312308
}
313-
sql """
314-
INSERT INTO t_char_map_nan_key
315-
SELECT 2, map(named_struct('c', 'k', 'd', cast(neg AS DOUBLE)), 10) FROM internal.${dbName}.char_nan_source
316-
"""
317-
order_qt_map_nan_key_padded """
318-
SELECT id, map_size(m), concat('[', struct_element(map_keys(m)[1], 'c'), ']'),
319-
struct_element(map_keys(m)[1], 'd'), map_values(m)[1]
320-
FROM t_char_map_nan_key
321-
"""
309+
test {
310+
sql """
311+
INSERT INTO t_char_map_nan_key
312+
SELECT 2, map(named_struct('c', 'k', 'd', cast(neg AS DOUBLE)), 10) FROM internal.${dbName}.char_nan_source
313+
"""
314+
exception "MAP key type must be a primitive type"
315+
}
316+
order_qt_map_nan_key_rejected """SELECT count(*) FROM t_char_map_nan_key"""
322317

323-
// Paimon's Avro reader keeps the last of the entries of a map whose keys are equal, so a nested map holding
324-
// NaN -> 1 and -NaN -> 2 reads back as {NaN: 2}, and the keys (k, {NaN: 1, -NaN: 2}) and (k , {NaN: 2}),
325-
// which padding then makes equal, fail the row too. Keys whose nested maps keep other last values are padded.
318+
// Nesting those NaNs inside another MAP also fails at the outer STRUCT key.
326319
test {
327320
sql """
328321
INSERT INTO t_char_map_nested_nan_key
329322
SELECT 1, map(named_struct('c', 'k', 'm', map(cast(pos AS DOUBLE), 1, cast(neg AS DOUBLE), 2)), 10,
330323
named_struct('c', 'k ', 'm', map(cast(pos AS DOUBLE), 2)), 20)
331324
FROM internal.${dbName}.char_nan_source
332325
"""
333-
exception "into the same key"
326+
exception "MAP key type must be a primitive type"
334327
}
335-
sql """
336-
INSERT INTO t_char_map_nested_nan_key
337-
SELECT 2, map(named_struct('c', 'k', 'm', map(cast(pos AS DOUBLE), 1, cast(neg AS DOUBLE), 2)), 10,
338-
named_struct('c', 'k ', 'm', map(cast(pos AS DOUBLE), 1)), 20)
339-
FROM internal.${dbName}.char_nan_source
340-
"""
341-
order_qt_map_nested_nan_key_padded """
342-
SELECT id, map_size(m),
343-
array_join(array_sort(array_map(k -> concat('[', struct_element(k, 'c'), ']'), map_keys(m))), '|'),
344-
array_join(array_sort(map_values(m)), '|')
345-
FROM t_char_map_nested_nan_key
346-
"""
328+
test {
329+
sql """
330+
INSERT INTO t_char_map_nested_nan_key
331+
SELECT 2, map(named_struct('c', 'k', 'm', map(cast(pos AS DOUBLE), 1, cast(neg AS DOUBLE), 2)), 10,
332+
named_struct('c', 'k ', 'm', map(cast(pos AS DOUBLE), 1)), 20)
333+
FROM internal.${dbName}.char_nan_source
334+
"""
335+
exception "MAP key type must be a primitive type"
336+
}
337+
order_qt_map_nested_nan_key_rejected """SELECT count(*) FROM t_char_map_nested_nan_key"""
347338

348339
// A partition value is padded whether it comes from the rows or from a static PARTITION clause, so a
349340
// second static overwrite of 'x' replaces the partition the first one wrote.

0 commit comments

Comments
 (0)