Skip to content

Commit 01cb339

Browse files
committed
[test](paimon) Forward-port table write coverage
### What problem does this PR solve? Issue Number: close apache#65086 Related PR: apache#65868, apache#66325, apache#67023, apache#67085 Problem Summary: Forward-port the final branch-4.1 Paimon write regression matrix to master, including append and primary-key modes, bucket routing, transactions, schema evolution, row-level DML, Variant V2, snapshot references, writer lifecycle, cache coherence, and stabilized external paths. Generated golden outputs are preserved byte-for-byte from branch-4.1. ### Release note None ### Check List (For Author) - Test: Regression test - Test cases and generated expected outputs were forward-ported; execution is deferred until the final integrated validation. - Behavior changed: No - Does this need documentation: No
1 parent 1865105 commit 01cb339

64 files changed

Lines changed: 10949 additions & 49 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !paimon_alter_initial_desc --
3+
id int Yes true \N identifier
4+
required_value bigint Yes true \N
5+
score int Yes true \N initial score
6+
MixedCase text Yes true \N
7+
obsolete text Yes true \N
8+
amount decimal(8,2) Yes true \N
9+
10+
-- !paimon_alter_initial_schema --
11+
0 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}] [] ["id"]
12+
13+
-- !paimon_alter_add_column_desc --
14+
id int Yes true \N identifier
15+
required_value bigint Yes true \N
16+
score int Yes true \N initial score
17+
added_after text Yes true \N added column
18+
MixedCase text Yes true \N
19+
obsolete text Yes true \N
20+
amount decimal(8,2) Yes true \N
21+
22+
-- !paimon_alter_add_column_schema --
23+
1 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":6,"name":"added_after","type":"STRING","description":"added column","defaultValue":"unknown"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}]
24+
25+
-- !paimon_alter_add_columns_desc --
26+
id int Yes true \N identifier
27+
required_value bigint Yes true \N
28+
score int Yes true \N initial score
29+
added_after text Yes true \N added column
30+
MixedCase text Yes true \N
31+
obsolete text Yes true \N
32+
amount decimal(8,2) Yes true \N
33+
tiny_col tinyint Yes true \N
34+
small_col smallint Yes true \N small column
35+
profile struct<city:text,zip:int> Yes true \N
36+
37+
-- !paimon_alter_add_first_desc --
38+
first_col bigint Yes true \N
39+
id int Yes true \N identifier
40+
required_value bigint Yes true \N
41+
score int Yes true \N initial score
42+
added_after text Yes true \N added column
43+
MixedCase text Yes true \N
44+
obsolete text Yes true \N
45+
amount decimal(8,2) Yes true \N
46+
tiny_col tinyint Yes true \N
47+
small_col smallint Yes true \N small column
48+
profile struct<city:text,zip:int> Yes true \N
49+
50+
-- !paimon_alter_rename_column_desc --
51+
first_col bigint Yes true \N
52+
id int Yes true \N identifier
53+
required_value bigint Yes true \N
54+
score int Yes true \N initial score
55+
added_after text Yes true \N added column
56+
display_name text Yes true \N
57+
obsolete text Yes true \N
58+
amount decimal(8,2) Yes true \N
59+
tiny_col tinyint Yes true \N
60+
small_col smallint Yes true \N small column
61+
profile struct<city:text,zip:int> Yes true \N
62+
63+
-- !paimon_alter_modify_column_desc --
64+
score bigint Yes true \N updated score
65+
first_col bigint Yes true \N
66+
id int Yes true \N identifier
67+
required_value bigint Yes true \N
68+
added_after text Yes true \N
69+
display_name text Yes true \N
70+
obsolete text Yes true \N
71+
amount decimal(12,2) Yes true \N
72+
tiny_col tinyint Yes true \N
73+
small_col int Yes true \N
74+
profile struct<city:text,zip:int> Yes true \N
75+
76+
-- !paimon_alter_modify_column_schema --
77+
9 [{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":12,"name":"first_col","type":"BIGINT","description":""},{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""}]
78+
79+
-- !paimon_alter_drop_column_desc --
80+
score bigint Yes true \N updated score
81+
first_col bigint Yes true \N
82+
id int Yes true \N identifier
83+
required_value bigint Yes true \N
84+
added_after text Yes true \N
85+
display_name text Yes true \N
86+
amount decimal(12,2) Yes true \N
87+
tiny_col tinyint Yes true \N
88+
small_col int Yes true \N
89+
profile struct<city:text,zip:int> Yes true \N
90+
91+
-- !paimon_alter_reorder_columns_desc --
92+
id int Yes true \N identifier
93+
display_name text Yes true \N
94+
score bigint Yes true \N updated score
95+
required_value bigint Yes true \N
96+
small_col int Yes true \N
97+
tiny_col tinyint Yes true \N
98+
added_after text Yes true \N
99+
amount decimal(12,2) Yes true \N
100+
profile struct<city:text,zip:int> Yes true \N
101+
first_col bigint Yes true \N
102+
103+
-- !paimon_alter_final_schema --
104+
11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}] [] ["id"]
105+
106+
-- !paimon_alter_failed_batch_schema --
107+
11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}]
108+
109+
-- !paimon_alter_partition_initial_desc --
110+
id int Yes true \N
111+
pt text Yes true \N
112+
payload int Yes true \N
113+
114+
-- !paimon_alter_partition_initial_schema --
115+
0 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"INT","description":""}] ["pt"] ["id","pt"]
116+
117+
-- !paimon_alter_partition_final_desc --
118+
id int Yes true \N
119+
pt text Yes true \N
120+
payload bigint Yes true \N
121+
extra text Yes true \N
122+
123+
-- !paimon_alter_partition_final_schema --
124+
2 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"BIGINT","description":""},{"id":3,"name":"extra","type":"STRING","description":""}] ["pt"] ["id","pt"]
125+

‎regression-test/data/external_table_p0/paimon/test_paimon_jdbc_catalog.out‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,19 @@
33
1 alice 2025-01-01
44
2 bob 2025-01-02
55

6+
-- !paimon_jdbc_concurrent_append --
7+
left 128 128 8128
8+
right 128 128 136128
9+
10+
-- !paimon_jdbc_same_partition_append --
11+
128 128 324032
12+
13+
-- !paimon_jdbc_concurrent_aggregation --
14+
1 30
15+
16+
-- !paimon_jdbc_concurrent_dynamic --
17+
left 64 64
18+
right 64 64
19+
20+
-- !paimon_jdbc_merge_during_compact --
21+
2 merge-during-compact 200

‎regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,14 @@
66
-- !before_snapshots --
77
1
88

9-
-- !after_rows --
9+
-- !after_append_rows --
1010
1 10 base-1
1111
2 20 base-2
12+
3 30 insert-values
13+
4 40 insert-select
1214

13-
-- !after_snapshots --
14-
1
15+
-- !after_rows --
16+
6 60 merge-insert
1517

18+
-- !after_snapshots --
19+
7
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !create_custom_location_absent --
3+
4+
-- !create_sequence_result --
5+
1 100 newer p1
6+
2 20 updated-2 p1
7+
3 5 initial-3 p2
8+
9+
-- !create_sequence_schema --
10+
["dt"] ["id","dt"] created by Doris with write properties
11+
12+
-- !create_sequence_file_format --
13+
orc
14+
15+
-- !create_partial_result --
16+
1 alice 15 updated
17+
2 bob 20 initial
18+
19+
-- !create_first_row_result --
20+
1 first-1 10
21+
2 first-2 20
22+
3 first-3 30
23+
24+
-- !create_aggregation_result --
25+
1 17 90 latest
26+
2 8 70 stable
27+
3 4 50 new
28+
29+
-- !create_lookup_changelog --
30+
+I 1 new 11
31+
+I 2 stable 20
32+
+I 3 added 30
33+
34+
-- !create_lookup_result --
35+
1 new 11
36+
2 stable 20
37+
3 added 30
38+
39+
-- !create_dynamic_bucket_result --
40+
12 0 11
41+
42+
-- !create_dynamic_bucket_rows --
43+
p1 0 v0
44+
p1 1 v1
45+
p1 10 v10
46+
p1 11 v11
47+
p1 2 v2
48+
p1 3 v3
49+
p1 4 v4
50+
p1 5 v5
51+
p1 6 v6
52+
p1 7 v7
53+
p1 8 v8
54+
p1 9 v9
55+
56+
-- !create_dynamic_bucket_files --
57+
0
58+
1
59+
2
60+
3
61+
62+
-- !create_write_options --
63+
aggregation bucket 1
64+
aggregation fields.highest.aggregate-function max
65+
aggregation fields.total.aggregate-function sum
66+
aggregation merge-engine aggregation
67+
dynamic_bucket bucket -1
68+
dynamic_bucket dynamic-bucket.initial-buckets 1
69+
dynamic_bucket dynamic-bucket.max-buckets 4
70+
dynamic_bucket dynamic-bucket.target-row-num 2
71+
first_row bucket 1
72+
first_row merge-engine first-row
73+
lookup bucket 1
74+
lookup changelog-producer lookup
75+
partial_update bucket 2
76+
partial_update bucket-key id
77+
partial_update merge-engine partial-update
78+
sequence bucket 2
79+
sequence bucket-key id
80+
sequence file.format orc
81+
sequence sequence.field seq
82+
sequence snapshot.num-retained.max 5
83+
sequence snapshot.num-retained.min 2
84+
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !ao_basic --
3+
1 alice 95.5
4+
2 bob 87
5+
3 charlie 92.3
6+
7+
-- !ao_part --
8+
1 alice 95.5 east
9+
2 bob 87 west
10+
3 charlie 92.3 east
11+
4 diana 88 north
12+
5 erin 86.5 south
13+
6 \N \N east
14+
15+
-- !ao_auto_partition_data --
16+
1 alpha 2026-07-01
17+
2 beta 2026-07-02
18+
3 gamma 2026-07-01
19+
4 delta 2026-07-01
20+
5 epsilon 2026-07-03
21+
6 default_partition \N
22+
23+
-- !ao_auto_partition_metadata --
24+
dt=2026-07-01 3
25+
dt=2026-07-02 1
26+
dt=2026-07-03 1
27+
dt=__DEFAULT_PARTITION__ 1
28+
29+
-- !ao_empty --
30+
1 \N
31+
2 reordered
32+
33+
-- !ao_default_value --
34+
1 unknown
35+
36+
-- !ao_default_after_explicit_null --
37+
1 unknown
38+
39+
-- !ao_partition_default_data --
40+
1 omitted-partition 2026-07-01
41+
42+
-- !ao_partition_default_metadata --
43+
dt=2026-07-01 1
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !bucket_hash_fixed --
3+
16 0 15 2
4+
5+
-- !bucket_rescale_partial --
6+
p1 1 p1-old-1
7+
p1 10 p1-after-rescale
8+
p1 2 p1-old-2
9+
p2 3 p2-old-3
10+
p2 4 p2-old-4
11+
12+
-- !bucket_hash_dynamic --
13+
p1 1 v1_updated
14+
p1 2 v2
15+
p1 3 v3
16+
p1 4 v4_updated
17+
p1 5 v5
18+
p1 6 v6
19+
p2 1 p2_v1
20+
p2 2 p2_v2_updated
21+
22+
-- !bucket_hash_dynamic_partial --
23+
p1 1 alice 15
24+
p1 2 bob 20
25+
p1 3 \N 30
26+
27+
-- !bucket_hash_dynamic_overwrite --
28+
10 new_10
29+
11 new_11
30+
12 new_12
31+
32+
-- !bucket_key_dynamic --
33+
p2 1 id1_moved
34+
p2 2 id2_stable
35+
p2 4 id4_added
36+
p3 3 id3_moved
37+
38+
-- !bucket_key_dynamic_partial --
39+
p1 10 old_10 15
40+
p2 20 stable_20 20
41+
p3 30 \N 30
42+
43+
-- !bucket_key_dynamic_first_row --
44+
p1 1 first_1
45+
p2 2 first_2
46+
47+
-- !bucket_key_dynamic_aggregation --
48+
p1 1 17
49+
p2 2 20
50+
51+
-- !bucket_key_dynamic_scale_samples --
52+
p2 1023 txn2_1023
53+
p2 2047 txn2_2047
54+
p3 0 txn2_0
55+
p4 3071 txn3_3071
56+
p4 4095 txn3_4095
57+
p5 2048 txn3_2048
58+
59+
-- !bucket_unaware --
60+
32 0 31 2
61+
62+
-- !bucket_postpone --
63+
0
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !changelog_input_partial --
3+
1 alice 15
4+
2 bob 20
5+
3 \N 30
6+
7+
-- !changelog_lookup --
8+
1 new 11
9+
2 stable 20
10+
3 added 30
11+
12+
-- !changelog_lookup_aggregation --
13+
1 17
14+
2 20
15+
3 30
16+
17+
-- !changelog_full_compaction --
18+
p1 1 new
19+
p2 2 stable
20+
p2 3 added
21+
22+
-- !changelog_full_compaction_dynamic --
23+
p1 1 new_1
24+
p1 2 stable_2
25+
p1 4 added_4
26+
p2 3 new_3
27+
p2 5 added_5
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !compaction_pk --
3+
1 new 11
4+
2 stable 20
5+
3 added 30
6+
7+
-- !compaction_append --
8+
1 a
9+
2 b
10+
3 c
11+
4 d

0 commit comments

Comments
 (0)