Skip to content

Commit 090b476

Browse files
authored
[regression-test](streaming-job) add cdc cases (composite/concurrent-dml/id-gap/decimal/datetime pk) and fix split-bound java.time deserialize (apache#63471)
### What problem does this PR solve? Extend CDC streaming-job regression coverage across MySQL/Postgres for primary-key related scenarios that the existing suites did not exercise; plus a small cdc-client follow-up to apache#63219 so split-bound restore handles the modern java.time types that some JDBC drivers return by default. #### Regression cases added | Case | Tables | Guarded by | |---|---|---| | `*_composite_pk` | composite PK + full-PK mapping table | composite PK chunk split + INSERT/UPDATE/DELETE locating across all PK columns; tenant/org-boundary chunk slicing | | `*_snapshot_with_concurrent_dml` | 1000-row table + unrelated decoy | source-side INSERT/UPDATE/DELETE during snapshot phase converges correctly; unrelated table outside include_tables not leaked | | `*_id_gap_completeness` | dense id 1~100 + outlier id 10000000 + post-outlier 10000001~10000100 | snapshot reader covers tail rows past the chunk-time max without dropping; binlog DML across all three id regions applies correctly | | `*_decimal_pk` | DECIMAL(20,4) PK + (MySQL only) BIGINT UNSIGNED PK | evenly-path BigDecimal / BigInteger arithmetic in chunk splitter; bigint unsigned values up to 2^64-1 | | \`mysql_*_datetime_pk\` | DATETIME(6) PK + composite (DATETIME, id) PK | java.time.LocalDateTime split-bound JSON round-trip; composite temporal PK locating |
1 parent d0536c0 commit 090b476

20 files changed

Lines changed: 1743 additions & 16 deletions

‎fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/AbstractCdcSourceReader.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.io.IOException;
3131
import java.util.Map;
3232
import java.util.concurrent.ConcurrentHashMap;
33+
import java.util.function.Function;
3334

3435
import com.fasterxml.jackson.databind.JsonNode;
3536
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -88,22 +89,26 @@ protected static Object[] convertBounds(Object[] raw, Class<?> target, ObjectMap
8889
return out;
8990
}
9091

92+
private static final Map<Class<?>, Function<String, Object>> BOUND_PARSERS =
93+
Map.of(
94+
java.sql.Date.class, java.sql.Date::valueOf,
95+
java.sql.Timestamp.class, java.sql.Timestamp::valueOf,
96+
java.sql.Time.class, java.sql.Time::valueOf,
97+
java.time.LocalDateTime.class, java.time.LocalDateTime::parse,
98+
java.time.LocalDate.class, java.time.LocalDate::parse,
99+
java.time.LocalTime.class, java.time.LocalTime::parse,
100+
java.time.OffsetDateTime.class, java.time.OffsetDateTime::parse);
101+
91102
private static Object convertBound(Object v, Class<?> target, ObjectMapper mapper) {
92103
if (v == null) {
93104
return null;
94105
}
95106
if (target.isInstance(v)) {
96107
return v;
97108
}
98-
String s = v.toString();
99-
if (target == java.sql.Date.class) {
100-
return java.sql.Date.valueOf(s);
101-
}
102-
if (target == java.sql.Timestamp.class) {
103-
return java.sql.Timestamp.valueOf(s);
104-
}
105-
if (target == java.sql.Time.class) {
106-
return java.sql.Time.valueOf(s);
109+
Function<String, Object> parser = BOUND_PARSERS.get(target);
110+
if (parser != null) {
111+
return parser.apply(v.toString());
107112
}
108113
return mapper.convertValue(v, target);
109114
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !desc_composite_pk --
3+
order_id bigint No true \N
4+
tenant_id int No true \N
5+
order_no varchar(192) Yes false \N NONE
6+
amount decimal(10,2) Yes false \N NONE
7+
8+
-- !select_snapshot --
9+
1 1001 A_001 10.00
10+
1 1002 A_002 20.00
11+
1 1003 A_003 30.00
12+
1 1004 A_004 40.00
13+
1 1005 A_005 50.00
14+
2 2001 B_001 100.00
15+
2 2002 B_002 200.00
16+
2 2003 B_003 300.00
17+
2 2004 B_004 400.00
18+
2 2005 B_005 500.00
19+
20+
-- !select_snapshot_map --
21+
10 100
22+
10 101
23+
10 102
24+
20 200
25+
20 201
26+
20 202
27+
28+
-- !select_after_incr --
29+
1 1002 A_002 20.00
30+
1 1003 A_003 999.99
31+
1 1004 A_004 40.00
32+
1 1005 A_005 50.00
33+
1 1006 A_006 60.00
34+
2 2001 B_001 100.00
35+
2 2002 B_002 888.88
36+
2 2003 B_003 300.00
37+
2 2004 B_004 400.00
38+
2 2006 B_006 600.00
39+
40+
-- !select_after_incr_map --
41+
10 101
42+
10 102
43+
10 103
44+
20 200
45+
20 201
46+
30 300
47+
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_snapshot_datetime_pk --
3+
2024-01-01T00:00 A1
4+
2024-06-15T12:00:00.123456 B1
5+
2025-01-01T00:00 C1
6+
2025-06-15T12:34:56.999999 D1
7+
2026-01-01T00:00 E1
8+
9+
-- !select_snapshot_composite_pk --
10+
2024-02-01T00:00 1 A2
11+
2024-02-01T00:00 2 B2
12+
2024-02-02T12:00:00.500 3 C2
13+
2024-02-03T23:59:59.999999 4 D2
14+
2024-02-04T00:00 5 E2
15+
16+
-- !select_after_incr_datetime_pk --
17+
2024-01-01T00:00 A1
18+
2024-06-15T12:00:00.123456 B2_upd
19+
2025-01-01T00:00 C1
20+
2026-01-01T00:00 E1
21+
2026-06-01T00:00 F2
22+
23+
-- !select_after_incr_composite_pk --
24+
2024-02-01T00:00 1 A2
25+
2024-02-01T00:00 2 B2
26+
2024-02-02T12:00:00.500 3 C3_upd
27+
2024-02-04T00:00 5 E2
28+
2024-02-05T00:00 6 F3
29+
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_snapshot_decimal --
3+
1.0000 A1
4+
2.5000 B1
5+
3.7500 C1
6+
4.1234 D1
7+
9999999999999.9999 E1
8+
9+
-- !select_snapshot_bigint_unsigned --
10+
0 zero
11+
1 one
12+
9223372036854775807 signed_max
13+
9223372036854775808 signed_max_plus1
14+
18446744073709551615 unsigned_max
15+
16+
-- !select_after_incr_decimal --
17+
1.0000 A1
18+
2.5000 B2_upd
19+
3.7500 C1
20+
5.5555 F2
21+
9999999999999.9999 E1
22+
23+
-- !select_after_incr_bigint_unsigned --
24+
0 zero
25+
1 one
26+
9223372036854775807 signed_max_upd
27+
10000000000000000000 incr_huge
28+
18446744073709551615 unsigned_max
29+
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_snapshot_count --
3+
201
4+
5+
-- !select_snapshot_dense_sample --
6+
1 dense
7+
50 dense
8+
100 dense
9+
10+
-- !select_snapshot_outlier_sample --
11+
10000000 outlier
12+
10000050 post_outlier
13+
10000100 post_outlier
14+
15+
-- !select_after_incr_count --
16+
201
17+
18+
-- !select_after_incr_changed --
19+
50 dense 99
20+
10000200 incr_outlier 1
21+
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_count --
3+
1007
4+
5+
-- !select_updates --
6+
1 99
7+
100 99
8+
500 99
9+
999 99
10+
11+
-- !select_deletes --
12+
0
13+
14+
-- !select_inserts --
15+
1001 concurrent_ins 1
16+
1002 concurrent_ins 1
17+
1003 concurrent_ins 1
18+
1004 concurrent_ins 1
19+
1005 concurrent_ins 1
20+
1006 concurrent_ins 1
21+
1007 concurrent_ins 1
22+
1008 concurrent_ins 1
23+
1009 concurrent_ins 1
24+
1010 concurrent_ins 1
25+
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !desc_composite_pk --
3+
org_id int No true \N
4+
item_id bigint No true \N
5+
item_no text Yes false \N NONE
6+
qty int Yes false \N NONE
7+
8+
-- !select_snapshot --
9+
1 1001 A_001 10
10+
1 1002 A_002 20
11+
1 1003 A_003 30
12+
1 1004 A_004 40
13+
1 1005 A_005 50
14+
2 2001 B_001 100
15+
2 2002 B_002 200
16+
2 2003 B_003 300
17+
2 2004 B_004 400
18+
2 2005 B_005 500
19+
20+
-- !select_snapshot_map --
21+
10 100
22+
10 101
23+
10 102
24+
20 200
25+
20 201
26+
20 202
27+
28+
-- !select_after_incr --
29+
1 1002 A_002 20
30+
1 1003 A_003 999
31+
1 1004 A_004 40
32+
1 1005 A_005 50
33+
1 1006 A_006 60
34+
2 2001 B_001 100
35+
2 2002 B_002 888
36+
2 2003 B_003 300
37+
2 2004 B_004 400
38+
2 2006 B_006 600
39+
40+
-- !select_after_incr_map --
41+
10 101
42+
10 102
43+
10 103
44+
20 200
45+
20 201
46+
30 300
47+
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_snapshot --
3+
1.0000 A1
4+
2.5000 B1
5+
3.7500 C1
6+
4.1234 D1
7+
9999999999999.9999 E1
8+
9+
-- !select_after_incr --
10+
1.0000 A1
11+
2.5000 B2_upd
12+
3.7500 C1
13+
5.5555 F2
14+
9999999999999.9999 E1
15+
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_snapshot_count --
3+
201
4+
5+
-- !select_snapshot_dense_sample --
6+
1 dense
7+
50 dense
8+
100 dense
9+
10+
-- !select_snapshot_outlier_sample --
11+
10000000 outlier
12+
10000050 post_outlier
13+
10000100 post_outlier
14+
15+
-- !select_after_incr_count --
16+
201
17+
18+
-- !select_after_incr_changed --
19+
50 dense 99
20+
10000200 incr_outlier 1
21+
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select_count --
3+
1007
4+
5+
-- !select_updates --
6+
1 99
7+
100 99
8+
500 99
9+
999 99
10+
11+
-- !select_deletes --
12+
0
13+
14+
-- !select_inserts --
15+
1001 concurrent_ins 1
16+
1002 concurrent_ins 1
17+
1003 concurrent_ins 1
18+
1004 concurrent_ins 1
19+
1005 concurrent_ins 1
20+
1006 concurrent_ins 1
21+
1007 concurrent_ins 1
22+
1008 concurrent_ins 1
23+
1009 concurrent_ins 1
24+
1010 concurrent_ins 1
25+

0 commit comments

Comments
 (0)