Skip to content

Commit 3902dc5

Browse files
committed
[fix](streaming-job) preserve gtids when rebuilding MySQL specific-offset from JSON
1 parent 19268b5 commit 3902dc5

4 files changed

Lines changed: 232 additions & 11 deletions

File tree

‎fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -957,17 +957,16 @@ private MySqlSourceConfig generateMySqlConfig(
957957
if (MapUtils.isEmpty(offsetMap)) {
958958
throw new RuntimeException("Incorrect offset " + startupMode);
959959
}
960-
if (offsetMap.containsKey(BinlogOffset.BINLOG_FILENAME_OFFSET_KEY)
961-
&& offsetMap.containsKey(BinlogOffset.BINLOG_POSITION_OFFSET_KEY)) {
962-
BinlogOffset binlogOffset =
963-
BinlogOffset.builder()
964-
.setBinlogFilePosition(
965-
offsetMap.get(BinlogOffset.BINLOG_FILENAME_OFFSET_KEY),
966-
Long.parseLong(
967-
offsetMap.get(
968-
BinlogOffset.BINLOG_POSITION_OFFSET_KEY)))
969-
.build();
970-
configFactory.startupOptions(StartupOptions.specificOffset(binlogOffset));
960+
boolean hasFilePosition =
961+
offsetMap.containsKey(BinlogOffset.BINLOG_FILENAME_OFFSET_KEY)
962+
&& offsetMap.containsKey(BinlogOffset.BINLOG_POSITION_OFFSET_KEY);
963+
boolean hasGtids = offsetMap.containsKey(BinlogOffset.GTID_SET_KEY);
964+
if (hasFilePosition || hasGtids) {
965+
// Rebuild from full map so gtids survive; supplement kind.
966+
offsetMap.putIfAbsent(
967+
BinlogOffset.OFFSET_KIND_KEY, BinlogOffsetKind.SPECIFIC.name());
968+
configFactory.startupOptions(
969+
StartupOptions.specificOffset(new BinlogOffset(offsetMap)));
971970
} else {
972971
throw new RuntimeException("Incorrect offset " + startupMode);
973972
}
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.cdcclient.itcase;
19+
20+
import static org.assertj.core.api.Assertions.assertThat;
21+
22+
import org.apache.doris.cdcclient.common.Env;
23+
24+
import com.fasterxml.jackson.databind.JsonNode;
25+
import com.fasterxml.jackson.databind.ObjectMapper;
26+
import org.junit.jupiter.api.AfterEach;
27+
import org.junit.jupiter.api.BeforeEach;
28+
import org.junit.jupiter.api.Test;
29+
import org.testcontainers.containers.MySQLContainer;
30+
import org.testcontainers.junit.jupiter.Container;
31+
import org.testcontainers.junit.jupiter.Testcontainers;
32+
import org.testcontainers.utility.DockerImageName;
33+
34+
import java.sql.Connection;
35+
import java.sql.DriverManager;
36+
import java.sql.ResultSet;
37+
import java.sql.Statement;
38+
import java.time.Duration;
39+
import java.util.ArrayList;
40+
import java.util.List;
41+
import java.util.concurrent.atomic.AtomicLong;
42+
43+
/**
44+
* GTID counterpart of {@link MySqlStartupSpecificOffsetITCase}: given a recorded GTID set as the
45+
* specific-offset (no binlog file/position), the job must replay from exactly that point forward —
46+
* proving the GTID-only resume offset is honored end-to-end, not just preserved in config.
47+
*/
48+
@Testcontainers
49+
class MySqlStartupGtidOffsetITCase {
50+
51+
private static final String ROOT_USER = "root";
52+
private static final String ROOT_PASSWORD = "123456";
53+
private static final ObjectMapper MAPPER = new ObjectMapper();
54+
private static final AtomicLong JOB_ID_SEQ = new AtomicLong(980_000);
55+
56+
@Container
57+
static final MySQLContainer<?> MYSQL =
58+
new MySQLContainer<>(DockerImageName.parse("mysql:8.0"))
59+
.withCommand("--gtid-mode=ON", "--enforce-gtid-consistency=ON")
60+
.withDatabaseName("cdc_test")
61+
.withUsername("cdc")
62+
.withPassword("123456")
63+
.withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD);
64+
65+
private String jobId;
66+
private String database;
67+
68+
@BeforeEach
69+
void setUp() throws Exception {
70+
jobId = String.valueOf(JOB_ID_SEQ.incrementAndGet());
71+
database = "gtidoff_db_" + jobId;
72+
try (Connection conn = rootConnection("");
73+
Statement st = conn.createStatement()) {
74+
st.execute("CREATE DATABASE " + database);
75+
st.execute("USE " + database);
76+
st.execute("CREATE TABLE t_user (id INT PRIMARY KEY, name VARCHAR(50))");
77+
st.execute("INSERT INTO t_user VALUES (1,'alice'), (2,'bob')");
78+
}
79+
}
80+
81+
@AfterEach
82+
void tearDown() throws Exception {
83+
Env.getCurrentEnv().close(jobId);
84+
try (Connection conn = rootConnection("");
85+
Statement st = conn.createStatement()) {
86+
st.execute("DROP DATABASE IF EXISTS " + database);
87+
}
88+
}
89+
90+
@Test
91+
void specificGtidOffsetReplaysFromRecordedPositionForward() throws Exception {
92+
// Record the GTID set right after the pre-existing rows; resume must start past it.
93+
String offset = String.format("{\"gtids\":\"%s\"}", currentGtidExecuted());
94+
95+
// This change happens after the recorded GTID but before the job starts — it must be read.
96+
insert("INSERT INTO t_user VALUES (3,'carol')");
97+
98+
try (MockDorisServer mock = new MockDorisServer();
99+
CdcClientWriteHarness harness =
100+
CdcClientWriteHarness.mysql(
101+
jobId,
102+
MYSQL.getHost(),
103+
MYSQL.getMappedPort(MySQLContainer.MYSQL_PORT),
104+
ROOT_USER,
105+
ROOT_PASSWORD,
106+
database,
107+
"t_user",
108+
offset,
109+
"doris_target_db",
110+
mock)) {
111+
112+
// Resume positions by GTID, so row 3 (already in the binlog) is read straight away.
113+
List<Integer> first = ids(harness.readBinlogFromStartupMode(1, Duration.ofSeconds(90)));
114+
assertThat(first).containsExactly(3);
115+
116+
insert("INSERT INTO t_user VALUES (4,'dave')");
117+
List<Integer> second = ids(harness.continueBinlog(1, Duration.ofSeconds(90)));
118+
assertThat(second).containsExactly(4);
119+
120+
// Replays from the recorded GTID forward: rows 3 and 4 only, never 1 or 2.
121+
List<Integer> all = ids(harness.loadedRecords());
122+
assertThat(all).doesNotContain(1, 2);
123+
assertThat(all).containsExactlyInAnyOrder(3, 4);
124+
}
125+
}
126+
127+
private String currentGtidExecuted() throws Exception {
128+
try (Connection conn = rootConnection(database);
129+
Statement st = conn.createStatement();
130+
ResultSet rs = st.executeQuery("SELECT @@gtid_executed")) {
131+
if (!rs.next()) {
132+
throw new IllegalStateException("@@gtid_executed returned no row");
133+
}
134+
// Single server uuid fits on one line; strip newlines defensively.
135+
return rs.getString(1).replace("\n", "").trim();
136+
}
137+
}
138+
139+
private List<Integer> ids(List<String> records) throws Exception {
140+
List<Integer> result = new ArrayList<>();
141+
for (String record : records) {
142+
JsonNode node = MAPPER.readTree(record);
143+
result.add(node.get("id").asInt());
144+
}
145+
return result;
146+
}
147+
148+
private void insert(String sql) throws Exception {
149+
try (Connection conn = rootConnection(database);
150+
Statement st = conn.createStatement()) {
151+
st.execute(sql);
152+
}
153+
}
154+
155+
private Connection rootConnection(String db) throws Exception {
156+
String url =
157+
"jdbc:mysql://"
158+
+ MYSQL.getHost()
159+
+ ":"
160+
+ MYSQL.getMappedPort(MySQLContainer.MYSQL_PORT)
161+
+ "/"
162+
+ db;
163+
return DriverManager.getConnection(url, ROOT_USER, ROOT_PASSWORD);
164+
}
165+
}

‎fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/GtidMultiIntervalOffsetTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ void splitIntervalSurvivesGtidSetSerializationRoundTrip() {
6565
2, new GtidSet(reSerialized).getUUIDSet(SERVER_UUID).getIntervals().size());
6666
}
6767

68+
// BinlogOffset-layer guard only; the real generateMySqlConfig resume path is covered by
69+
// MySqlSourceReaderTest / MySqlStartupGtidOffsetITCase.
6870
@Test
6971
void splitIntervalSurvivesFeOffsetMapRoundTrip() {
7072
// FE persists/restores the binlog offset as a string-valued map; rebuild from it.

‎fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReaderTest.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,24 @@
1818
package org.apache.doris.cdcclient.source.reader.mysql;
1919

2020
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertNotNull;
2122
import static org.junit.jupiter.api.Assertions.assertThrows;
2223

24+
import org.apache.doris.job.cdc.DataSourceConfigKeys;
25+
26+
import com.github.shyiko.mysql.binlog.GtidSet;
27+
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig;
28+
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset;
2329
import org.junit.jupiter.api.Test;
2430

31+
import java.lang.reflect.Method;
32+
import java.util.HashMap;
33+
import java.util.Map;
34+
2535
public class MySqlSourceReaderTest {
2636

37+
private static final String SERVER_UUID = "24bc7850-2c16-11e6-a073-0242ac110002";
38+
2739
@Test
2840
void testNormalizeSslModeMapsAllLegalValues() {
2941
assertEquals("disabled", MySqlSourceReader.normalizeSslModeForMysql("disable"));
@@ -59,4 +71,47 @@ void testNormalizeSslModeRejectsUppercase() {
5971
IllegalArgumentException.class,
6072
() -> MySqlSourceReader.normalizeSslModeForMysql("DISABLE"));
6173
}
74+
75+
// A JSON resume offset carrying file/pos AND a multi-interval gtid set must keep the gtids
76+
// (and a non-null kind) through generateMySqlConfig, not just file/pos.
77+
@Test
78+
void jsonOffsetWithGtidsPreservesMultiIntervalGtidSet() throws Exception {
79+
String gtid = SERVER_UUID + ":1-3:5-7";
80+
BinlogOffset off =
81+
startupBinlogOffset(
82+
"{\"file\":\"mysql-bin.000004\",\"pos\":\"1024\",\"gtids\":\""
83+
+ gtid
84+
+ "\"}");
85+
assertEquals(gtid, off.getGtidSet());
86+
assertNotNull(off.getOffsetKind());
87+
assertEquals(
88+
2, new GtidSet(off.getGtidSet()).getUUIDSet(SERVER_UUID).getIntervals().size());
89+
}
90+
91+
// A gtid-only JSON offset (no file/pos) is accepted, matching Flink CDC's specific-offset rules.
92+
@Test
93+
void jsonOffsetGtidOnlyIsAccepted() throws Exception {
94+
String gtid = SERVER_UUID + ":1-7";
95+
BinlogOffset off = startupBinlogOffset("{\"gtids\":\"" + gtid + "\"}");
96+
assertEquals(gtid, off.getGtidSet());
97+
assertNotNull(off.getOffsetKind());
98+
}
99+
100+
// Drive the real generateMySqlConfig JSON-offset path and return the rebuilt startup offset.
101+
private BinlogOffset startupBinlogOffset(String offsetJson) throws Exception {
102+
Map<String, String> cfg = new HashMap<>();
103+
cfg.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://localhost:3306/testdb");
104+
cfg.put(DataSourceConfigKeys.USER, "u");
105+
cfg.put(DataSourceConfigKeys.PASSWORD, "p");
106+
cfg.put(DataSourceConfigKeys.DATABASE, "testdb");
107+
cfg.put(DataSourceConfigKeys.TABLE, "t_test");
108+
cfg.put(DataSourceConfigKeys.OFFSET, offsetJson);
109+
Method m =
110+
MySqlSourceReader.class.getDeclaredMethod(
111+
"generateMySqlConfig", Map.class, String.class, int.class);
112+
m.setAccessible(true);
113+
MySqlSourceConfig config =
114+
(MySqlSourceConfig) m.invoke(new MySqlSourceReader(), cfg, "job-1", 0);
115+
return config.getStartupOptions().binlogOffset;
116+
}
62117
}

0 commit comments

Comments
 (0)