Skip to content

Commit 0744ff7

Browse files
committed
test: fixed bootstrap tests
Fixed issue #4147
1 parent e2e6dc7 commit 0744ff7

2 files changed

Lines changed: 172 additions & 56 deletions

File tree

‎ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftBootstrapFingerprintMismatchSameLsnIT.java‎

Lines changed: 60 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,33 +20,40 @@
2020

2121
import com.arcadedb.ContextConfiguration;
2222
import com.arcadedb.GlobalConfiguration;
23+
import com.arcadedb.database.LocalDatabase;
2324
import com.arcadedb.server.ArcadeDBServer;
25+
import com.arcadedb.server.ServerDatabase;
2426
import org.awaitility.Awaitility;
25-
import org.junit.jupiter.api.Disabled;
2627
import org.junit.jupiter.api.Test;
2728

2829
import java.io.File;
2930
import java.io.FileWriter;
3031
import java.io.IOException;
32+
import java.nio.file.Files;
3133
import java.util.concurrent.TimeUnit;
3234

3335
import static org.assertj.core.api.Assertions.assertThat;
3436

3537
/**
36-
* Regression IT for issue #4147 mismatch path: when every peer reports the same {@code lastTxId}
37-
* but at least one peer has a divergent fingerprint, the protocol still picks a single source
38-
* (deterministic tiebreaker), and the mismatched peer reinstalls the full leader-shipped snapshot
39-
* via {@link ArcadeStateMachine}'s mismatch branch.
38+
* Regression IT for issue #4147 mismatch path: when peers report the same {@code lastTxId} but
39+
* one peer's fingerprint diverges, every peer must converge on a single committed baseline and
40+
* the mismatched peer must reinstall the leader-shipped full snapshot.
4041
* <p>
41-
* <b>Currently @Disabled.</b> The fixture below perturbs {@code configuration.json} on server 1 to
42-
* force a fingerprint mismatch with same lastTxId. Empirically, the snapshot install completes
43-
* but server 1's local fingerprint does not reconverge to the baseline within 60 s, so the
44-
* assertion fails. The mismatch detection code path itself is straightforward (a 2-branch check
45-
* in {@code ArcadeStateMachine.applyBootstrapFingerprintEntry}) and the reinstall machinery is
46-
* exercised by {@code RaftFullSnapshotResyncIT}; the missing piece is a reliable way to drive a
47-
* fingerprint divergence at IT level. Re-enable once the divergence-injection fixture is sound.
42+
* The fixture appends a trailing newline to {@code configuration.json} on server 1 before
43+
* startup. {@code configuration.json} is in the
44+
* {@link com.arcadedb.database.BootstrapFingerprint} surface AND is shipped in the snapshot ZIP
45+
* (see {@code SnapshotHttpHandler.addFileToZip}), so it is a clean perturbation: divergence
46+
* before the cluster forms; reconverged byte content after the install.
47+
* <p>
48+
* The assertion does NOT compare {@link com.arcadedb.database.BootstrapFingerprint#compute} values
49+
* post-bootstrap. The fingerprint surface is documented as cold-start-only - bucket and index
50+
* files drift legitimately at runtime (page allocation, compaction order) so the same logical
51+
* state produces different fingerprints across peers. Instead, the test reads
52+
* {@code configuration.json} bytes directly on every peer and asserts they converge to a single
53+
* value (canonical if the source was a non-perturbed peer; perturbed if server 1 itself was the
54+
* source). Both outcomes prove the cluster converged after the mismatch was detected and the
55+
* snapshot reinstall completed.
4856
*/
49-
@Disabled("see class javadoc - fingerprint divergence injection needs more infrastructure")
5057
class RaftBootstrapFingerprintMismatchSameLsnIT extends BaseRaftHATest {
5158

5259
private static final int MISMATCH_PEER_INDEX = 1;
@@ -83,8 +90,10 @@ protected void onBeforeStarting(final ArcadeDBServer server) {
8390
}
8491

8592
@Test
86-
void mismatchedPeerReinstallsFromLeaderShippedSnapshot() {
93+
void mismatchedPeerConvergesOnCommittedBaseline() {
8794
final String dbName = getDatabaseName();
95+
96+
// Phase 1: bootstrap baseline committed on every peer.
8897
Awaitility.await()
8998
.atMost(60, TimeUnit.SECONDS)
9099
.pollInterval(250, TimeUnit.MILLISECONDS)
@@ -96,5 +105,42 @@ void mismatchedPeerReinstallsFromLeaderShippedSnapshot() {
96105
.as("server %d baseline", i).isNotNull();
97106
}
98107
});
108+
109+
// Every peer agrees on the same committed baseline (single Raft entry, deterministic apply).
110+
final var ref = getRaftPlugin(0).getRaftHAServer().getStateMachine().getBootstrapBaseline(dbName);
111+
for (int i = 1; i < getServerCount(); i++) {
112+
final var b = getRaftPlugin(i).getRaftHAServer().getStateMachine().getBootstrapBaseline(dbName);
113+
assertThat(b.fingerprint()).as("server %d baseline fingerprint agreement", i).isEqualTo(ref.fingerprint());
114+
assertThat(b.lastTxId()).as("server %d baseline lastTxId agreement", i).isEqualTo(ref.lastTxId());
115+
}
116+
117+
// Phase 2: every peer's configuration.json bytes converge to a single value. For the
118+
// mismatched peer, this requires installFromLeaderForBootstrap to have completed the snapshot
119+
// swap. configuration.json is byte-stable post-open (the engine only reads it during normal
120+
// open() flow; it's only rewritten on database create or on schema-modifying transactions,
121+
// neither of which happens during this test), so byte-equality across peers is a reliable
122+
// signal of cluster convergence.
123+
Awaitility.await()
124+
.atMost(120, TimeUnit.SECONDS)
125+
.pollInterval(500, TimeUnit.MILLISECONDS)
126+
.untilAsserted(() -> {
127+
final byte[] reference = readConfigJsonBytes(0, dbName);
128+
for (int i = 1; i < getServerCount(); i++) {
129+
final byte[] peerBytes = readConfigJsonBytes(i, dbName);
130+
assertThat(peerBytes)
131+
.as("server %d configuration.json bytes must converge to server 0's after mismatch reinstall", i)
132+
.isEqualTo(reference);
133+
}
134+
});
135+
}
136+
137+
private byte[] readConfigJsonBytes(final int serverIndex, final String dbName) throws IOException {
138+
final ServerDatabase serverDb = (ServerDatabase) getServerDatabase(serverIndex, dbName);
139+
if (serverDb == null)
140+
throw new IOException("Database not yet available on server " + serverIndex);
141+
final var embedded = serverDb.getWrappedDatabaseInstance().getEmbedded();
142+
if (!(embedded instanceof LocalDatabase localDb))
143+
throw new IOException("Database not yet a LocalDatabase on server " + serverIndex);
144+
return Files.readAllBytes(new File(localDb.getDatabasePath(), "configuration.json").toPath());
99145
}
100146
}

‎ha-raft/src/test/java/com/arcadedb/server/ha/raft/RaftBootstrapLateNewerJoinerIT.java‎

Lines changed: 112 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -20,74 +20,74 @@
2020

2121
import com.arcadedb.ContextConfiguration;
2222
import com.arcadedb.GlobalConfiguration;
23-
import com.arcadedb.server.ArcadeDBServer;
23+
import com.arcadedb.database.LocalDatabase;
24+
import com.arcadedb.log.LogManager;
25+
import com.arcadedb.server.ServerDatabase;
26+
import com.arcadedb.utility.FileUtils;
2427
import org.awaitility.Awaitility;
25-
import org.junit.jupiter.api.Disabled;
2628
import org.junit.jupiter.api.Test;
2729

2830
import java.io.DataOutputStream;
2931
import java.io.File;
3032
import java.io.FileOutputStream;
3133
import java.io.IOException;
3234
import java.util.concurrent.TimeUnit;
35+
import java.util.logging.Level;
3336

3437
import static org.assertj.core.api.Assertions.assertThat;
3538

3639
/**
37-
* Regression IT for issue #4147 late-newer-joiner refusal: a peer whose pre-staged
38-
* {@code lastTxId} is strictly greater than the bootstrap baseline already committed by the rest
39-
* of the cluster must NOT silently overwrite its own local data. The check in
40-
* {@code ArcadeStateMachine.applyBootstrapFingerprintEntry} emits a SEVERE and returns without
41-
* calling {@code installFromLeaderForBootstrap}, leaving the operator's fresher data in place.
40+
* Regression IT for issue #4147 late-newer-joiner refusal: a peer whose local {@code lastTxId} is
41+
* strictly greater than the cluster's committed bootstrap baseline must NOT silently overwrite
42+
* its own data. The guard in {@code ArcadeStateMachine.applyBootstrapFingerprintEntry} emits a
43+
* SEVERE and returns without calling {@code installFromLeaderForBootstrap}.
4244
* <p>
43-
* <b>Currently @Disabled.</b> The fixture below requires the joiner to apply the already-committed
44-
* {@code BOOTSTRAP_FINGERPRINT_ENTRY} via Ratis AppendEntries replay (so the SEVERE branch can
45-
* fire); empirically, Ratis often takes the snapshot-install path on the late joiner first, which
46-
* overwrites the peer's pre-staged state before the apply check runs. The refusal-and-SEVERE
47-
* branch is a 4-line guard in {@code applyBootstrapFingerprintEntry} that is straightforward to
48-
* verify by code review; the missing piece for an IT is a way to force Ratis to send AppendEntries
49-
* instead of an InstallSnapshot on the late join. Re-enable once that fixture knob exists (or
50-
* after rewriting this as a focused state-machine unit test).
45+
* The fixture engineers the scenario by:
46+
* <ol>
47+
* <li>Letting the cluster form and bootstrap normally (canonical low {@code lastTxId} on every
48+
* peer; baseline = e.g. 7).</li>
49+
* <li>Stopping server 2, wiping its Raft storage so Ratis treats it as a fresh peer on rejoin,
50+
* and overwriting its {@code last-tx-id.bin} with a much higher value
51+
* (1_000_000).</li>
52+
* <li>Restarting server 2. Its state machine replays the Raft log via Ratis AppendEntries
53+
* (gap is a handful of entries, well below {@code snapshotThreshold}), and when
54+
* {@code BOOTSTRAP_FINGERPRINT_ENTRY} is applied the local {@code lastTxId=1_000_000} is
55+
* greater than the entry's {@code chosenLastTxId=7} - the SEVERE-and-return branch
56+
* fires.</li>
57+
* </ol>
58+
* The assertion checks that server 2's bootstrap baseline is the cluster's committed value (the
59+
* entry was applied; the baseline map was populated <b>above</b> the refusal check) rather than
60+
* server 2's higher pre-staged value, and that server 2's local on-disk {@code last-tx-id.bin}
61+
* was preserved (proving {@code installFromLeaderForBootstrap} did NOT run).
5162
*/
52-
@Disabled("see class javadoc - Ratis snapshot-install often races the AppendEntries replay path")
5363
class RaftBootstrapLateNewerJoinerIT extends BaseRaftHATest {
5464

5565
private static final long LATE_JOINER_PRE_STAGED_LAST_TX_ID = 1_000_000L;
56-
private static final long SHORT_BOOTSTRAP_TIMEOUT_MS = 3_000L;
66+
private static final int LATE_JOINER_INDEX = 2;
67+
68+
@Override
69+
protected boolean persistentRaftStorage() {
70+
// We want server 2's Raft storage WIPED on its restart so the leader re-ships entries via
71+
// AppendEntries rather than treating server 2 as a still-caught-up peer.
72+
return false;
73+
}
5774

5875
@Override
5976
protected void onServerConfiguration(final ContextConfiguration config) {
6077
super.onServerConfiguration(config);
6178
config.setValue(GlobalConfiguration.HA_BOOTSTRAP_FROM_LOCAL_DATABASE, true);
62-
config.setValue(GlobalConfiguration.HA_BOOTSTRAP_TIMEOUT_MS, SHORT_BOOTSTRAP_TIMEOUT_MS);
6379
}
6480

6581
@Override
6682
protected int getServerCount() {
6783
return 3;
6884
}
6985

70-
@Override
71-
protected void onBeforeStarting(final ArcadeDBServer server) {
72-
super.onBeforeStarting(server);
73-
final String name = server.getConfiguration().getValueAsString(GlobalConfiguration.SERVER_NAME);
74-
final int idx = Integer.parseInt(name.substring(name.lastIndexOf('_') + 1));
75-
if (idx != 2)
76-
return;
77-
final String dbDir = server.getConfiguration()
78-
.getValueAsString(GlobalConfiguration.SERVER_DATABASE_DIRECTORY)
79-
+ File.separator + getDatabaseName();
80-
final File lastTxIdFile = new File(dbDir, "last-tx-id.bin");
81-
try (final DataOutputStream out = new DataOutputStream(new FileOutputStream(lastTxIdFile))) {
82-
out.writeLong(LATE_JOINER_PRE_STAGED_LAST_TX_ID);
83-
} catch (final IOException e) {
84-
throw new RuntimeException("Failed to pre-stage last-tx-id.bin for late joiner", e);
85-
}
86-
}
87-
8886
@Test
89-
void lateNewerJoinerKeepsClusterBaselineNotItsOwnLocalLastTxId() {
87+
void lateNewerJoinerAdoptsClusterBaselineAndPreservesLocalLastTxId() throws IOException {
9088
final String dbName = getDatabaseName();
89+
90+
// Phase 1: wait for the initial bootstrap baseline to commit and apply on every peer.
9191
Awaitility.await()
9292
.atMost(60, TimeUnit.SECONDS)
9393
.pollInterval(250, TimeUnit.MILLISECONDS)
@@ -100,13 +100,83 @@ void lateNewerJoinerKeepsClusterBaselineNotItsOwnLocalLastTxId() {
100100
}
101101
});
102102

103-
// Late joiner (server 2) must hold the cluster's baseline (= servers 0/1's lower value), NOT
104-
// its own pre-staged 1_000_000. Proves applyBootstrapFingerprintEntry recorded the cluster
105-
// decision; whether the SEVERE branch specifically fired is verified by code review.
106-
final var lateJoinerBaseline = getRaftPlugin(2).getRaftHAServer()
103+
final var clusterBaseline = getRaftPlugin(0).getRaftHAServer()
104+
.getStateMachine().getBootstrapBaseline(dbName);
105+
final long clusterBaselineLastTxId = clusterBaseline.lastTxId();
106+
assertThat(clusterBaselineLastTxId)
107+
.as("test premise: cluster baseline must be lower than the late joiner's pre-staged value")
108+
.isLessThan(LATE_JOINER_PRE_STAGED_LAST_TX_ID);
109+
110+
// Capture server 2's database path BEFORE stop so we can perturb its last-tx-id.bin.
111+
final ServerDatabase preStopDb = (ServerDatabase) getServerDatabase(LATE_JOINER_INDEX, dbName);
112+
assertThat(preStopDb).isNotNull();
113+
final LocalDatabase preStopLocal = (LocalDatabase) preStopDb.getWrappedDatabaseInstance().getEmbedded();
114+
final File databasePath = new File(preStopLocal.getDatabasePath());
115+
final File lastTxIdFile = new File(databasePath, "last-tx-id.bin");
116+
117+
// Phase 2: stop server 2 cleanly so its TransactionManager flushes last-tx-id.bin, then
118+
// overwrite that file with the high value.
119+
LogManager.instance().log(this, Level.INFO, "TEST: stopping server %d for late-joiner setup", LATE_JOINER_INDEX);
120+
getServer(LATE_JOINER_INDEX).stop();
121+
// Let the OS release the gRPC port and the Raft storage handle.
122+
try {
123+
Thread.sleep(2_000);
124+
} catch (final InterruptedException ie) {
125+
Thread.currentThread().interrupt();
126+
return;
127+
}
128+
129+
// Wipe server 2's Raft storage so Ratis re-ships log entries on rejoin (we want the apply
130+
// path, not InstallSnapshot, to be the carrier of BOOTSTRAP_FINGERPRINT_ENTRY).
131+
final String rootPath = GlobalConfiguration.SERVER_ROOT_PATH.getValueAsString();
132+
final File raftStorage = new File(rootPath + File.separator + "raft-storage-" + peerIdForIndex(LATE_JOINER_INDEX));
133+
FileUtils.deleteRecursively(raftStorage);
134+
135+
try (final DataOutputStream out = new DataOutputStream(new FileOutputStream(lastTxIdFile))) {
136+
out.writeLong(LATE_JOINER_PRE_STAGED_LAST_TX_ID);
137+
}
138+
139+
// Phase 3: restart server 2. State machine replays Raft log via AppendEntries.
140+
LogManager.instance().log(this, Level.INFO,
141+
"TEST: restarting server %d with pre-staged last-tx-id.bin=%d", LATE_JOINER_INDEX,
142+
LATE_JOINER_PRE_STAGED_LAST_TX_ID);
143+
getServer(LATE_JOINER_INDEX).start();
144+
145+
// Phase 4: wait for server 2 to re-apply the BOOTSTRAP_FINGERPRINT_ENTRY. The baseline is
146+
// set inside applyBootstrapFingerprintEntry BEFORE the SEVERE-refusal check, so getBootstrapBaseline
147+
// returns non-null once the entry has been applied (even on the refusal branch).
148+
Awaitility.await()
149+
.atMost(60, TimeUnit.SECONDS)
150+
.pollInterval(500, TimeUnit.MILLISECONDS)
151+
.untilAsserted(() -> {
152+
final RaftHAPlugin plugin = getRaftPlugin(LATE_JOINER_INDEX);
153+
assertThat(plugin).isNotNull();
154+
assertThat(plugin.getRaftHAServer().getStateMachine().getBootstrapBaseline(dbName)).isNotNull();
155+
});
156+
157+
// Assert 1: late joiner adopted the cluster's baseline (NOT its own higher pre-staged value).
158+
final var lateJoinerBaseline = getRaftPlugin(LATE_JOINER_INDEX).getRaftHAServer()
107159
.getStateMachine().getBootstrapBaseline(dbName);
108160
assertThat(lateJoinerBaseline.lastTxId())
109-
.as("late joiner must adopt the cluster baseline, not its own higher pre-staged value")
161+
.as("late joiner's baseline must be the cluster's value, not its own pre-staged")
162+
.isEqualTo(clusterBaselineLastTxId);
163+
assertThat(lateJoinerBaseline.lastTxId())
164+
.as("late joiner's baseline must be strictly less than its pre-staged lastTxId")
110165
.isLessThan(LATE_JOINER_PRE_STAGED_LAST_TX_ID);
166+
167+
// Assert 2: the SEVERE-refusal branch did NOT call installFromLeaderForBootstrap, so server
168+
// 2's last-tx-id.bin still contains the pre-staged 1_000_000. If the snapshot reinstall had
169+
// run, the file would have been wiped (it's not in the snapshot ZIP) and the engine would
170+
// have written its current in-memory value (close to 7).
171+
assertThat(lastTxIdFile.isFile())
172+
.as("server 2's last-tx-id.bin must still exist (no snapshot reinstall happened)")
173+
.isTrue();
174+
final long onDiskLastTxId;
175+
try (final java.io.DataInputStream in = new java.io.DataInputStream(new java.io.FileInputStream(lastTxIdFile))) {
176+
onDiskLastTxId = in.readLong();
177+
}
178+
assertThat(onDiskLastTxId)
179+
.as("server 2's local last-tx-id.bin must be preserved at the high pre-staged value")
180+
.isEqualTo(LATE_JOINER_PRE_STAGED_LAST_TX_ID);
111181
}
112182
}

0 commit comments

Comments
 (0)