2020
2121import com .arcadedb .ContextConfiguration ;
2222import com .arcadedb .GlobalConfiguration ;
23- import com .arcadedb .server .ArcadeDBServer ;
23+ import com .arcadedb .database .LocalDatabase ;
24+ import com .arcadedb .log .LogManager ;
25+ import com .arcadedb .server .ServerDatabase ;
26+ import com .arcadedb .utility .FileUtils ;
2427import org .awaitility .Awaitility ;
25- import org .junit .jupiter .api .Disabled ;
2628import org .junit .jupiter .api .Test ;
2729
2830import java .io .DataOutputStream ;
2931import java .io .File ;
3032import java .io .FileOutputStream ;
3133import java .io .IOException ;
3234import java .util .concurrent .TimeUnit ;
35+ import java .util .logging .Level ;
3336
3437import static org .assertj .core .api .Assertions .assertThat ;
3538
3639/**
37- * Regression IT for issue #4147 late-newer-joiner refusal: a peer whose pre-staged
38- * {@code lastTxId} is strictly greater than the bootstrap baseline already committed by the rest
39- * of the cluster must NOT silently overwrite its own local data. The check in
40- * {@code ArcadeStateMachine.applyBootstrapFingerprintEntry} emits a SEVERE and returns without
41- * calling {@code installFromLeaderForBootstrap}, leaving the operator's fresher data in place.
40+ * Regression IT for issue #4147 late-newer-joiner refusal: a peer whose local {@code lastTxId} is
41+ * strictly greater than the cluster's committed bootstrap baseline must NOT silently overwrite
42+ * its own data. The guard in {@code ArcadeStateMachine.applyBootstrapFingerprintEntry} emits a
43+ * SEVERE and returns without calling {@code installFromLeaderForBootstrap}.
4244 * <p>
43- * <b>Currently @Disabled.</b> The fixture below requires the joiner to apply the already-committed
44- * {@code BOOTSTRAP_FINGERPRINT_ENTRY} via Ratis AppendEntries replay (so the SEVERE branch can
45- * fire); empirically, Ratis often takes the snapshot-install path on the late joiner first, which
46- * overwrites the peer's pre-staged state before the apply check runs. The refusal-and-SEVERE
47- * branch is a 4-line guard in {@code applyBootstrapFingerprintEntry} that is straightforward to
48- * verify by code review; the missing piece for an IT is a way to force Ratis to send AppendEntries
49- * instead of an InstallSnapshot on the late join. Re-enable once that fixture knob exists (or
50- * after rewriting this as a focused state-machine unit test).
45+ * The fixture engineers the scenario by:
46+ * <ol>
47+ * <li>Letting the cluster form and bootstrap normally (canonical low {@code lastTxId} on every
48+ * peer; baseline = e.g. 7).</li>
49+ * <li>Stopping server 2, wiping its Raft storage so Ratis treats it as a fresh peer on rejoin,
50+ * and overwriting its {@code last-tx-id.bin} with a much higher value
51+ * (1_000_000).</li>
52+ * <li>Restarting server 2. Its state machine replays the Raft log via Ratis AppendEntries
53+ * (gap is a handful of entries, well below {@code snapshotThreshold}), and when
54+ * {@code BOOTSTRAP_FINGERPRINT_ENTRY} is applied the local {@code lastTxId=1_000_000} is
55+ * greater than the entry's {@code chosenLastTxId=7} - the SEVERE-and-return branch
56+ * fires.</li>
57+ * </ol>
58+ * The assertion checks that server 2's bootstrap baseline is the cluster's committed value (the
59+ * entry was applied; the baseline map was populated <b>above</b> the refusal check) rather than
60+ * server 2's higher pre-staged value, and that server 2's local on-disk {@code last-tx-id.bin}
61+ * was preserved (proving {@code installFromLeaderForBootstrap} did NOT run).
5162 */
52- @ Disabled ("see class javadoc - Ratis snapshot-install often races the AppendEntries replay path" )
5363class RaftBootstrapLateNewerJoinerIT extends BaseRaftHATest {
5464
5565 private static final long LATE_JOINER_PRE_STAGED_LAST_TX_ID = 1_000_000L ;
56- private static final long SHORT_BOOTSTRAP_TIMEOUT_MS = 3_000L ;
66+ private static final int LATE_JOINER_INDEX = 2 ;
67+
68+ @ Override
69+ protected boolean persistentRaftStorage () {
70+ // We want server 2's Raft storage WIPED on its restart so the leader re-ships entries via
71+ // AppendEntries rather than treating server 2 as a still-caught-up peer.
72+ return false ;
73+ }
5774
5875 @ Override
5976 protected void onServerConfiguration (final ContextConfiguration config ) {
6077 super .onServerConfiguration (config );
6178 config .setValue (GlobalConfiguration .HA_BOOTSTRAP_FROM_LOCAL_DATABASE , true );
62- config .setValue (GlobalConfiguration .HA_BOOTSTRAP_TIMEOUT_MS , SHORT_BOOTSTRAP_TIMEOUT_MS );
6379 }
6480
6581 @ Override
6682 protected int getServerCount () {
6783 return 3 ;
6884 }
6985
70- @ Override
71- protected void onBeforeStarting (final ArcadeDBServer server ) {
72- super .onBeforeStarting (server );
73- final String name = server .getConfiguration ().getValueAsString (GlobalConfiguration .SERVER_NAME );
74- final int idx = Integer .parseInt (name .substring (name .lastIndexOf ('_' ) + 1 ));
75- if (idx != 2 )
76- return ;
77- final String dbDir = server .getConfiguration ()
78- .getValueAsString (GlobalConfiguration .SERVER_DATABASE_DIRECTORY )
79- + File .separator + getDatabaseName ();
80- final File lastTxIdFile = new File (dbDir , "last-tx-id.bin" );
81- try (final DataOutputStream out = new DataOutputStream (new FileOutputStream (lastTxIdFile ))) {
82- out .writeLong (LATE_JOINER_PRE_STAGED_LAST_TX_ID );
83- } catch (final IOException e ) {
84- throw new RuntimeException ("Failed to pre-stage last-tx-id.bin for late joiner" , e );
85- }
86- }
87-
8886 @ Test
89- void lateNewerJoinerKeepsClusterBaselineNotItsOwnLocalLastTxId () {
87+ void lateNewerJoinerAdoptsClusterBaselineAndPreservesLocalLastTxId () throws IOException {
9088 final String dbName = getDatabaseName ();
89+
90+ // Phase 1: wait for the initial bootstrap baseline to commit and apply on every peer.
9191 Awaitility .await ()
9292 .atMost (60 , TimeUnit .SECONDS )
9393 .pollInterval (250 , TimeUnit .MILLISECONDS )
@@ -100,13 +100,83 @@ void lateNewerJoinerKeepsClusterBaselineNotItsOwnLocalLastTxId() {
100100 }
101101 });
102102
103- // Late joiner (server 2) must hold the cluster's baseline (= servers 0/1's lower value), NOT
104- // its own pre-staged 1_000_000. Proves applyBootstrapFingerprintEntry recorded the cluster
105- // decision; whether the SEVERE branch specifically fired is verified by code review.
106- final var lateJoinerBaseline = getRaftPlugin (2 ).getRaftHAServer ()
103+ final var clusterBaseline = getRaftPlugin (0 ).getRaftHAServer ()
104+ .getStateMachine ().getBootstrapBaseline (dbName );
105+ final long clusterBaselineLastTxId = clusterBaseline .lastTxId ();
106+ assertThat (clusterBaselineLastTxId )
107+ .as ("test premise: cluster baseline must be lower than the late joiner's pre-staged value" )
108+ .isLessThan (LATE_JOINER_PRE_STAGED_LAST_TX_ID );
109+
110+ // Capture server 2's database path BEFORE stop so we can perturb its last-tx-id.bin.
111+ final ServerDatabase preStopDb = (ServerDatabase ) getServerDatabase (LATE_JOINER_INDEX , dbName );
112+ assertThat (preStopDb ).isNotNull ();
113+ final LocalDatabase preStopLocal = (LocalDatabase ) preStopDb .getWrappedDatabaseInstance ().getEmbedded ();
114+ final File databasePath = new File (preStopLocal .getDatabasePath ());
115+ final File lastTxIdFile = new File (databasePath , "last-tx-id.bin" );
116+
117+ // Phase 2: stop server 2 cleanly so its TransactionManager flushes last-tx-id.bin, then
118+ // overwrite that file with the high value.
119+ LogManager .instance ().log (this , Level .INFO , "TEST: stopping server %d for late-joiner setup" , LATE_JOINER_INDEX );
120+ getServer (LATE_JOINER_INDEX ).stop ();
121+ // Let the OS release the gRPC port and the Raft storage handle.
122+ try {
123+ Thread .sleep (2_000 );
124+ } catch (final InterruptedException ie ) {
125+ Thread .currentThread ().interrupt ();
126+ return ;
127+ }
128+
129+ // Wipe server 2's Raft storage so Ratis re-ships log entries on rejoin (we want the apply
130+ // path, not InstallSnapshot, to be the carrier of BOOTSTRAP_FINGERPRINT_ENTRY).
131+ final String rootPath = GlobalConfiguration .SERVER_ROOT_PATH .getValueAsString ();
132+ final File raftStorage = new File (rootPath + File .separator + "raft-storage-" + peerIdForIndex (LATE_JOINER_INDEX ));
133+ FileUtils .deleteRecursively (raftStorage );
134+
135+ try (final DataOutputStream out = new DataOutputStream (new FileOutputStream (lastTxIdFile ))) {
136+ out .writeLong (LATE_JOINER_PRE_STAGED_LAST_TX_ID );
137+ }
138+
139+ // Phase 3: restart server 2. State machine replays Raft log via AppendEntries.
140+ LogManager .instance ().log (this , Level .INFO ,
141+ "TEST: restarting server %d with pre-staged last-tx-id.bin=%d" , LATE_JOINER_INDEX ,
142+ LATE_JOINER_PRE_STAGED_LAST_TX_ID );
143+ getServer (LATE_JOINER_INDEX ).start ();
144+
145+ // Phase 4: wait for server 2 to re-apply the BOOTSTRAP_FINGERPRINT_ENTRY. The baseline is
146+ // set inside applyBootstrapFingerprintEntry BEFORE the SEVERE-refusal check, so getBootstrapBaseline
147+ // returns non-null once the entry has been applied (even on the refusal branch).
148+ Awaitility .await ()
149+ .atMost (60 , TimeUnit .SECONDS )
150+ .pollInterval (500 , TimeUnit .MILLISECONDS )
151+ .untilAsserted (() -> {
152+ final RaftHAPlugin plugin = getRaftPlugin (LATE_JOINER_INDEX );
153+ assertThat (plugin ).isNotNull ();
154+ assertThat (plugin .getRaftHAServer ().getStateMachine ().getBootstrapBaseline (dbName )).isNotNull ();
155+ });
156+
157+ // Assert 1: late joiner adopted the cluster's baseline (NOT its own higher pre-staged value).
158+ final var lateJoinerBaseline = getRaftPlugin (LATE_JOINER_INDEX ).getRaftHAServer ()
107159 .getStateMachine ().getBootstrapBaseline (dbName );
108160 assertThat (lateJoinerBaseline .lastTxId ())
109- .as ("late joiner must adopt the cluster baseline, not its own higher pre-staged value" )
161+ .as ("late joiner's baseline must be the cluster's value, not its own pre-staged" )
162+ .isEqualTo (clusterBaselineLastTxId );
163+ assertThat (lateJoinerBaseline .lastTxId ())
164+ .as ("late joiner's baseline must be strictly less than its pre-staged lastTxId" )
110165 .isLessThan (LATE_JOINER_PRE_STAGED_LAST_TX_ID );
166+
167+ // Assert 2: the SEVERE-refusal branch did NOT call installFromLeaderForBootstrap, so server
168+ // 2's last-tx-id.bin still contains the pre-staged 1_000_000. If the snapshot reinstall had
169+ // run, the file would have been wiped (it's not in the snapshot ZIP) and the engine would
170+ // have written its current in-memory value (close to 7).
171+ assertThat (lastTxIdFile .isFile ())
172+ .as ("server 2's last-tx-id.bin must still exist (no snapshot reinstall happened)" )
173+ .isTrue ();
174+ final long onDiskLastTxId ;
175+ try (final java .io .DataInputStream in = new java .io .DataInputStream (new java .io .FileInputStream (lastTxIdFile ))) {
176+ onDiskLastTxId = in .readLong ();
177+ }
178+ assertThat (onDiskLastTxId )
179+ .as ("server 2's local last-tx-id.bin must be preserved at the high pre-staged value" )
180+ .isEqualTo (LATE_JOINER_PRE_STAGED_LAST_TX_ID );
111181 }
112182}
0 commit comments