|
32 | 32 | import org.apache.doris.nereids.exceptions.AnalysisException; |
33 | 33 | import org.apache.doris.persist.AlterMTMV; |
34 | 34 | import org.apache.doris.persist.ReplaceTableOperationLog; |
| 35 | +import org.apache.doris.persist.gson.GsonUtils; |
35 | 36 | import org.apache.doris.utframe.TestWithFeService; |
36 | 37 |
|
| 38 | +import com.google.gson.JsonObject; |
| 39 | +import com.google.gson.JsonParser; |
37 | 40 | import org.junit.jupiter.api.Assertions; |
38 | 41 | import org.junit.jupiter.api.Test; |
39 | 42 |
|
| 43 | +import java.io.ByteArrayInputStream; |
| 44 | +import java.io.ByteArrayOutputStream; |
| 45 | +import java.io.DataInputStream; |
| 46 | +import java.io.DataOutputStream; |
| 47 | +import java.io.IOException; |
| 48 | +import java.nio.charset.StandardCharsets; |
| 49 | +import java.util.LinkedHashMap; |
| 50 | +import java.util.Map; |
40 | 51 | import java.util.Set; |
41 | 52 |
|
42 | 53 |
|
@@ -403,6 +414,84 @@ public void testAlterIvmInfoPersistence() throws Exception { |
403 | 414 | Assertions.assertEquals(schemaChangeVersion, mtmv.getSchemaChangeVersion()); |
404 | 415 | } |
405 | 416 |
|
| 417 | + @Test |
| 418 | + public void testReplayAlterPartitionStates() throws Exception { |
| 419 | + Config.enable_table_stream = true; |
| 420 | + createDatabaseAndUse("alter_partition_states_test"); |
| 421 | + createTable("CREATE TABLE alter_partition_states_test.states_base (k1 int, v1 int)\n" |
| 422 | + + "DUPLICATE KEY(k1)\n" |
| 423 | + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" |
| 424 | + + "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')"); |
| 425 | + createMvByNereids("CREATE MATERIALIZED VIEW states_mv\n" |
| 426 | + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" |
| 427 | + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" |
| 428 | + + " PROPERTIES ('replication_num' = '1')\n" |
| 429 | + + " AS SELECT k1, v1 FROM states_base"); |
| 430 | + |
| 431 | + MTMV mtmv = (MTMV) Env.getCurrentInternalCatalog() |
| 432 | + .getDb("alter_partition_states_test").get() |
| 433 | + .getTableOrMetaException("states_mv"); |
| 434 | + String partitionName = mtmv.getPartitionNames().iterator().next(); |
| 435 | + TableNameInfo tableName = new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName()); |
| 436 | + |
| 437 | + // A payload carrying state applies it. The live map keeps moving after the payload was taken, |
| 438 | + // and for a restart only the bytes in the journal matter, so both are driven here. |
| 439 | + MTMVPartitionState state = new MTMVPartitionState(0, 1); |
| 440 | + Map<String, MTMVPartitionState> states = new LinkedHashMap<>(); |
| 441 | + states.put(partitionName, state); |
| 442 | + AlterMTMV withState = new AlterMTMV(tableName, MTMVAlterOpType.ALTER_PARTITION_STATES); |
| 443 | + withState.setPartitionStates(states); |
| 444 | + state.setLatestEpoch(7); |
| 445 | + // The MV starts without any state, so only the replayed payload can put it there. |
| 446 | + mtmv.alterPartitionStates(Map.of()); |
| 447 | + |
| 448 | + replayFromJournal(withState); |
| 449 | + |
| 450 | + Map<String, MTMVPartitionState> applied = mtmv.getPartitionStates(); |
| 451 | + Assertions.assertEquals(Set.of(partitionName), applied.keySet()); |
| 452 | + Assertions.assertEquals(0, applied.get(partitionName).getRefreshEpoch()); |
| 453 | + Assertions.assertEquals(1, applied.get(partitionName).getLatestEpoch()); |
| 454 | + |
| 455 | + // An explicit empty map empties the states. |
| 456 | + AlterMTMV empty = new AlterMTMV(tableName, MTMVAlterOpType.ALTER_PARTITION_STATES); |
| 457 | + empty.setPartitionStates(Map.of()); |
| 458 | + Assertions.assertTrue(new String(journalBytes(empty), StandardCharsets.UTF_8).contains("\"pst\""), |
| 459 | + "the state member should be written under its serialized name"); |
| 460 | + |
| 461 | + replayFromJournal(empty); |
| 462 | + |
| 463 | + Assertions.assertTrue(mtmv.getPartitionStates().isEmpty()); |
| 464 | + |
| 465 | + // A payload written before the member existed leaves the states alone instead of clearing them. |
| 466 | + mtmv.alterPartitionStates(Map.of(partitionName, new MTMVPartitionState(4, 6))); |
| 467 | + JsonObject legacy = JsonParser.parseString(GsonUtils.GSON.toJson(withState)).getAsJsonObject(); |
| 468 | + Assertions.assertNotNull(legacy.remove("pst")); |
| 469 | + |
| 470 | + replayFromJournal(GsonUtils.GSON.fromJson(legacy.toString(), AlterMTMV.class)); |
| 471 | + |
| 472 | + MTMVPartitionState kept = mtmv.getPartitionStates().get(partitionName); |
| 473 | + Assertions.assertEquals(4, kept.getRefreshEpoch()); |
| 474 | + Assertions.assertEquals(6, kept.getLatestEpoch()); |
| 475 | + } |
| 476 | + |
| 477 | + /** The bytes the edit log writes for an alter record. */ |
| 478 | + private static byte[] journalBytes(AlterMTMV alter) throws IOException { |
| 479 | + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); |
| 480 | + try (DataOutputStream out = new DataOutputStream(bytes)) { |
| 481 | + alter.write(out); |
| 482 | + } |
| 483 | + return bytes.toByteArray(); |
| 484 | + } |
| 485 | + |
| 486 | + /** Replays an alter record the way a restart does: from what the journal wrote, not from memory. */ |
| 487 | + private static void replayFromJournal(AlterMTMV alter) throws Exception { |
| 488 | + AlterMTMV replayed; |
| 489 | + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(journalBytes(alter)))) { |
| 490 | + replayed = AlterMTMV.read(in); |
| 491 | + } |
| 492 | + Env.getCurrentEnv().getAlterInstance().processAlterMTMV(replayed, true); |
| 493 | + } |
| 494 | + |
406 | 495 | @Test |
407 | 496 | public void testCreateIncrementalMtmvAutoCreatesStream() throws Exception { |
408 | 497 | createDatabaseAndUse("stream_test"); |
|
0 commit comments