Skip to content

Commit e885b46

Browse files
authored
[refactor](ivm) Rename IvmInfo.refreshVersion to sequencePrefix (apache#68336)
`IvmInfo.refreshVersion` is the high part of the sequence values an IVM MV's rows are stamped with -- the low part next to it is a delta index. The name reads like an epoch or like the MV's own version, which is what the per-partition `refreshEpoch` being added alongside it is not, and the two sitting in the same code base is a trap for the next reader. This renames it to `sequencePrefix`: the prefix of the `(sequence prefix, delta index, op)` triple that `IvmSequenceCalculator` encodes into the sequence column. Nothing about the value changes -- it still counts committing IVM transactions and still prefixes the sequence column. - field and accessors: `sequencePrefix`, `getSequencePrefix()`, `advanceSequencePrefix()` - `MTMV.getNextRefreshVersion()` -> `MTMV.getNextSequencePrefix()` - `IvmSequenceCalculator` identifiers, including `LARGEINT_SEQUENCE_PREFIX_SHIFT` and the range-check messages - the persisted name changes outright from `"rv"` to `"sp"`: IVM is not released, so there is no image or journal in the wild that writes the old name Trace: apache#65418
1 parent 4b848da commit e885b46

12 files changed

Lines changed: 74 additions & 63 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -255,8 +255,8 @@ public boolean isIvm() {
255255
return getIvmInfo().isEnableIvm();
256256
}
257257

258-
public long getNextRefreshVersion() {
259-
return Config.isCloudMode() ? getNextVersion() : getIvmInfo().getRefreshVersion() + 1;
258+
public long getNextSequencePrefix() {
259+
return Config.isCloudMode() ? getNextVersion() : getIvmInfo().getSequencePrefix() + 1;
260260
}
261261

262262
public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) {

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ __DORIS_IVM_ROW_ID_COL__ | k1 | cnt | sum_v1 | __DORIS_IVM_DML_FACTOR_COL__ | __
155155

156156
### Semantics
157157

158-
- **Read-only.** No insert transaction is built. Stream offsets, refresh version, and MV metadata
158+
- **Read-only.** No insert transaction is built. Stream offsets, sequence prefix, and MV metadata
159159
are never modified.
160160
- **Idempotent.** Repeating the same dry run returns identical rows as long as the base table data
161161
has not changed.

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteState.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,11 @@ class IvmDeltaRewriteState {
5757
private int nextDeltaScanIndex;
5858

5959
IvmDeltaRewriteState(Map<OlapTable, OlapTableStream> streams,
60-
boolean includeExhaustedStreams, long refreshVersion, DataType sequenceType,
60+
boolean includeExhaustedStreams, long sequencePrefix, DataType sequenceType,
6161
Map<OlapTable, List<Long>> windowPartitionIdsByTable) {
6262
this.streams = new HashMap<>(streams);
6363
this.includeExhaustedStreams = includeExhaustedStreams;
64-
this.sequenceCalculator = IvmSequenceCalculator.create(refreshVersion, sequenceType);
64+
this.sequenceCalculator = IvmSequenceCalculator.create(sequencePrefix, sequenceType);
6565
this.windowPartitionIdsByTable = windowPartitionIdsByTable;
6666
}
6767

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriter.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ public Plan generateIncrRefreshPlan(Plan sinkChild, IvmRewriteResult rewriteResu
6767
rewriteContext.isIncludeExhaustedStreams());
6868
Pair<Plan, List<LogicalProject<?>>> prefixChain = helper.detachAdaptProjectChain(sinkChild);
6969
Plan rootPlan = prefixChain.first;
70-
long refreshVersion = refreshContext.getMtmv().getNextRefreshVersion();
71-
IvmDeltaRewriteState rewriteState = createDeltaRewriteState(rootPlan, refreshContext, refreshVersion,
70+
long sequencePrefix = refreshContext.getMtmv().getNextSequencePrefix();
71+
IvmDeltaRewriteState rewriteState = createDeltaRewriteState(rootPlan, refreshContext, sequencePrefix,
7272
rewriteContext.getIncrementalScopePartitionIds());
7373
Optional<IvmDeltaRewriteResult> deltaResult = rewriteDelta(rootPlan, refreshContext, rewriteState);
7474
if (!deltaResult.isPresent()) {
@@ -156,7 +156,7 @@ private static Pair<Plan, Map<Slot, Slot>> rewriteSnapshot(Plan plan, IvmDeltaRe
156156
return IvmDeltaRewriteHelper.INSTANCE.freshPlan(rewritten);
157157
}
158158

159-
private IvmDeltaRewriteState createDeltaRewriteState(Plan plan, IvmIncrRefreshContext ctx, long refreshVersion,
159+
private IvmDeltaRewriteState createDeltaRewriteState(Plan plan, IvmIncrRefreshContext ctx, long sequencePrefix,
160160
Map<BaseTableInfo, Set<Long>> scopePartitionIds) {
161161
Map<OlapTable, OlapTableStream> streams = new HashMap<>();
162162
// Window limits apply to every base table in the plan, including excluded
@@ -186,7 +186,7 @@ private IvmDeltaRewriteState createDeltaRewriteState(Plan plan, IvmIncrRefreshCo
186186
}
187187
}
188188
applyScopePartitionIds(windowPartitionIdsByTable, planTables, scopePartitionIds);
189-
return new IvmDeltaRewriteState(streams, ctx.isIncludeExhaustedStreams(), refreshVersion,
189+
return new IvmDeltaRewriteState(streams, ctx.isIncludeExhaustedStreams(), sequencePrefix,
190190
DataType.fromCatalogType(ctx.getMtmv().getColumn(Column.SEQUENCE_COL).getType()),
191191
windowPartitionIdsByTable);
192192
}

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,9 @@ public class IvmInfo {
5353
@SerializedName("ps")
5454
private String planSignature;
5555

56-
@SerializedName("rv")
57-
private long refreshVersion;
56+
/** The prefix of the sequence values this MV's rows are stamped with; see IvmSequenceCalculator. */
57+
@SerializedName("sp")
58+
private long sequencePrefix;
5859

5960
public IvmInfo() {
6061
}
@@ -65,7 +66,7 @@ public IvmInfo(IvmInfo other) {
6566
this.pendingBaselineRebuildPartitions = new HashSet<>(other.pendingBaselineRebuildPartitions);
6667
this.useFullKeys = other.useFullKeys;
6768
this.planSignature = other.planSignature;
68-
this.refreshVersion = other.refreshVersion;
69+
this.sequencePrefix = other.sequencePrefix;
6970
}
7071

7172
public boolean isEnableIvm() {
@@ -121,12 +122,12 @@ public void setPlanSignature(String planSignature) {
121122
this.planSignature = planSignature;
122123
}
123124

124-
public long getRefreshVersion() {
125-
return refreshVersion;
125+
public long getSequencePrefix() {
126+
return sequencePrefix;
126127
}
127128

128-
public void advanceRefreshVersion() {
129-
refreshVersion++;
129+
public void advanceSequencePrefix() {
130+
sequencePrefix++;
130131
}
131132

132133
@Override

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmSequenceCalculator.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -33,32 +33,32 @@
3333
/**
3434
* Encodes IVM delta ordering into the MTMV sequence column.
3535
*
36-
* <p>BIGINT encodes {@code (refresh version, delta index, op)}. LARGEINT additionally encodes
37-
* a 64-bit binlog sequence: {@code (refresh version, delta index, binlog sequence, op)}.
36+
* <p>BIGINT encodes {@code (sequence prefix, delta index, op)}. LARGEINT additionally encodes
37+
* a 64-bit binlog sequence: {@code (sequence prefix, delta index, binlog sequence, op)}.
3838
*/
3939
abstract class IvmSequenceCalculator {
4040
static final int DELTA_INDEX_BITS = 10;
4141
private static final int BIGINT_LOW_BITS = DELTA_INDEX_BITS + 1;
4242
private static final int LARGEINT_BINLOG_BITS = 64;
4343
private static final int LARGEINT_DELTA_INDEX_SHIFT = LARGEINT_BINLOG_BITS + 1;
44-
private static final int LARGEINT_REFRESH_VERSION_SHIFT =
44+
private static final int LARGEINT_SEQUENCE_PREFIX_SHIFT =
4545
LARGEINT_DELTA_INDEX_SHIFT + DELTA_INDEX_BITS;
4646
private static final int MAX_DELTA_INDEX = 1 << DELTA_INDEX_BITS;
4747
private static final BigInteger MAX_BINLOG_SEQUENCE =
4848
BigInteger.ONE.shiftLeft(LARGEINT_BINLOG_BITS).subtract(BigInteger.ONE);
4949

50-
final long refreshVersion;
50+
final long sequencePrefix;
5151

52-
private IvmSequenceCalculator(long refreshVersion) {
53-
this.refreshVersion = refreshVersion;
52+
private IvmSequenceCalculator(long sequencePrefix) {
53+
this.sequencePrefix = sequencePrefix;
5454
}
5555

56-
static IvmSequenceCalculator create(long refreshVersion, DataType sequenceType) {
56+
static IvmSequenceCalculator create(long sequencePrefix, DataType sequenceType) {
5757
if (sequenceType.equals(BigIntType.INSTANCE)) {
58-
return new BigIntSequenceCalculator(refreshVersion);
58+
return new BigIntSequenceCalculator(sequencePrefix);
5959
}
6060
if (sequenceType.equals(LargeIntType.INSTANCE)) {
61-
return new LargeIntSequenceCalculator(refreshVersion);
61+
return new LargeIntSequenceCalculator(sequencePrefix);
6262
}
6363
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
6464
"unsupported IVM sequence type: " + sequenceType.simpleString());
@@ -87,11 +87,11 @@ final void checkBinlogSequence(BigInteger binlogSequence) {
8787
}
8888

8989
private static class BigIntSequenceCalculator extends IvmSequenceCalculator {
90-
private BigIntSequenceCalculator(long refreshVersion) {
91-
super(refreshVersion);
92-
if (refreshVersion < 0 || refreshVersion > (Long.MAX_VALUE >>> BIGINT_LOW_BITS)) {
90+
private BigIntSequenceCalculator(long sequencePrefix) {
91+
super(sequencePrefix);
92+
if (sequencePrefix < 0 || sequencePrefix > (Long.MAX_VALUE >>> BIGINT_LOW_BITS)) {
9393
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
94-
"IVM refresh version exceeds the BIGINT sequence encoding range: " + refreshVersion);
94+
"IVM sequence prefix exceeds the BIGINT sequence encoding range: " + sequencePrefix);
9595
}
9696
}
9797

@@ -102,26 +102,26 @@ Literal encode(int deltaIndex, BigInteger binlogSequence, boolean positive) {
102102
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
103103
"BIGINT IVM sequence does not support binlog sequence");
104104
}
105-
long sequence = (refreshVersion << BIGINT_LOW_BITS)
105+
long sequence = (sequencePrefix << BIGINT_LOW_BITS)
106106
| ((long) deltaIndex << 1) | (positive ? 1 : 0);
107107
return new BigIntLiteral(sequence);
108108
}
109109
}
110110

111111
private static class LargeIntSequenceCalculator extends IvmSequenceCalculator {
112-
private LargeIntSequenceCalculator(long refreshVersion) {
113-
super(refreshVersion);
114-
if (refreshVersion < 0 || refreshVersion > (Long.MAX_VALUE >>> BIGINT_LOW_BITS)) {
112+
private LargeIntSequenceCalculator(long sequencePrefix) {
113+
super(sequencePrefix);
114+
if (sequencePrefix < 0 || sequencePrefix > (Long.MAX_VALUE >>> BIGINT_LOW_BITS)) {
115115
throw new IvmException(IvmFailureReason.PLAN_REWRITE_FAILED,
116-
"IVM refresh version exceeds the LARGEINT sequence encoding range: " + refreshVersion);
116+
"IVM sequence prefix exceeds the LARGEINT sequence encoding range: " + sequencePrefix);
117117
}
118118
}
119119

120120
@Override
121121
Literal encode(int deltaIndex, BigInteger binlogSequence, boolean positive) {
122122
checkDeltaIndex(deltaIndex);
123123
checkBinlogSequence(binlogSequence);
124-
BigInteger sequence = BigInteger.valueOf(refreshVersion).shiftLeft(LARGEINT_REFRESH_VERSION_SHIFT)
124+
BigInteger sequence = BigInteger.valueOf(sequencePrefix).shiftLeft(LARGEINT_SEQUENCE_PREFIX_SHIFT)
125125
.or(BigInteger.valueOf(deltaIndex).shiftLeft(LARGEINT_DELTA_INDEX_SHIFT))
126126
.or(binlogSequence.shiftLeft(1));
127127
if (positive) {

‎fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2418,15 +2418,15 @@ private void updateCatalogAfterCommitted(TransactionState transactionState, Data
24182418
// update table stream offset if necessary
24192419
if (!CollectionUtils.isEmpty(transactionState.getStreamUpdateInfos())) {
24202420
updateStreamOffset(transactionState, transactionState.getCommitTime());
2421-
updateIvmRefreshVersion(transactionState, db);
2421+
updateIvmSequencePrefix(transactionState, db);
24222422
}
24232423
}
24242424

2425-
private void updateIvmRefreshVersion(TransactionState transactionState, Database db) {
2425+
private void updateIvmSequencePrefix(TransactionState transactionState, Database db) {
24262426
for (Long tableId : transactionState.getTableIdList()) {
24272427
Table table = db.getTableNullable(tableId);
24282428
if (table instanceof MTMV && ((MTMV) table).isIvm()) {
2429-
((MTMV) table).getIvmInfo().advanceRefreshVersion();
2429+
((MTMV) table).getIvmInfo().advanceSequencePrefix();
24302430
}
24312431
}
24322432
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmAggDeltaHandlerTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ class IvmAggDeltaHandlerTest extends IvmDeltaTestBase {
7070
private AggRewriteResult rewriteAgg(LogicalAggregate<? extends Plan> agg) {
7171
PlanBundle bundle = normalizeAggPlan(agg);
7272
MTMV mtmv = buildMtmvFromPlan(bundle.normalizedPlan.getOutput());
73-
mtmv.getIvmInfo().advanceRefreshVersion();
74-
mtmv.getIvmInfo().advanceRefreshVersion();
73+
mtmv.getIvmInfo().advanceSequencePrefix();
74+
mtmv.getIvmInfo().advanceSequencePrefix();
7575
Plan rewritten = new IvmDeltaRewriter().generateIncrRefreshPlan(
7676
bundle.normalizedPlan, bundle.rewriteResult,
7777
IvmRewriteContext.incremental(mtmv), bundle.connectContext);
@@ -89,8 +89,8 @@ private AggRewriteResult rewriteAggWithIdentityKeys(LogicalAggregate<? extends P
8989
PlanBundle bundle = normalizeAggPlan(agg);
9090
bundle.rewriteResult.setIdentityKeySlots(identityKeys);
9191
MTMV mtmv = buildMtmvFromPlan(bundle.normalizedPlan.getOutput());
92-
mtmv.getIvmInfo().advanceRefreshVersion();
93-
mtmv.getIvmInfo().advanceRefreshVersion();
92+
mtmv.getIvmInfo().advanceSequencePrefix();
93+
mtmv.getIvmInfo().advanceSequencePrefix();
9494
Plan rewritten = new IvmDeltaRewriter().generateIncrRefreshPlan(
9595
bundle.normalizedPlan, bundle.rewriteResult,
9696
IvmRewriteContext.incremental(mtmv), bundle.connectContext);

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteStateTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
class IvmDeltaRewriteStateTest extends IvmDeltaTestBase {
4444

4545
@Test
46-
void testSequenceEncodesRefreshVersionAndDeltaIndex() {
46+
void testSequenceEncodesSequencePrefixAndDeltaIndex() {
4747
IvmDeltaRewriteState state = new IvmDeltaRewriteState(
4848
ImmutableMap.of(), false, 7L, BigIntType.INSTANCE, ImmutableMap.of());
4949

@@ -172,7 +172,7 @@ private void addTablet(OlapTable table, int partitionIndex, long tabletId) {
172172
}
173173

174174
@Test
175-
void testLargeIntSequenceEncodesRefreshVersionAndDeltaIndex() {
175+
void testLargeIntSequenceEncodesSequencePrefixAndDeltaIndex() {
176176
IvmDeltaRewriteState state = new IvmDeltaRewriteState(
177177
ImmutableMap.of(), false, 7L, LargeIntType.INSTANCE, ImmutableMap.of());
178178

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmInfoTest.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,31 @@
2626

2727
class IvmInfoTest {
2828
@Test
29-
void testRefreshVersionAdvancesAfterCommittedRefresh() {
29+
void testSequencePrefixAdvancesAfterCommittedRefresh() {
3030
IvmInfo info = new IvmInfo();
3131

32-
Assertions.assertEquals(0, info.getRefreshVersion());
33-
info.advanceRefreshVersion();
34-
Assertions.assertEquals(1, info.getRefreshVersion());
32+
Assertions.assertEquals(0, info.getSequencePrefix());
33+
info.advanceSequencePrefix();
34+
Assertions.assertEquals(1, info.getSequencePrefix());
3535
}
3636

3737
@Test
38-
void testRefreshVersionPersistsThroughGson() {
38+
void testSequencePrefixPersistsThroughGson() {
3939
IvmInfo info = new IvmInfo();
40-
info.advanceRefreshVersion();
40+
info.advanceSequencePrefix();
4141

4242
IvmInfo recovered = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(info), IvmInfo.class);
43-
Assertions.assertEquals(1, recovered.getRefreshVersion());
43+
Assertions.assertEquals(1, recovered.getSequencePrefix());
44+
}
45+
46+
@Test
47+
void testSequencePrefixIsPersistedAsSp() {
48+
IvmInfo info = new IvmInfo();
49+
info.advanceSequencePrefix();
50+
51+
String json = GsonUtils.GSON.toJson(info);
52+
Assertions.assertTrue(json.contains("\"sp\":1"), json);
53+
Assertions.assertFalse(json.contains("\"rv\""), json);
4454
}
4555

4656
@Test
@@ -59,7 +69,7 @@ void testCopyConstructor() {
5969
info.requireCompleteBaselineRebuild();
6070
info.setUseFullKeys(true);
6171
info.setPlanSignature("abc123");
62-
info.advanceRefreshVersion();
72+
info.advanceSequencePrefix();
6373

6474
IvmInfo copy = new IvmInfo(info);
6575
info.clearBaselineRebuild();
@@ -68,7 +78,7 @@ void testCopyConstructor() {
6878
Assertions.assertTrue(copy.isBaselineRebuildRequired());
6979
Assertions.assertTrue(copy.isUseFullKeys());
7080
Assertions.assertEquals("abc123", copy.getPlanSignature());
71-
Assertions.assertEquals(1, copy.getRefreshVersion());
81+
Assertions.assertEquals(1, copy.getSequencePrefix());
7282
}
7383

7484
@Test

0 commit comments

Comments
 (0)