Skip to content

Commit 9fb10c4

Browse files
committed
[fix](fe) Update partition changelog row counts on publish
### What problem does this PR solve? Issue Number: None Related PR: #69850, #69896 Problem Summary: Stream P/Q snapshots read partition index row counts, but changelog load deltas were stored in a separate lifetime counter in table statistics. That counter was not read by the partition lookup, lost its baseline after DROP STATS, and retained removed rows after TTL or partial TRUNCATE. Apply published changelog deltas directly to the partition index row-count estimate. Periodic tablet statistics replace the estimate, and partition replacement naturally discards it. Normalize each changelog tablet's rows using its partition replica allocation and ignore late reports for removed tablets or partitions so changelog rows cannot enter the base counter. Remove the unused lifetime counter and cover the partition lookup and statistics/partition lifecycle in AnalysisManagerTest. Cloud commits and partial publish retries remain subject to the existing asynchronous statistics limitations. A report that already includes the load may temporarily cause an overestimate until the next report. ### Release note Shared-nothing loads update partition changelog row-count estimates before periodic tablet statistics arrive. ### Check List (For Author) - Test: Unit Test - AnalysisManagerTest: 28 passed - TableStatsMetaTest: 15 passed - FE Checkstyle validation passed - Behavior changed: Yes, publish deltas update partition changelog estimates - Does this need documentation: No
1 parent fecb190 commit 9fb10c4

4 files changed

Lines changed: 128 additions & 74 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/AnalysisManager.java‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,13 +1516,33 @@ public void replayUpdateRowsRecord(UpdateRowsEvent event) {
15161516
long changelogIndexId = olapTable.hasChangelog()
15171517
? olapTable.getChangelogIndexMeta().getIndexId() : -1;
15181518
long rowsForAllReplica = 0;
1519-
long changelogRowsForAllReplica = 0;
15201519
for (Entry<Long, Long> entry : tabletRows.entrySet()) {
1521-
if (changelogIndexId > 0 && isChangelogTablet(entry.getKey(), changelogIndexId)) {
1522-
// The change rows of the load, which are neither the rows loaded into the
1523-
// base index nor a lower bound of the table's rows: see TableStatsMeta.
1524-
changelogRowsForAllReplica += entry.getValue();
1525-
continue;
1520+
if (changelogIndexId > 0) {
1521+
TabletMeta tabletMeta = Env.getCurrentEnv().getTabletInvertedIndex()
1522+
.getTabletMeta(entry.getKey());
1523+
// A partition can be force-dropped before the transaction becomes visible.
1524+
// Its changelog tablets must not be mistaken for base tablets after removal.
1525+
if (tabletMeta == null) {
1526+
continue;
1527+
}
1528+
Partition partition = olapTable.getPartition(tabletMeta.getPartitionId());
1529+
if (partition == null) {
1530+
continue;
1531+
}
1532+
if (tabletMeta.getIndexId() == changelogIndexId) {
1533+
short partitionReplicaNum = olapTable.getPartitionInfo()
1534+
.getReplicaAllocation(tabletMeta.getPartitionId()).getTotalReplicaNum();
1535+
MaterializedIndex changelogIndex = partition.getIndex(changelogIndexId);
1536+
// Update the partition index that stream P/Q snapshots read. A periodic
1537+
// tablet report replaces this estimate with the retained row count, so
1538+
// TTL does not leave a lifetime accumulator behind. The transaction
1539+
// visibility path holds the table write lock, excluding tablet stats.
1540+
// A report may already include this load; adding it again temporarily
1541+
// overestimates the count until the next report.
1542+
changelogIndex.setRowCount(Math.max(0, changelogIndex.getRowCount())
1543+
+ entry.getValue() / partitionReplicaNum);
1544+
continue;
1545+
}
15261546
}
15271547
rowsForAllReplica += entry.getValue();
15281548
if (LOG.isDebugEnabled()) {
@@ -1535,7 +1555,6 @@ public void replayUpdateRowsRecord(UpdateRowsEvent event) {
15351555
+ "rows for all replica {}, tablets count {}",
15361556
olapTable.getName(), tableUpdateRows, replicaNum, rowsForAllReplica, tabletRows.size());
15371557
statsStatus.updatedRows.addAndGet(tableUpdateRows);
1538-
statsStatus.addIndexUpdatedRows(changelogIndexId, changelogRowsForAllReplica / replicaNum);
15391558
if (StatisticsUtil.enablePartitionAnalyze()) {
15401559
updatePartitionRows(olapTable, tabletRows, statsStatus, replicaNum);
15411560
}
@@ -1563,11 +1582,6 @@ public void replayUpdateRowsRecord(UpdateRowsEvent event) {
15631582
}
15641583
}
15651584

1566-
private static boolean isChangelogTablet(long tabletId, long changelogIndexId) {
1567-
TabletMeta tabletMeta = Env.getCurrentEnv().getTabletInvertedIndex().getTabletMeta(tabletId);
1568-
return tabletMeta != null && tabletMeta.getIndexId() == changelogIndexId;
1569-
}
1570-
15711585
protected void updatePartitionRows(OlapTable table, Map<Long, Long> originTabletToRows,
15721586
TableStatsMeta tableStats, short replicaNum) {
15731587
if (!table.isPartitionedTable()) {

‎fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/TableStatsMeta.java‎

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,6 @@ public class TableStatsMeta implements Writable, GsonPostProcessable {
9999
@SerializedName("newPartitionLoaded")
100100
public AtomicBoolean partitionChanged = new AtomicBoolean(false);
101101

102-
// Rows a load wrote to an index which the collected row counts do not describe, accumulated
103-
// since the record was created. The changelog index is one: no analysis collects its row count,
104-
// and the change rows a load writes are not the rows it loaded into the base index.
105-
@SerializedName("iur")
106-
private ConcurrentMap<Long, Long> indexUpdatedRows;
107-
108102
@SerializedName("userInjected")
109103
public boolean userInjected;
110104

@@ -397,12 +391,6 @@ public long getBaseIndexDeltaRowCount(OlapTable table) {
397391
* {@link #updatedRows} and reports them as delta rows.
398392
*/
399393
public synchronized long getRowCountWithDeltaRows(OlapTable table, long indexId) {
400-
if (isChangelogIndex(table, indexId)) {
401-
// The changelog index has no collected row count: the rows a load wrote to it are the
402-
// only count this record keeps, and they are added rather than taken as a lower bound
403-
// of the table's rows.
404-
return indexUpdatedRows().getOrDefault(indexId, 0L);
405-
}
406394
long rowCount = getRowCount(indexId);
407395
if (!keepsOneRowPerBaseRow(table, indexId)) {
408396
// The index aggregates, or it merges the rows of a unique key table: it has its own, smaller row
@@ -426,31 +414,6 @@ public synchronized long getRowCountWithDeltaRows(OlapTable table, long indexId)
426414
return rowCount + getBaseIndexDeltaRowCount(table);
427415
}
428416

429-
/**
430-
* Add the rows a load wrote to an index which has no collected row count of its own.
431-
*/
432-
public synchronized void addIndexUpdatedRows(long indexId, long rows) {
433-
if (rows <= 0) {
434-
return;
435-
}
436-
indexUpdatedRows().merge(indexId, rows, Long::sum);
437-
}
438-
439-
public synchronized long getIndexUpdatedRows(long indexId) {
440-
return indexUpdatedRows().getOrDefault(indexId, 0L);
441-
}
442-
443-
private static boolean isChangelogIndex(OlapTable table, long indexId) {
444-
return table.hasChangelog() && table.getChangelogIndexMeta().getIndexId() == indexId;
445-
}
446-
447-
private ConcurrentMap<Long, Long> indexUpdatedRows() {
448-
if (indexUpdatedRows == null) {
449-
indexUpdatedRows = new ConcurrentHashMap<>();
450-
}
451-
return indexUpdatedRows;
452-
}
453-
454417
public boolean isColumnsStatsEmpty() {
455418
return colToColStatsMeta == null || colToColStatsMeta.isEmpty();
456419
}

‎fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalysisManagerTest.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,20 @@
2222
import org.apache.doris.catalog.Database;
2323
import org.apache.doris.catalog.DatabaseIf;
2424
import org.apache.doris.catalog.Env;
25+
import org.apache.doris.catalog.KeysType;
26+
import org.apache.doris.catalog.LocalTabletInvertedIndex;
27+
import org.apache.doris.catalog.MaterializedIndex;
28+
import org.apache.doris.catalog.MaterializedIndexMeta;
2529
import org.apache.doris.catalog.OlapTable;
2630
import org.apache.doris.catalog.Partition;
31+
import org.apache.doris.catalog.PartitionInfo;
2732
import org.apache.doris.catalog.PrimitiveType;
33+
import org.apache.doris.catalog.RecyclePartitionParam;
2834
import org.apache.doris.catalog.ReplicaAllocation;
2935
import org.apache.doris.catalog.TableIf;
3036
import org.apache.doris.catalog.TableProperty;
37+
import org.apache.doris.catalog.TabletInvertedIndex;
38+
import org.apache.doris.catalog.TabletMeta;
3139
import org.apache.doris.catalog.info.TableNameInfo;
3240
import org.apache.doris.common.Config;
3341
import org.apache.doris.common.DdlException;
@@ -52,6 +60,7 @@
5260
import org.apache.doris.statistics.util.DBObjects;
5361
import org.apache.doris.statistics.util.StatisticsUtil;
5462
import org.apache.doris.thrift.TQueryColumn;
63+
import org.apache.doris.thrift.TStorageMedium;
5564

5665
import com.google.common.collect.ImmutableList;
5766
import com.google.common.collect.ImmutableMap;
@@ -725,6 +734,99 @@ private OlapTable mockOlapTableInReplica(long tableId, String name, Database db)
725734
return table;
726735
}
727736

737+
@Test
738+
public void testChangelogRowCountUpdatedByLoadsAndPartitionLifecycle() throws Exception {
739+
AnalysisManager manager = new AnalysisManager();
740+
InternalCatalog catalog = new InternalCatalog();
741+
Database db = new Database(20001L, "test_changelog_tablet_rows");
742+
catalog.unprotectCreateDb(db);
743+
PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class);
744+
OlapTable table = Mockito.spy(new OlapTable(30001L, "t_changelog", ImmutableList.of(),
745+
KeysType.DUP_KEYS, partitionInfo, null));
746+
table.setBaseIndexId(10001L);
747+
Mockito.doReturn(db).when(table).getDatabase();
748+
TableProperty tableProperty = Mockito.mock(TableProperty.class);
749+
Mockito.doReturn(tableProperty).when(table).getTableProperty();
750+
Mockito.when(tableProperty.getReplicaAllocation()).thenReturn(new ReplicaAllocation((short) 3));
751+
db.registerTable(table);
752+
long changelogIndexId = 10002L;
753+
MaterializedIndexMeta changelogMeta = Mockito.mock(MaterializedIndexMeta.class);
754+
Mockito.when(changelogMeta.getIndexId()).thenReturn(changelogIndexId);
755+
Mockito.doReturn(true).when(table).hasChangelog();
756+
Mockito.doReturn(changelogMeta).when(table).getChangelogIndexMeta();
757+
Mockito.when(partitionInfo.getReplicaAllocation(10L)).thenReturn(new ReplicaAllocation((short) 1));
758+
Mockito.when(partitionInfo.getReplicaAllocation(20L)).thenReturn(new ReplicaAllocation((short) 3));
759+
Partition partitionOne = new Partition(10L, "p_one", new MaterializedIndex(
760+
table.getBaseIndexId(), MaterializedIndex.IndexState.NORMAL), null);
761+
Partition partitionThree = new Partition(20L, "p_three", new MaterializedIndex(
762+
table.getBaseIndexId(), MaterializedIndex.IndexState.NORMAL), null);
763+
MaterializedIndex indexOne = new MaterializedIndex(changelogIndexId, MaterializedIndex.IndexState.NORMAL);
764+
MaterializedIndex indexThree = new MaterializedIndex(changelogIndexId, MaterializedIndex.IndexState.NORMAL);
765+
indexOne.setIsChangelog(true);
766+
indexThree.setIsChangelog(true);
767+
indexThree.setRowCount(100);
768+
partitionOne.createRollupIndex(indexOne);
769+
partitionThree.createRollupIndex(indexThree);
770+
table.addPartition(partitionOne);
771+
table.addPartition(partitionThree);
772+
773+
TabletInvertedIndex invertedIndex = new LocalTabletInvertedIndex();
774+
invertedIndex.addTablet(1L, new TabletMeta(db.getId(), table.getId(), 20L,
775+
table.getBaseIndexId(), 0, TStorageMedium.HDD, false));
776+
invertedIndex.addTablet(2L, new TabletMeta(db.getId(), table.getId(), 10L,
777+
changelogIndexId, 0, TStorageMedium.HDD, true));
778+
invertedIndex.addTablet(3L, new TabletMeta(db.getId(), table.getId(), 20L,
779+
changelogIndexId, 0, TStorageMedium.HDD, true));
780+
Env env = Mockito.mock(Env.class);
781+
Mockito.when(env.getTabletInvertedIndex()).thenReturn(invertedIndex);
782+
Map<Long, Map<Long, Long>> tabletRecords = ImmutableMap.of(table.getId(),
783+
ImmutableMap.of(1L, 12L, 2L, 9L, 3L, 18L));
784+
try (MockedStatic<Env> envMockedStatic = Mockito.mockStatic(Env.class)) {
785+
envMockedStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog);
786+
envMockedStatic.when(Env::getCurrentEnv).thenReturn(env);
787+
manager.replayUpdateRowsRecord(new UpdateRowsEvent(tabletRecords, db.getId()));
788+
TableStatsMeta record = manager.findTableStatsStatus(table.getId());
789+
Assertions.assertEquals(4, record.updatedRows.get());
790+
Assertions.assertEquals(9, indexOne.getRowCount());
791+
Assertions.assertEquals(106, indexThree.getRowCount());
792+
Assertions.assertEquals(9, table.getRowCountForPartitionIndex(10L, changelogIndexId, false));
793+
Assertions.assertEquals(106, table.getRowCountForPartitionIndex(20L, changelogIndexId, false));
794+
Assertions.assertEquals(115, table.getRowCountForIndex(changelogIndexId, false));
795+
796+
// A late transaction report can still contain a force-dropped changelog tablet.
797+
invertedIndex.deleteTablet(2L);
798+
manager.replayUpdateRowsRecord(new UpdateRowsEvent(tabletRecords, db.getId()));
799+
Assertions.assertEquals(8, record.updatedRows.get());
800+
Assertions.assertEquals(9, indexOne.getRowCount());
801+
Assertions.assertEquals(112, indexThree.getRowCount());
802+
803+
// DROP STATS/recreation does not erase the changelog row-count baseline.
804+
manager.removeTableStats(table.getId());
805+
manager.replayUpdateTableStatsStatus(new TableStatsMeta(table));
806+
record = manager.findTableStatsStatus(table.getId());
807+
Assertions.assertEquals(121, table.getRowCountForIndex(changelogIndexId, false));
808+
809+
// Periodic tablet statistics replace the estimate after TTL removes rowsets.
810+
indexThree.setRowCount(5);
811+
Assertions.assertEquals(5, table.getRowCountForPartitionIndex(20L, changelogIndexId, false));
812+
manager.replayUpdateRowsRecord(new UpdateRowsEvent(tabletRecords, db.getId()));
813+
Assertions.assertEquals(4, record.updatedRows.get());
814+
Assertions.assertEquals(11, table.getRowCountForPartitionIndex(20L, changelogIndexId, false));
815+
816+
// A partial TRUNCATE replaces just this partition and its changelog index. Old
817+
// reports must not repopulate it, even while the old tablet metadata still exists.
818+
Partition replacement = new Partition(30L, "p_three", new MaterializedIndex(
819+
table.getBaseIndexId(), MaterializedIndex.IndexState.NORMAL), null);
820+
replacement.createRollupIndex(new MaterializedIndex(changelogIndexId, MaterializedIndex.IndexState.NORMAL));
821+
table.replacePartition(replacement, new RecyclePartitionParam());
822+
manager.replayUpdateRowsRecord(new UpdateRowsEvent(tabletRecords, db.getId()));
823+
Assertions.assertEquals(4, record.updatedRows.get());
824+
Assertions.assertEquals(0, table.getRowCountForPartitionIndex(30L, changelogIndexId, false));
825+
Assertions.assertEquals(9, table.getRowCountForIndex(changelogIndexId, false));
826+
Assertions.assertEquals(9, indexOne.getRowCount());
827+
}
828+
}
829+
728830
/**
729831
* A load recreates the statistics record of a table whose record was removed, e.g. by TRUNCATE TABLE, and
730832
* accumulates the loaded rows into it. The record is created on the transaction visibility of the load,

‎fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/TableStatsMetaTest.java‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.doris.catalog.MaterializedIndexMeta;
2626
import org.apache.doris.catalog.OlapTable;
2727
import org.apache.doris.catalog.PrimitiveType;
28-
import org.apache.doris.catalog.TableIf;
2928
import org.apache.doris.common.Pair;
3029
import org.apache.doris.datasource.CatalogIf;
3130

@@ -94,30 +93,6 @@ void update() {
9493
Assertions.assertEquals(4, tableStatsMeta.rowCount);
9594
}
9695

97-
@Test
98-
void testTheChangelogRowsOfALoadAreAccumulated() {
99-
OlapTable table = mockOlapTable();
100-
long changelogIndexId = 10003L;
101-
MaterializedIndexMeta changelogMeta = Mockito.mock(MaterializedIndexMeta.class);
102-
Mockito.doReturn(changelogIndexId).when(changelogMeta).getIndexId();
103-
Mockito.doReturn(changelogMeta).when(table).getChangelogIndexMeta();
104-
Mockito.doReturn(true).when(table).hasChangelog();
105-
106-
TableStatsMeta meta = new TableStatsMeta();
107-
meta.addIndexUpdatedRows(changelogIndexId, 12);
108-
meta.addIndexUpdatedRows(changelogIndexId, 8);
109-
// Nothing to add: a load which wrote no change row must not create an entry.
110-
meta.addIndexUpdatedRows(changelogIndexId, 0);
111-
112-
// A changelog index has no collected row count, so the rows the loads wrote to it are the
113-
// count. They are neither the rows of the base index nor a lower bound of the table's rows:
114-
// an update adds a change row without adding a base row, a delete adds none to the base.
115-
Assertions.assertEquals(20, meta.getRowCountWithDeltaRows(table, changelogIndexId));
116-
Assertions.assertEquals(20, meta.getIndexUpdatedRows(changelogIndexId));
117-
Assertions.assertEquals(0, meta.getIndexUpdatedRows(BASE_INDEX_ID));
118-
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, meta.getRowCountWithDeltaRows(table, BASE_INDEX_ID));
119-
}
120-
12196
@Test
12297
void testClearStaleIndexRowCount() {
12398
TableStatsMeta meta = new TableStatsMeta();

0 commit comments

Comments
 (0)