Skip to content

Commit 190eb6a

Browse files
authored
[improvement](statistics) Share partitionUpdateRows map between analyze job and its tasks (apache#66488)
### What problem does this PR solve? Problem Summary: Every column task deep-copies the whole `partitionUpdateRows` map from the analyze job through the `AnalysisInfo` copy constructor. With a large number of partitions this yields an O(Task x Partition) memory amplification: the copied maps are retained by all history records in `analysisJobInfoMap` / `analysisTaskInfoMap` (up to `analyze_record_limit` each), so a 4200-partition table analyzed with many columns keeps tens of millions of `ConcurrentHashMap` nodes alive in the FE heap. This change makes tasks share the job's map reference instead of deep-copying it, and clears the shared map once at the job terminal state, so the memory held by every job/task record is released at once.
1 parent 910b741 commit 190eb6a

9 files changed

Lines changed: 366 additions & 27 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,15 @@ public enum ScheduleType {
189189
@SerializedName("tv")
190190
public final long tableVersion;
191191

192-
public final Map<Long, Long> partitionUpdateRows = new ConcurrentHashMap<>();
192+
/**
193+
* Number of updated rows per partition. This map is shared between a job and all
194+
* of its tasks: each task's AnalysisInfo holds the same reference as the job's,
195+
* instead of a deep copy, so that a single clear at the job terminal state (see
196+
* AnalysisManager#updateTaskStatus) releases the memory retained by every task
197+
* record at once. It is null when the job is recovered from the image because the
198+
* field is not persisted.
199+
*/
200+
public final Map<Long, Long> partitionUpdateRows;
193201

194202
@SerializedName("tblUpdateTime")
195203
public final long tblUpdateTime;
@@ -254,9 +262,11 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
254262
this.updateRows = updateRows;
255263
this.tableVersion = tableVersion;
256264
this.priority = priority;
257-
if (partitionUpdateRows != null) {
258-
this.partitionUpdateRows.putAll(partitionUpdateRows);
259-
}
265+
// Share the same partitionUpdateRows map reference between the job and all its
266+
// tasks instead of deep-copying it per task, so that clearing it once at the
267+
// job terminal state releases the memory retained by every task record at once.
268+
this.partitionUpdateRows = partitionUpdateRows == null
269+
? new ConcurrentHashMap<>() : partitionUpdateRows;
260270
this.enablePartition = enablePartition;
261271
}
262272

‎fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,14 @@ public void taskFailed(BaseAnalysisTask task, String reason) {
182182
updateTaskState(AnalysisState.FAILED, reason);
183183
cancel();
184184
} finally {
185+
// All task threads have been marked killed by cancel(), so the guarded
186+
// partitionUpdateRows writers have stopped. Clear the shared map here, after
187+
// the cancels (the terminal-state clear inside updateTaskStatus runs while
188+
// sibling threads may still be executing and can miss in-flight writes), so
189+
// the job and all its task records release the map at once.
190+
if (jobInfo.partitionUpdateRows != null) {
191+
jobInfo.partitionUpdateRows.clear();
192+
}
185193
deregisterJob();
186194
}
187195
}

‎fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java‎

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,9 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTableCommand command) {
429429
}
430430
infoBuilder.setTableVersion(version);
431431
infoBuilder.setPriority(JobPriority.MANUAL);
432-
infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows);
432+
// Must not alias TableStatsMeta.partitionUpdateRows: it is cleared at the job terminal state.
433+
infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null
434+
: new ConcurrentHashMap<>(tableStatsStatus.partitionUpdateRows));
433435
infoBuilder.setEnablePartition(StatisticsUtil.enablePartitionAnalyze());
434436
return infoBuilder.build();
435437
}
@@ -548,6 +550,13 @@ public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String
548550
if (MetricRepo.isInit) {
549551
MetricRepo.COUNTER_STATISTICS_FAILED_ANALYZE_JOB.increase(1L);
550552
}
553+
// The job reached a terminal state: all tasks share the job's
554+
// partitionUpdateRows map, so clearing it here releases the memory
555+
// retained by every task record in the history at once. The success
556+
// path clears it inside updateTableStats.
557+
if (job.partitionUpdateRows != null) {
558+
job.partitionUpdateRows.clear();
559+
}
551560
} else {
552561
job.markFinished();
553562
if (MetricRepo.isInit) {
@@ -567,22 +576,27 @@ public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String
567576

568577
@VisibleForTesting
569578
public void updateTableStats(AnalysisInfo jobInfo) {
570-
TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
571-
TableStatsMeta tableStats = findTableStatsStatus(tbl.getId());
572-
if (tableStats == null) {
573-
updateTableStatsStatus(new TableStatsMeta(jobInfo.rowCount, jobInfo, tbl));
574-
} else {
575-
tableStats.update(jobInfo, tbl);
576-
logCreateTableStats(tableStats);
577-
}
578-
if (jobInfo.jobColumns != null) {
579-
jobInfo.jobColumns.clear();
580-
}
581-
if (jobInfo.partitionNames != null) {
582-
jobInfo.partitionNames.clear();
583-
}
584-
if (jobInfo.partitionUpdateRows != null) {
585-
jobInfo.partitionUpdateRows.clear();
579+
// Clear the shared maps even when the stats update fails, so the job and all its
580+
// task records release their memory once the job reaches a terminal state.
581+
try {
582+
TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
583+
TableStatsMeta tableStats = findTableStatsStatus(tbl.getId());
584+
if (tableStats == null) {
585+
updateTableStatsStatus(new TableStatsMeta(jobInfo.rowCount, jobInfo, tbl));
586+
} else {
587+
tableStats.update(jobInfo, tbl);
588+
logCreateTableStats(tableStats);
589+
}
590+
} finally {
591+
if (jobInfo.jobColumns != null) {
592+
jobInfo.jobColumns.clear();
593+
}
594+
if (jobInfo.partitionNames != null) {
595+
jobInfo.partitionNames.clear();
596+
}
597+
if (jobInfo.partitionUpdateRows != null) {
598+
jobInfo.partitionUpdateRows.clear();
599+
}
586600
}
587601
}
588602

@@ -992,11 +1006,22 @@ public void handleKillAnalyzeJob(KillAnalyzeJobCommand killAnalyzeJobCommand) th
9921006
return;
9931007
}
9941008
checkPriv(anyTask);
995-
logKilled(analysisJobInfoMap.get(anyTask.getJobId()));
1009+
AnalysisInfo job = analysisJobInfoMap.get(anyTask.getJobId());
1010+
// The job record may have been evicted from analysisJobInfoMap while the job was
1011+
// still running; its shared partitionUpdateRows map was already cleared at
1012+
// eviction time (see replayCreateAnalysisJob), so skip the job-level update.
1013+
if (job != null) {
1014+
logKilled(job);
1015+
}
9961016
for (BaseAnalysisTask taskInfo : analysisTaskMap.values()) {
9971017
taskInfo.cancel();
9981018
logKilled(taskInfo.info);
9991019
}
1020+
// The job reached a terminal state: all tasks share the job's partitionUpdateRows
1021+
// map, so clearing it here releases the memory retained by every task record.
1022+
if (job != null && job.partitionUpdateRows != null) {
1023+
job.partitionUpdateRows.clear();
1024+
}
10001025
}
10011026

10021027
private void logKilled(AnalysisInfo info) {
@@ -1035,7 +1060,15 @@ private BaseAnalysisTask createTask(AnalysisInfo analysisInfo) throws DdlExcepti
10351060
public void replayCreateAnalysisJob(AnalysisInfo jobInfo) {
10361061
synchronized (analysisJobInfoMap) {
10371062
while (analysisJobInfoMap.size() >= Config.analyze_record_limit) {
1038-
analysisJobInfoMap.remove(analysisJobInfoMap.pollFirstEntry().getKey());
1063+
// pollFirstEntry removes the oldest entry from the map.
1064+
AnalysisInfo evicted = analysisJobInfoMap.pollFirstEntry().getValue();
1065+
// The evicted job may still be running: its updateTaskStatus will return
1066+
// early on the "job == null" guard and never reach the terminal-state
1067+
// clear, so clear the shared partitionUpdateRows map here to release the
1068+
// memory held by the evicted job and all its task records at once.
1069+
if (evicted.partitionUpdateRows != null) {
1070+
evicted.partitionUpdateRows.clear();
1071+
}
10391072
}
10401073
if (jobInfo.message != null && jobInfo.message.length() >= StatisticConstants.MSG_LEN_UPPER_BOUND) {
10411074
jobInfo.message = jobInfo.message.substring(0, StatisticConstants.MSG_LEN_UPPER_BOUND);

‎fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -528,14 +528,22 @@ protected void doPartitionTable() throws Exception {
528528
if (partitionRowCount > hugePartitionThreshold && AnalysisInfo.JobType.SYSTEM.equals(info.jobType)) {
529529
hasHughPartition = true;
530530
// -1 means it's skipped because this partition is too large.
531-
jobInfo.partitionUpdateRows.putIfAbsent(partition.getId(), -1L);
531+
// Skip the write when the task is cancelled: the job may already be in a
532+
// terminal state and have cleared the shared map, so a late write would
533+
// re-populate it and keep the memory alive.
534+
if (!killed) {
535+
jobInfo.partitionUpdateRows.putIfAbsent(partition.getId(), -1L);
536+
}
532537
LOG.info("Partition {} in table {} is too large, skip it.", part, tbl.getName());
533538
continue;
534539
}
535540
batchRowCount += partitionRowCount;
536541
// For cluster upgrade compatible (older version metadata doesn't have partition update rows map)
537542
// and insert before first analyze, set partition update rows to 0.
538-
jobInfo.partitionUpdateRows.putIfAbsent(partition.getId(), 0L);
543+
// Skip the write when the task is cancelled: see the comment above.
544+
if (!killed) {
545+
jobInfo.partitionUpdateRows.putIfAbsent(partition.getId(), 0L);
546+
}
539547
}
540548
params.put("partId", partition == null ? "-1" : String.valueOf(partition.getId()));
541549
// Skip partitions that not changed after last analyze.

‎fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,13 @@ protected void deleteNotExistPartitionStats(AnalysisInfo jobInfo) throws DdlExce
416416
if (partition == null) {
417417
columnStats.partitionUpdateRows.remove(partId);
418418
tableStats.partitionUpdateRows.remove(partId);
419-
jobInfo.partitionUpdateRows.remove(partId);
419+
// Skip the write when the task is cancelled: the job may already be in a
420+
// terminal state and have cleared the shared map, so no write should touch
421+
// it afterwards (a remove is a no-op on the cleared map, but keep the
422+
// invariant that cancelled tasks never write to the shared map).
423+
if (!killed) {
424+
jobInfo.partitionUpdateRows.remove(partId);
425+
}
420426
expiredPartition.add(partId);
421427
if (expiredPartition.size() == Config.max_allowed_in_element_num_of_delete) {
422428
String partitionCondition = " AND part_id in (" + Joiner.on(", ").join(expiredPartition) + ")";

‎fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.util.Optional;
4646
import java.util.Set;
4747
import java.util.StringJoiner;
48+
import java.util.concurrent.ConcurrentHashMap;
4849
import java.util.concurrent.ExecutionException;
4950
import java.util.concurrent.Future;
5051
import java.util.concurrent.TimeUnit;
@@ -275,7 +276,9 @@ protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<Pair<String, St
275276
.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get())
276277
.setTableVersion(version)
277278
.setPriority(priority)
278-
.setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows)
279+
// Must not alias TableStatsMeta.partitionUpdateRows: cleared at the job terminal state.
280+
.setPartitionUpdateRows(tableStatsStatus == null ? null
281+
: new ConcurrentHashMap<>(tableStatsStatus.partitionUpdateRows))
279282
.setEnablePartition(StatisticsUtil.enablePartitionAnalyze())
280283
.build();
281284
}

‎fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.statistics;
1919

20+
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
2021
import org.apache.doris.statistics.util.StatisticsUtil;
2122

2223
import org.junit.jupiter.api.Assertions;
@@ -28,6 +29,8 @@
2829
import java.util.Arrays;
2930
import java.util.Collections;
3031
import java.util.HashSet;
32+
import java.util.concurrent.ConcurrentHashMap;
33+
import java.util.concurrent.ConcurrentMap;
3134
import java.util.concurrent.atomic.AtomicInteger;
3235

3336
public class AnalysisJobTest {
@@ -124,6 +127,25 @@ public void testAppendBufTest3() {
124127
Assertions.assertEquals(1, writeBufInvokeTimes.get());
125128
}
126129

130+
@Test
131+
public void testTaskFailedClearsSharedPartitionUpdateRows() {
132+
ConcurrentMap<Long, Long> partitionRows = new ConcurrentHashMap<>();
133+
partitionRows.put(1L, 10L);
134+
partitionRows.put(2L, 20L);
135+
AnalysisInfo jobInfo = new AnalysisInfoBuilder().setJobId(1).setPartitionUpdateRows(partitionRows)
136+
.setState(AnalysisState.RUNNING).setAnalysisType(AnalysisType.FUNDAMENTALS)
137+
.setJobType(AnalysisInfo.JobType.MANUAL).build();
138+
BaseAnalysisTask task = Mockito.mock(BaseAnalysisTask.class);
139+
140+
AnalysisJob job = Mockito.spy(new AnalysisJob(jobInfo, Collections.singletonList(task)));
141+
job.analysisManager = Mockito.mock(AnalysisManager.class);
142+
Mockito.doNothing().when(job).deregisterJob();
143+
144+
job.taskFailed(task, "boom");
145+
Assertions.assertTrue(partitionRows.isEmpty(),
146+
"taskFailed must clear the shared partitionUpdateRows map after cancelling tasks");
147+
}
148+
127149
@Test
128150
public void testUpdateTaskState() {
129151
AnalysisInfo info = Mockito.mock(AnalysisInfo.class);

0 commit comments

Comments
 (0)