Skip to content

Commit a5bb3aa

Browse files
committed
[refactor](nereids) Remove minidump and Nereids tracing
### What problem does this PR solve? Issue Number: None Related PR: #66817 Problem Summary: The remaining minidump dump path and Nereids trace framework are no longer needed after offline plan replay was removed. Remove PLAN REPLAYER DUMP and its REST endpoint, minidump serialization, replay-only statistics and context plumbing, the enable_nereids_trace variables, and the event tracing implementation. Also remove replay-specific privilege and row-policy bypasses so normal planning always uses catalog statistics and standard authorization paths. ### Release note Remove `PLAN REPLAYER DUMP`, the `/api/minidump` endpoint, and the `enable_nereids_trace` and `nereids_trace_event_mode` session variables. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - `NereidsParserTest` - `UserAuthenticationTest` - [ ] Manual test - [ ] No need to test or manual test - Behavior changed: - [ ] No. - [x] Yes. Remove the minidump, plan replayer dump, and Nereids trace interfaces. - Does this need documentation? - [x] No. These developer-only interfaces are not documented. - [ ] Yes.
1 parent e870abe commit a5bb3aa

60 files changed

Lines changed: 48 additions & 3315 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2785,9 +2785,6 @@ public class Config extends ConfigBase {
27852785
+ "password `*A00C34073A26B40AB4307650BFB9309D6BFA6999`")
27862786
public static String initial_root_password = "";
27872787

2788-
@ConfField(description = "The path of the nereids trace file.")
2789-
public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace";
2790-
27912788
@ConfField(mutable = true, masterOnly = true, description = "The maximum number of snapshots assigned to an "
27922789
+ "upload task during the backup process. The default " + "value is 10.")
27932790
public static int backup_upload_snapshot_batch_size = 10;
@@ -2965,9 +2962,6 @@ public void handle(Field field, String value) throws Exception {
29652962
@ConfField
29662963
public static String spilled_profile_storage_path = System.getenv("LOG_DIR") + File.separator + "profile";
29672964

2968-
@ConfField
2969-
public static String spilled_minidump_storage_path = System.getenv("LOG_DIR") + File.separator + "minidump";
2970-
29712965
// The max number of profiles that can be stored to storage.
29722966
@ConfField
29732967
public static int max_spilled_profile_num = 500;

‎fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MinidumpAction.java‎

Lines changed: 0 additions & 69 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,6 @@
4545
import org.apache.doris.nereids.memo.Group;
4646
import org.apache.doris.nereids.memo.GroupExpression;
4747
import org.apache.doris.nereids.memo.Memo;
48-
import org.apache.doris.nereids.metrics.event.CounterEvent;
49-
import org.apache.doris.nereids.minidump.MinidumpUtils;
50-
import org.apache.doris.nereids.minidump.NereidsTracer;
5148
import org.apache.doris.nereids.processor.post.PlanPostProcessors;
5249
import org.apache.doris.nereids.processor.pre.PlanPreprocessors;
5350
import org.apache.doris.nereids.properties.PhysicalProperties;
@@ -98,7 +95,6 @@
9895
import org.apache.logging.log4j.LogManager;
9996
import org.apache.logging.log4j.Logger;
10097

101-
import java.io.IOException;
10298
import java.lang.management.GarbageCollectorMXBean;
10399
import java.lang.management.ManagementFactory;
104100
import java.nio.ByteBuffer;
@@ -146,11 +142,6 @@ public NereidsPlanner(StatementContext statementContext) {
146142
@Override
147143
public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions queryOptions) throws UserException {
148144
this.queryOptions = queryOptions;
149-
if (statementContext.getConnectContext().getSessionVariable().isEnableNereidsTrace()) {
150-
NereidsTracer.init();
151-
} else {
152-
NereidsTracer.disable();
153-
}
154145
if (!(queryStmt instanceof LogicalPlanAdapter)) {
155146
throw new RuntimeException("Wrong type of queryStmt, expected: <? extends LogicalPlanAdapter>");
156147
}
@@ -160,7 +151,6 @@ public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions
160151
ExplainLevel explainLevel = getExplainLevel(queryStmt.getExplainOptions());
161152

162153
LogicalPlan parsedPlan = logicalPlanAdapter.getLogicalPlan();
163-
NereidsTracer.logImportantTime("EndParsePlan");
164154
setParsedPlan(parsedPlan);
165155

166156
PhysicalProperties requireProperties = buildInitRequireProperties();
@@ -291,12 +281,6 @@ private Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties
291281
private Plan planWithoutLock(
292282
LogicalPlan plan, PhysicalProperties requireProperties, ExplainLevel explainLevel,
293283
boolean showPlanProcess) {
294-
// minidump of input must be serialized first, this process ensure minidump string not null
295-
try {
296-
MinidumpUtils.serializeInputsToDumpFile(plan, statementContext);
297-
} catch (IOException e) {
298-
throw new RuntimeException(e);
299-
}
300284
// analyze this query, resolve column, table and function
301285
analyze(showAnalyzeProcess(explainLevel, showPlanProcess));
302286
if (explainLevel == ExplainLevel.ANALYZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) {
@@ -358,9 +342,6 @@ private Plan planWithoutLock(
358342
|| explainLevel == ExplainLevel.SHAPE_PLAN) {
359343
optimizedPlan = physicalPlan;
360344
}
361-
// serialize optimized plan to dumpfile, dumpfile do not have this part means optimize failed
362-
MinidumpUtils.serializeOutputToDumpFile(physicalPlan);
363-
NereidsTracer.output(statementContext.getConnectContext());
364345
return physicalPlan;
365346
}
366347

@@ -462,7 +443,6 @@ protected void collectAndLockTable(boolean showPlanProcess, boolean waitForChang
462443
}
463444
statementContext.lock();
464445
cascadesContext.setCteContext(new CTEContext());
465-
NereidsTracer.logImportantTime("EndCollectAndLockTables");
466446
if (LOG.isDebugEnabled()) {
467447
LOG.debug("End collect and lock table");
468448
}
@@ -498,7 +478,6 @@ protected void analyze(boolean showPlanProcess) {
498478
statementContext.getPlannerHooks().forEach(hook -> hook.beforeAnalyze(this));
499479
keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newAnalyzer().analyze());
500480
statementContext.getPlannerHooks().forEach(hook -> hook.afterAnalyze(this));
501-
NereidsTracer.logImportantTime("EndAnalyzePlan");
502481
if (LOG.isDebugEnabled()) {
503482
LOG.debug("End analyze plan");
504483
}
@@ -519,7 +498,6 @@ protected void rewrite(boolean showPlanProcess) {
519498
keepOrShowPlanProcess(showPlanProcess, () -> {
520499
Rewriter.getWholeTreeRewriter(cascadesContext).execute();
521500
});
522-
NereidsTracer.logImportantTime("EndRewritePlan");
523501
if (LOG.isDebugEnabled()) {
524502
LOG.debug("End rewrite plan");
525503
}
@@ -587,7 +565,6 @@ protected void preMaterializedViewRewrite() {
587565
if (!plansWhichContainMv.isEmpty()) {
588566
plansWhichContainMv.forEach(statementContext::addRewrittenPlanByMv);
589567
}
590-
NereidsTracer.logImportantTime("EndPreRewritePlanByMv");
591568
if (LOG.isDebugEnabled()) {
592569
LOG.debug("End pre rewrite plan by mv");
593570
}
@@ -620,7 +597,6 @@ protected void optimize(boolean showPlanProcess) {
620597
keepOrShowPlanProcess(showPlanProcess, () -> {
621598
new Optimizer(cascadesContext).execute();
622599
});
623-
NereidsTracer.logImportantTime("EndOptimizePlan");
624600
if (LOG.isDebugEnabled()) {
625601
LOG.debug("End optimize plan");
626602
}
@@ -671,12 +647,6 @@ protected void splitFragments(PhysicalPlan resultPlan) {
671647
PhysicalPlanTranslator physicalPlanTranslator = new PhysicalPlanTranslator(planTranslatorContext,
672648
statementContext.getConnectContext().getStatsErrorEstimator());
673649
SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
674-
if (sessionVariable.isEnableNereidsTrace()) {
675-
CounterEvent.clearCounter();
676-
}
677-
if (sessionVariable.isPlayNereidsDump()) {
678-
return;
679-
}
680650
PlanFragment root = physicalPlanTranslator.translatePlan(physicalPlan);
681651
if (statementContext.getConnectContext().getExecutor() != null) {
682652
statementContext.getConnectContext().getExecutor().getSummaryProfile()

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,6 @@
1919

2020
import org.apache.doris.nereids.memo.CopyInResult;
2121
import org.apache.doris.nereids.memo.Group;
22-
import org.apache.doris.nereids.memo.GroupExpression;
23-
import org.apache.doris.nereids.memo.Memo;
24-
import org.apache.doris.nereids.metrics.CounterType;
25-
import org.apache.doris.nereids.metrics.EventChannel;
26-
import org.apache.doris.nereids.metrics.EventProducer;
27-
import org.apache.doris.nereids.metrics.TracerSupplier;
28-
import org.apache.doris.nereids.metrics.consumer.LogConsumer;
29-
import org.apache.doris.nereids.metrics.enhancer.AddCounterEventEnhancer;
30-
import org.apache.doris.nereids.metrics.event.CounterEvent;
31-
import org.apache.doris.nereids.metrics.event.TransformEvent;
3222
import org.apache.doris.nereids.rules.Rule;
3323
import org.apache.doris.nereids.rules.RuleSet;
3424
import org.apache.doris.nereids.trees.expressions.CTEId;
@@ -46,12 +36,7 @@
4636
/**
4737
* Abstract class for all job using for analyze and optimize query plan in Nereids.
4838
*/
49-
public abstract class Job implements TracerSupplier {
50-
// counter tracer to count expression transform times.
51-
protected static final EventProducer COUNTER_TRACER = new EventProducer(CounterEvent.class,
52-
EventChannel.getDefaultChannel()
53-
.addEnhancers(new AddCounterEventEnhancer())
54-
.addConsumers(new LogConsumer(CounterEvent.class, EventChannel.LOG)));
39+
public abstract class Job {
5540
protected JobType type;
5641
protected JobContext context;
5742
protected boolean once;
@@ -89,15 +74,8 @@ public ConnectContext getConnectContext() {
8974

9075
public abstract void execute();
9176

92-
public EventProducer getEventTracer() {
93-
throw new UnsupportedOperationException("get_event_tracer is unsupported");
94-
}
95-
96-
protected Optional<CopyInResult> invokeRewriteRuleWithTrace(Rule rule, Plan before, Group targetGroup) {
77+
protected Optional<CopyInResult> invokeRewriteRule(Rule rule, Plan before, Group targetGroup) {
9778
context.onInvokeRule(rule.getRuleType());
98-
COUNTER_TRACER.log(CounterEvent.of(Memo.getStateId(),
99-
CounterType.EXPRESSION_TRANSFORM, targetGroup, targetGroup.getLogicalExpression(), before));
100-
10179
List<Plan> afters = rule.transform(before, context.getCascadesContext());
10280
Preconditions.checkArgument(afters.size() == 1);
10381
Plan after = afters.get(0);
@@ -110,24 +88,9 @@ protected Optional<CopyInResult> invokeRewriteRuleWithTrace(Rule rule, Plan befo
11088
.copyIn(after, targetGroup, rule.isRewrite(),
11189
context.getCascadesContext().getStatementContext().isDpHyp());
11290

113-
if (result.generateNewExpression || result.correspondingExpression.getOwnerGroup() != targetGroup) {
114-
getEventTracer().log(TransformEvent.of(targetGroup.getLogicalExpression(), before, afters,
115-
rule.getRuleType()), rule::isRewrite);
116-
}
117-
11891
return Optional.of(result);
11992
}
12093

121-
/**
122-
* count the job execution times of groupExpressions, all groupExpressions will be inclusive.
123-
* TODO: count a specific groupExpression.
124-
* @param groupExpression the groupExpression at current job.
125-
*/
126-
protected void countJobExecutionTimesOfGroupExpressions(GroupExpression groupExpression) {
127-
COUNTER_TRACER.log(CounterEvent.of(Memo.getStateId(), CounterType.JOB_EXECUTION,
128-
groupExpression.getOwnerGroup(), groupExpression, groupExpression.getPlan()));
129-
}
130-
13194
public static BitSet getDisableRules(JobContext context) {
13295
return context.getCascadesContext().getAndCacheDisableRules();
13396
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,6 @@
2323
import org.apache.doris.nereids.jobs.JobType;
2424
import org.apache.doris.nereids.memo.CopyInResult;
2525
import org.apache.doris.nereids.memo.GroupExpression;
26-
import org.apache.doris.nereids.metrics.EventChannel;
27-
import org.apache.doris.nereids.metrics.EventProducer;
28-
import org.apache.doris.nereids.metrics.consumer.LogConsumer;
29-
import org.apache.doris.nereids.metrics.event.TransformEvent;
30-
import org.apache.doris.nereids.minidump.NereidsTracer;
3126
import org.apache.doris.nereids.pattern.GroupExpressionMatching;
3227
import org.apache.doris.nereids.rules.Rule;
3328
import org.apache.doris.nereids.rules.RuleType;
@@ -43,8 +38,6 @@
4338
* Job to apply rule on {@link GroupExpression}.
4439
*/
4540
public class ApplyRuleJob extends Job {
46-
private static final EventProducer APPLY_RULE_TRACER = new EventProducer(TransformEvent.class,
47-
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(TransformEvent.class, EventChannel.LOG)));
4841
private final GroupExpression groupExpression;
4942
private final Rule rule;
5043

@@ -68,8 +61,6 @@ public final void execute() throws AnalysisException {
6861
|| groupExpression.isUnused()) {
6962
return;
7063
}
71-
countJobExecutionTimesOfGroupExpressions(groupExpression);
72-
7364
List<DeriveStatsJob> deriveStatsJobs = Lists.newArrayList();
7465
GroupExpressionMatching groupExpressionMatching
7566
= new GroupExpressionMatching(rule.getPattern(), groupExpression);
@@ -116,9 +107,6 @@ public final void execute() throws AnalysisException {
116107
}
117108
}
118109

119-
NereidsTracer.logApplyRuleEvent(rule.toString(), plan, newGroupExpression.getPlan());
120-
APPLY_RULE_TRACER.log(TransformEvent.of(groupExpression, plan, newPlans, rule.getRuleType()),
121-
rule::isRewrite);
122110
}
123111
// we do derive stats job eager to avoid un derive stats due to merge group and optimize group
124112
// consider:

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.doris.nereids.jobs.JobType;
2626
import org.apache.doris.nereids.memo.Group;
2727
import org.apache.doris.nereids.memo.GroupExpression;
28-
import org.apache.doris.nereids.minidump.NereidsTracer;
2928
import org.apache.doris.nereids.properties.ChildOutputPropertyDeriver;
3029
import org.apache.doris.nereids.properties.ChildrenPropertiesRegulator;
3130
import org.apache.doris.nereids.properties.EnforceMissingPropertiesHelper;
@@ -118,7 +117,6 @@ public void execute() {
118117
return;
119118
}
120119

121-
countJobExecutionTimesOfGroupExpressions(groupExpression);
122120
// Do init logic of root plan/groupExpr of `subplan`, only run once per task.
123121
if (curChildIndex == -1) {
124122
curNodeCost = Cost.zero();
@@ -343,8 +341,6 @@ private void recordPropertyAndCost(GroupExpression groupExpression,
343341
groupExpression.putOutputPropertiesMap(outputProperty, requestProperty);
344342
}
345343
this.groupExpression.getOwnerGroup().setBestPlan(groupExpression, curTotalCost, requestProperty);
346-
NereidsTracer.logPropertyAndCostEvent(groupExpression.getOwnerGroup().getGroupId(),
347-
groupExpression.children(), groupExpression.getPlan(), requestProperty, curTotalCost);
348344
}
349345

350346
private void clear() {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,6 @@
2222
import org.apache.doris.nereids.jobs.JobType;
2323
import org.apache.doris.nereids.memo.Group;
2424
import org.apache.doris.nereids.memo.GroupExpression;
25-
import org.apache.doris.nereids.metrics.EventChannel;
26-
import org.apache.doris.nereids.metrics.EventProducer;
27-
import org.apache.doris.nereids.metrics.consumer.LogConsumer;
28-
import org.apache.doris.nereids.metrics.event.StatsStateEvent;
29-
import org.apache.doris.nereids.minidump.MinidumpUtils;
3025
import org.apache.doris.nereids.stats.HboStatsCalculator;
3126
import org.apache.doris.nereids.stats.StatsCalculator;
3227
import org.apache.doris.nereids.trees.expressions.CTEId;
@@ -43,9 +38,6 @@
4338
* Job to derive stats for {@link GroupExpression} in {@link org.apache.doris.nereids.memo.Memo}.
4439
*/
4540
public class DeriveStatsJob extends Job {
46-
private static final EventProducer STATS_STATE_TRACER = new EventProducer(
47-
StatsStateEvent.class,
48-
EventChannel.getDefaultChannel().addConsumers(new LogConsumer(StatsStateEvent.class, EventChannel.LOG)));
4941
private final GroupExpression groupExpression;
5042
private boolean deriveChildren;
5143

@@ -76,7 +68,6 @@ public void execute() {
7668
if (groupExpression.isStatDerived() || groupExpression.isUnused()) {
7769
return;
7870
}
79-
countJobExecutionTimesOfGroupExpressions(groupExpression);
8071
if (!deriveChildren && groupExpression.arity() > 0) {
8172
pushJob(new DeriveStatsJob(groupExpression, true, context, cteIdToStats));
8273

@@ -112,26 +103,16 @@ public void execute() {
112103
if (isHboEnabled) {
113104
statsCalculator = new HboStatsCalculator(groupExpression,
114105
sessionVariable.getForbidUnknownColStats(),
115-
connectContext.getTotalColumnStatisticMap(),
116-
sessionVariable.isPlayNereidsDump(),
117106
cteIdToStats,
118107
context.getCascadesContext());
119108
statsCalculator.estimate();
120109
} else {
121110
statsCalculator = new StatsCalculator(groupExpression,
122111
sessionVariable.getForbidUnknownColStats(),
123-
connectContext.getTotalColumnStatisticMap(),
124-
sessionVariable.isPlayNereidsDump(),
125112
cteIdToStats,
126113
context.getCascadesContext());
127114
statsCalculator.estimate();
128115
}
129-
STATS_STATE_TRACER.log(StatsStateEvent.of(groupExpression,
130-
groupExpression.getOwnerGroup().getStatistics()));
131-
if (MinidumpUtils.isDump() && !sessionVariable.isPlayNereidsDump()) {
132-
connectContext.getTotalColumnStatisticMap().putAll(statsCalculator.getTotalColumnStatisticMap());
133-
connectContext.getTotalHistogramMap().putAll(statsCalculator.getTotalHistogramMap());
134-
}
135116

136117
if (groupExpression.getPlan() instanceof Project) {
137118
// In the context of reorder join, when a new plan is generated, it may include a project operation.

0 commit comments

Comments
 (0)