Skip to content

Commit 9f2756f

Browse files
authored
[feat](Nereids) support nereids hint position detaction (apache#39113)
When use hint in wrong position or use unsupport hint, use channel(2) to filter it out
1 parent b2b15d3 commit 9f2756f

11 files changed

Lines changed: 280 additions & 68 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,19 @@ lexer grammar DorisLexer;
7272
public void markUnclosedComment() {
7373
has_unclosed_bracketed_comment = true;
7474
}
75+
76+
// This variable will hold the external state
77+
private boolean channel2;
78+
79+
// Method to set the external state
80+
public void setChannel2(boolean value) {
81+
this.channel2 = value;
82+
}
83+
84+
// Method to decide the channel based on external state
85+
private boolean isChannel2() {
86+
return this.channel2;
87+
}
7588
}
7689

7790
SEMICOLON: ';';
@@ -686,6 +699,11 @@ BRACKETED_COMMENT
686699
: '/*' {!isHint()}? ( BRACKETED_COMMENT | . )*? ('*/' | {markUnclosedComment();} EOF) -> channel(HIDDEN)
687700
;
688701

702+
HINT_WITH_CHANNEL
703+
: {isChannel2()}? HINT_START .*? HINT_END -> channel(2)
704+
;
705+
706+
689707
FROM_DUAL
690708
: 'FROM' WS+ 'DUAL' -> channel(HIDDEN);
691709

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 72 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,12 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
514514
return pos1.getCharPositionInLine() - pos2.getCharPositionInLine();
515515
});
516516

517+
private final Map<Integer, ParserRuleContext> selectHintMap;
518+
519+
public LogicalPlanBuilder(Map<Integer, ParserRuleContext> selectHintMap) {
520+
this.selectHintMap = selectHintMap;
521+
}
522+
517523
@SuppressWarnings("unchecked")
518524
protected <T> T typedVisit(ParseTree ctx) {
519525
return (T) ctx.accept(this);
@@ -1352,7 +1358,16 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte
13521358
Optional.ofNullable(ctx.aggClause()),
13531359
Optional.ofNullable(ctx.havingClause()));
13541360
selectPlan = withQueryOrganization(selectPlan, ctx.queryOrganization());
1355-
return withSelectHint(selectPlan, selectCtx.selectHint());
1361+
if ((selectHintMap == null) || selectHintMap.isEmpty()) {
1362+
return selectPlan;
1363+
}
1364+
List<ParserRuleContext> selectHintContexts = Lists.newArrayList();
1365+
for (Integer key : selectHintMap.keySet()) {
1366+
if (key > selectCtx.getStart().getStopIndex() && key < selectCtx.getStop().getStartIndex()) {
1367+
selectHintContexts.add(selectHintMap.get(key));
1368+
}
1369+
}
1370+
return withSelectHint(selectPlan, selectHintContexts);
13561371
});
13571372
}
13581373

@@ -3110,67 +3125,70 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
31103125
return last;
31113126
}
31123127

3113-
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, SelectHintContext hintContext) {
3114-
if (hintContext == null) {
3128+
private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) {
3129+
if (hintContexts.isEmpty()) {
31153130
return logicalPlan;
31163131
}
31173132
Map<String, SelectHint> hints = Maps.newLinkedHashMap();
3118-
for (HintStatementContext hintStatement : hintContext.hintStatements) {
3119-
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3120-
switch (hintName) {
3121-
case "set_var":
3122-
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3123-
for (HintAssignmentContext kv : hintStatement.parameters) {
3124-
if (kv.key != null) {
3125-
String parameterName = visitIdentifierOrText(kv.key);
3126-
Optional<String> value = Optional.empty();
3127-
if (kv.constantValue != null) {
3128-
Literal literal = (Literal) visit(kv.constantValue);
3129-
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3130-
} else if (kv.identifierValue != null) {
3131-
// maybe we should throw exception when the identifierValue is quoted identifier
3132-
value = Optional.ofNullable(kv.identifierValue.getText());
3133+
for (ParserRuleContext hintContext : hintContexts) {
3134+
SelectHintContext selectHintContext = (SelectHintContext) hintContext;
3135+
for (HintStatementContext hintStatement : selectHintContext.hintStatements) {
3136+
String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT);
3137+
switch (hintName) {
3138+
case "set_var":
3139+
Map<String, Optional<String>> parameters = Maps.newLinkedHashMap();
3140+
for (HintAssignmentContext kv : hintStatement.parameters) {
3141+
if (kv.key != null) {
3142+
String parameterName = visitIdentifierOrText(kv.key);
3143+
Optional<String> value = Optional.empty();
3144+
if (kv.constantValue != null) {
3145+
Literal literal = (Literal) visit(kv.constantValue);
3146+
value = Optional.ofNullable(literal.toLegacyLiteral().getStringValue());
3147+
} else if (kv.identifierValue != null) {
3148+
// maybe we should throw exception when the identifierValue is quoted identifier
3149+
value = Optional.ofNullable(kv.identifierValue.getText());
3150+
}
3151+
parameters.put(parameterName, value);
31333152
}
3134-
parameters.put(parameterName, value);
31353153
}
3136-
}
3137-
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
3138-
break;
3139-
case "leading":
3140-
List<String> leadingParameters = new ArrayList<String>();
3141-
for (HintAssignmentContext kv : hintStatement.parameters) {
3142-
if (kv.key != null) {
3143-
String parameterName = visitIdentifierOrText(kv.key);
3144-
leadingParameters.add(parameterName);
3154+
hints.put(hintName, new SelectHintSetVar(hintName, parameters));
3155+
break;
3156+
case "leading":
3157+
List<String> leadingParameters = new ArrayList<String>();
3158+
for (HintAssignmentContext kv : hintStatement.parameters) {
3159+
if (kv.key != null) {
3160+
String parameterName = visitIdentifierOrText(kv.key);
3161+
leadingParameters.add(parameterName);
3162+
}
31453163
}
3146-
}
3147-
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
3148-
break;
3149-
case "ordered":
3150-
hints.put(hintName, new SelectHintOrdered(hintName));
3151-
break;
3152-
case "use_cbo_rule":
3153-
List<String> useRuleParameters = new ArrayList<String>();
3154-
for (HintAssignmentContext kv : hintStatement.parameters) {
3155-
if (kv.key != null) {
3156-
String parameterName = visitIdentifierOrText(kv.key);
3157-
useRuleParameters.add(parameterName);
3164+
hints.put(hintName, new SelectHintLeading(hintName, leadingParameters));
3165+
break;
3166+
case "ordered":
3167+
hints.put(hintName, new SelectHintOrdered(hintName));
3168+
break;
3169+
case "use_cbo_rule":
3170+
List<String> useRuleParameters = new ArrayList<String>();
3171+
for (HintAssignmentContext kv : hintStatement.parameters) {
3172+
if (kv.key != null) {
3173+
String parameterName = visitIdentifierOrText(kv.key);
3174+
useRuleParameters.add(parameterName);
3175+
}
31583176
}
3159-
}
3160-
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
3161-
break;
3162-
case "no_use_cbo_rule":
3163-
List<String> noUseRuleParameters = new ArrayList<String>();
3164-
for (HintAssignmentContext kv : hintStatement.parameters) {
3165-
String parameterName = visitIdentifierOrText(kv.key);
3166-
if (kv.key != null) {
3167-
noUseRuleParameters.add(parameterName);
3177+
hints.put(hintName, new SelectHintUseCboRule(hintName, useRuleParameters, false));
3178+
break;
3179+
case "no_use_cbo_rule":
3180+
List<String> noUseRuleParameters = new ArrayList<String>();
3181+
for (HintAssignmentContext kv : hintStatement.parameters) {
3182+
String parameterName = visitIdentifierOrText(kv.key);
3183+
if (kv.key != null) {
3184+
noUseRuleParameters.add(parameterName);
3185+
}
31683186
}
3169-
}
3170-
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3171-
break;
3172-
default:
3173-
break;
3187+
hints.put(hintName, new SelectHintUseCboRule(hintName, noUseRuleParameters, true));
3188+
break;
3189+
default:
3190+
break;
3191+
}
31743192
}
31753193
}
31763194
return new LogicalSelectHint<>(hints, logicalPlan);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForCreateView.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,15 @@
5252
import com.google.common.collect.ImmutableList;
5353
import org.antlr.v4.runtime.ParserRuleContext;
5454

55+
import java.util.Map;
5556
import java.util.Optional;
5657

5758
/**LogicalPlanBuilderForCreateView*/
5859
public class LogicalPlanBuilderForCreateView extends LogicalPlanBuilder {
60+
public LogicalPlanBuilderForCreateView(Map<Integer, ParserRuleContext> selectHintMap) {
61+
super(selectHintMap);
62+
}
63+
5964
@Override
6065
protected LogicalPlan withGenerate(LogicalPlan plan, LateralViewContext ctx) {
6166
ConnectContext.get().getStatementContext().addIndexInSqlToString(

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForSyncMv.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.doris.nereids.util.PlanUtils;
3838

3939
import com.google.common.collect.ImmutableList;
40+
import org.antlr.v4.runtime.ParserRuleContext;
4041

4142
import java.util.List;
4243
import java.util.Map;
@@ -49,6 +50,10 @@
4950
public class LogicalPlanBuilderForSyncMv extends LogicalPlanBuilder {
5051
private Optional<String> querySql;
5152

53+
public LogicalPlanBuilderForSyncMv(Map<Integer, ParserRuleContext> selectHintMap) {
54+
super(selectHintMap);
55+
}
56+
5257
@Override
5358
public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) {
5459
Expression expression = super.visitFunctionCallExpression(ctx);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.doris.qe.SessionVariable;
3737

3838
import com.google.common.collect.Lists;
39+
import com.google.common.collect.Maps;
3940
import org.antlr.v4.runtime.CharStreams;
4041
import org.antlr.v4.runtime.CommonTokenStream;
4142
import org.antlr.v4.runtime.ParserRuleContext;
@@ -273,26 +274,56 @@ private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
273274
Function<DorisParser, ParserRuleContext> parseFunction) {
274275
ParserRuleContext tree = toAst(sql, parseFunction);
275276
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
276-
? new LogicalPlanBuilder() : logicalPlanBuilder;
277+
? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder;
277278
return (T) realLogicalPlanBuilder.visit(tree);
278279
}
279280

280281
public LogicalPlan parseForCreateView(String sql) {
281282
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
282-
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView();
283+
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
284+
getHintMap(sql, DorisParser::selectHint));
283285
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
284286
}
285287

286288
public Optional<String> parseForSyncMv(String sql) {
287289
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
288-
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv();
290+
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
291+
getHintMap(sql, DorisParser::selectHint));
289292
logicalPlanBuilderForSyncMv.visit(tree);
290293
return logicalPlanBuilderForSyncMv.getQuerySql();
291294
}
292295

296+
/** get hint map */
297+
public static Map<Integer, ParserRuleContext> getHintMap(String sql,
298+
Function<DorisParser, ParserRuleContext> parseFunction) {
299+
// parse hint first round
300+
DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
301+
hintLexer.setChannel2(true);
302+
CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer);
303+
304+
Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();
305+
306+
Token hintToken = hintTokenStream.getTokenSource().nextToken();
307+
while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
308+
int tokenType = hintToken.getType();
309+
if (tokenType == DorisLexer.HINT_WITH_CHANNEL) {
310+
String hintSql = sql.substring(hintToken.getStartIndex(), hintToken.getStopIndex() + 1);
311+
DorisLexer newHintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(hintSql)));
312+
newHintLexer.setChannel2(false);
313+
CommonTokenStream newHintTokenStream = new CommonTokenStream(newHintLexer);
314+
DorisParser hintParser = new DorisParser(newHintTokenStream);
315+
ParserRuleContext hintContext = parseFunction.apply(hintParser);
316+
selectHintMap.put(hintToken.getStartIndex(), hintContext);
317+
}
318+
hintToken = hintTokenStream.getTokenSource().nextToken();
319+
}
320+
return selectHintMap;
321+
}
322+
293323
/** toAst */
294324
public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
295325
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
326+
lexer.setChannel2(true);
296327
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
297328
DorisParser parser = new DorisParser(tokenStream);
298329

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/plsql/PLSqlLogicalPlanBuilder.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@
3636
*/
3737
public class PLSqlLogicalPlanBuilder extends LogicalPlanBuilder {
3838

39+
public PLSqlLogicalPlanBuilder() {
40+
super(null);
41+
}
42+
3943
public List<String> visitMultipartIdentifier(MultipartIdentifierContext ctx) {
4044
return ctx.parts.stream()
4145
.map(RuleContext::getText)

‎fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -352,16 +352,10 @@ public void testJoinHint() {
352352
parsePlan("select * from t1 join [broadcast] t2 on t1.keyy=t2.keyy")
353353
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));
354354

355-
parsePlan("select * from t1 join /*+ broadcast */ t2 on t1.keyy=t2.keyy")
356-
.matches(logicalJoin().when(j -> j.getDistributeHint().distributeType == DistributeType.BROADCAST_RIGHT));
357-
358355
// invalid hint position
359356
parsePlan("select * from [shuffle] t1 join t2 on t1.keyy=t2.keyy")
360357
.assertThrowsExactly(ParseException.class);
361358

362-
parsePlan("select * from /*+ shuffle */ t1 join t2 on t1.keyy=t2.keyy")
363-
.assertThrowsExactly(ParseException.class);
364-
365359
// invalid hint content
366360
parsePlan("select * from t1 join [bucket] t2 on t1.keyy=t2.keyy")
367361
.assertThrowsExactly(ParseException.class)
@@ -372,8 +366,6 @@ public void testJoinHint() {
372366
+ "----------------------^^^");
373367

374368
// invalid multiple hints
375-
parsePlan("select * from t1 join /*+ shuffle , broadcast */ t2 on t1.keyy=t2.keyy")
376-
.assertThrowsExactly(ParseException.class);
377369

378370
parsePlan("select * from t1 join [shuffle,broadcast] t2 on t1.keyy=t2.keyy")
379371
.assertThrowsExactly(ParseException.class);

0 commit comments

Comments
 (0)