Skip to content

Commit 4e188d2

Browse files
committed
[improvement](mtmv) Add validation and improve test coverage for hour-offset partition rollup
### What problem does this PR solve? Issue Number: close apache#62395 Problem Summary: 1. MINVALUE/MAXVALUE partition bounds were not validated, causing potential infinite loop or parse errors 2. Hour offset range was not validated against timezone limits 3. Test coverage needed improvement ### Release note Added validation for MTMV hour-offset partition expressions: - Hour offset must be in range [-14, 14] (timezone limits) - MINVALUE/MAXVALUE partition bounds are rejected with clear error message ### Check List (For Author) - Test: Regression test / Unit Test - Behavior changed: No - Does this need documentation: No
1 parent dbd516f commit 4e188d2

7 files changed

Lines changed: 810 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTruncDateAddSub.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ public MTMVPartitionExprDateTruncDateAddSub(FunctionCallExpr functionCallExpr) t
9797
} else {
9898
throw new AnalysisException("only date_add/date_sub is supported in mtmv partition");
9999
}
100+
// Validate hour offset range based on real-world timezone limits:
101+
// UTC-12 (Baker Island) to UTC+14 (Line Islands, Kiribati)
102+
if (Math.abs(this.offsetHours) > 14) {
103+
throw new AnalysisException(
104+
"hour offset should be in range [-14, 14] (timezone limits), got: " + this.offsetHours);
105+
}
100106
}
101107

102108
@Override
@@ -139,6 +145,14 @@ public List<PartitionKeyDesc> generateRollUpPartitionKeyDescs(PartitionKeyDesc p
139145
Preconditions.checkState(partitionKeyDesc.getLowerValues().size() == 1,
140146
"only support one partition column");
141147

148+
// Validate that partition bounds are concrete datetime values, not MINVALUE/MAXVALUE
149+
String lowerStr = partitionKeyDesc.getLowerValues().get(0).getStringValue();
150+
String upperStr = partitionKeyDesc.getUpperValues().get(0).getStringValue();
151+
if ("MINVALUE".equalsIgnoreCase(lowerStr) || "MAXVALUE".equalsIgnoreCase(upperStr)) {
152+
throw new AnalysisException(
153+
"date_trunc + date_add/date_sub does not support MINVALUE/MAXVALUE partition bounds");
154+
}
155+
142156
DateTimeV2Literal beginBucket = dateTruncByOffset(
143157
partitionKeyDesc.getLowerValues().get(0).getStringValue(), Optional.empty());
144158

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.doris.mtmv.BaseTableInfo;
3535
import org.apache.doris.mtmv.MTMVPartitionExprDateTruncDateAddSub;
3636
import org.apache.doris.mtmv.MTMVPartitionExprFactory;
37+
import org.apache.doris.mtmv.MTMVPartitionExprService;
3738
import org.apache.doris.mtmv.MTMVRelatedTableIf;
3839
import org.apache.doris.nereids.StatementContext;
3940
import org.apache.doris.nereids.analyzer.UnboundRelation;
@@ -148,8 +149,8 @@ private static Map<TableIf, Set<Expression>> constructTableWithPredicates(MTMV m
148149
long hourOffset = 0;
149150
try {
150151
if (mv.getMvPartitionInfo() != null && mv.getMvPartitionInfo().getExpr() != null) {
151-
MTMVPartitionExprFactory.getExprService(mv.getMvPartitionInfo().getExpr());
152-
Object service = MTMVPartitionExprFactory.getExprService(mv.getMvPartitionInfo().getExpr());
152+
MTMVPartitionExprService service =
153+
MTMVPartitionExprFactory.getExprService(mv.getMvPartitionInfo().getExpr());
153154
if (service instanceof MTMVPartitionExprDateTruncDateAddSub) {
154155
hourOffset = ((MTMVPartitionExprDateTruncDateAddSub) service).getOffsetHours();
155156
}

‎fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,189 @@ public void testConstructorRejectsNonArithmeticArg() {
719719
}
720720
}
721721

722+
// =========================================================================
723+
// New tests — toSql(), dateFormat fallback, getOffsetHours
724+
// =========================================================================
725+
726+
@Test
727+
public void testToSqlReturnsExprString() throws AnalysisException {
728+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
729+
Lists.newArrayList(
730+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(3), "HOUR"),
731+
new StringLiteral("day")),
732+
true);
733+
MTMVPartitionExprDateTruncDateAddSub service = new MTMVPartitionExprDateTruncDateAddSub(expr);
734+
735+
MTMVPartitionInfo partitionInfo = Mockito.mock(MTMVPartitionInfo.class);
736+
Mockito.when(partitionInfo.getExpr()).thenReturn(expr);
737+
738+
String sql = service.toSql(partitionInfo);
739+
Assert.assertNotNull(sql);
740+
Assert.assertTrue("toSql should mention date_trunc: " + sql,
741+
sql.toLowerCase().contains("date_trunc"));
742+
}
743+
744+
@Test
745+
public void testGetRollUpIdentityWithDateFormat() throws AnalysisException {
746+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
747+
Lists.newArrayList(
748+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(3), "HOUR"),
749+
new StringLiteral("day")),
750+
true);
751+
MTMVPartitionExprDateTruncDateAddSub service = new MTMVPartitionExprDateTruncDateAddSub(expr);
752+
753+
// Use a non-standard date format that requires strToDate fallback
754+
Map<String, String> props = Maps.newHashMap();
755+
props.put(org.apache.doris.common.util.PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT, "%d/%m/%Y");
756+
PartitionKeyDesc inDesc = generateInDesc("24/07/2025");
757+
String identity = service.getRollUpIdentity(inDesc, props);
758+
Assert.assertEquals("2025-07-24 00:00:00", identity);
759+
}
760+
761+
@Test
762+
public void testConstructorRejectsNonLiteralOffset() {
763+
FunctionCallExpr exprBadParamCount = new FunctionCallExpr("date_trunc",
764+
Lists.newArrayList(new StringLiteral("day")),
765+
true);
766+
try {
767+
new MTMVPartitionExprDateTruncDateAddSub(exprBadParamCount);
768+
Assert.fail("Expected AnalysisException for wrong param count");
769+
} catch (org.apache.doris.common.AnalysisException e) {
770+
Assert.assertTrue(e.getMessage().contains("size should be 2"));
771+
}
772+
}
773+
774+
@Test
775+
public void testConstructorRejectsNonStringLiteralUnit() {
776+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
777+
Lists.newArrayList(
778+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(3), "HOUR"),
779+
new IntLiteral(1)),
780+
true);
781+
try {
782+
new MTMVPartitionExprDateTruncDateAddSub(expr);
783+
Assert.fail("Expected AnalysisException for non-string-literal unit");
784+
} catch (org.apache.doris.common.AnalysisException e) {
785+
Assert.assertTrue(e.getMessage().contains("not string literal"));
786+
}
787+
}
788+
789+
@Test
790+
public void testGetOffsetHoursPositive() throws AnalysisException {
791+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
792+
Lists.newArrayList(
793+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(8), "HOUR"),
794+
new StringLiteral("day")),
795+
true);
796+
MTMVPartitionExprDateTruncDateAddSub service = new MTMVPartitionExprDateTruncDateAddSub(expr);
797+
Assert.assertEquals(8, service.getOffsetHours());
798+
}
799+
800+
@Test
801+
public void testGetOffsetHoursNegative() throws AnalysisException {
802+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
803+
Lists.newArrayList(
804+
new TimestampArithmeticExpr("date_sub", new SlotRef(null, null), new IntLiteral(5), "HOUR"),
805+
new StringLiteral("day")),
806+
true);
807+
MTMVPartitionExprDateTruncDateAddSub service = new MTMVPartitionExprDateTruncDateAddSub(expr);
808+
Assert.assertEquals(-5, service.getOffsetHours());
809+
}
810+
811+
@Test
812+
public void testConstructorRejectsOffsetBeyondTimezoneLimit() {
813+
// Hour offset should be in range [-14, 14] based on real-world timezone limits
814+
// UTC-12 (Baker Island) to UTC+14 (Line Islands, Kiribati)
815+
816+
// Test +15 hours (exceeds +14 limit)
817+
FunctionCallExpr exprPositive = new FunctionCallExpr("date_trunc",
818+
Lists.newArrayList(
819+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(15), "HOUR"),
820+
new StringLiteral("day")),
821+
true);
822+
try {
823+
new MTMVPartitionExprDateTruncDateAddSub(exprPositive);
824+
Assert.fail("Expected AnalysisException for offset > 14");
825+
} catch (org.apache.doris.common.AnalysisException e) {
826+
Assert.assertTrue(e.getMessage().contains("[-14, 14]"));
827+
Assert.assertTrue(e.getMessage().contains("15"));
828+
}
829+
830+
// Test -15 hours (exceeds -14 limit)
831+
FunctionCallExpr exprNegative = new FunctionCallExpr("date_trunc",
832+
Lists.newArrayList(
833+
new TimestampArithmeticExpr("date_sub", new SlotRef(null, null), new IntLiteral(15), "HOUR"),
834+
new StringLiteral("day")),
835+
true);
836+
try {
837+
new MTMVPartitionExprDateTruncDateAddSub(exprNegative);
838+
Assert.fail("Expected AnalysisException for offset < -14");
839+
} catch (org.apache.doris.common.AnalysisException e) {
840+
Assert.assertTrue(e.getMessage().contains("[-14, 14]"));
841+
Assert.assertTrue(e.getMessage().contains("-15"));
842+
}
843+
}
844+
845+
@Test
846+
public void testConstructorAcceptsBoundaryOffsets() throws AnalysisException {
847+
// +14 hours should be accepted (UTC+14 - Line Islands)
848+
FunctionCallExpr expr14 = new FunctionCallExpr("date_trunc",
849+
Lists.newArrayList(
850+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(14), "HOUR"),
851+
new StringLiteral("day")),
852+
true);
853+
MTMVPartitionExprDateTruncDateAddSub service14 = new MTMVPartitionExprDateTruncDateAddSub(expr14);
854+
Assert.assertEquals(14, service14.getOffsetHours());
855+
856+
// -14 hours should be accepted (beyond UTC-12 but symmetric limit)
857+
FunctionCallExpr exprMinus14 = new FunctionCallExpr("date_trunc",
858+
Lists.newArrayList(
859+
new TimestampArithmeticExpr("date_sub", new SlotRef(null, null), new IntLiteral(14), "HOUR"),
860+
new StringLiteral("day")),
861+
true);
862+
MTMVPartitionExprDateTruncDateAddSub serviceMinus14 = new MTMVPartitionExprDateTruncDateAddSub(exprMinus14);
863+
Assert.assertEquals(-14, serviceMinus14.getOffsetHours());
864+
}
865+
866+
@Test
867+
public void testRejectsMinValueMaxValuePartitionBounds() throws AnalysisException {
868+
// Create service with +3h offset
869+
FunctionCallExpr expr = new FunctionCallExpr("date_trunc",
870+
Lists.newArrayList(
871+
new TimestampArithmeticExpr("date_add", new SlotRef(null, null), new IntLiteral(3), "HOUR"),
872+
new StringLiteral("day")),
873+
true);
874+
MTMVPartitionExprDateTruncDateAddSub service = new MTMVPartitionExprDateTruncDateAddSub(expr);
875+
876+
try (MockedStatic<MTMVPartitionUtil> mock = Mockito.mockStatic(MTMVPartitionUtil.class)) {
877+
mock.when(() -> MTMVPartitionUtil.getPartitionColumnType(
878+
Mockito.nullable(MTMVRelatedTableIf.class), Mockito.nullable(String.class)))
879+
.thenReturn(Type.DATETIMEV2);
880+
881+
// MINVALUE lower bound should be rejected
882+
PartitionKeyDesc minValueDesc = PartitionKeyDesc.createFixed(
883+
Lists.newArrayList(new PartitionValue("MINVALUE", true)),
884+
Lists.newArrayList(new PartitionValue("2025-01-01 00:00:00")));
885+
try {
886+
service.generateRollUpPartitionKeyDescs(minValueDesc, mtmvPartitionInfo, null);
887+
Assert.fail("Expected AnalysisException for MINVALUE partition bound");
888+
} catch (org.apache.doris.common.AnalysisException e) {
889+
Assert.assertTrue(e.getMessage().contains("MINVALUE/MAXVALUE"));
890+
}
891+
892+
// MAXVALUE upper bound should be rejected
893+
PartitionKeyDesc maxValueDesc = PartitionKeyDesc.createFixed(
894+
Lists.newArrayList(new PartitionValue("2025-01-01 00:00:00")),
895+
Lists.newArrayList(new PartitionValue("MAXVALUE", true)));
896+
try {
897+
service.generateRollUpPartitionKeyDescs(maxValueDesc, mtmvPartitionInfo, null);
898+
Assert.fail("Expected AnalysisException for MAXVALUE partition bound");
899+
} catch (org.apache.doris.common.AnalysisException e) {
900+
Assert.assertTrue(e.getMessage().contains("MINVALUE/MAXVALUE"));
901+
}
902+
}
903+
}
904+
722905
// =========================================================================
723906
// Helper
724907
// =========================================================================

0 commit comments

Comments
 (0)