Skip to content

Commit 10b72d4

Browse files
morningmanclaude
andcommitted
[feat](connector) [P3-T05] real EQ/IN partition pruning in hudi applyFilter
HudiConnectorMetadata.applyFilter was a stub: for any partitioned table it listed ALL partition names from HMS, ignored the predicate, and unconditionally set prunedPartitionPaths. Two problems: - no EQ/IN partition pruning -> every partition scanned; - setting prunedPartitionPaths unconditionally short-circuits HudiScanPlanProvider.resolvePartitions, silently switching the partition source from Hudi metadata (getAllPartitionPaths) to HMS for any filtered query -- an undeclared behavior divergence from the no-filter path. applyFilter now mirrors HiveConnectorMetadata: extract EQ/IN predicates on partition columns from the expression tree, list candidate partition names, prune them, and return a pruned handle only when pruning has effect; otherwise Optional.empty() (handle untouched -> resolvePartitions falls back to Hudi's own listing). Keeps Hudi's List<String> path representation (resolvePartitions feeds paths to the FileSystemView, not HmsPartitionInfo) and maxParts=-1 (no silent truncation, strictly safer than Hive's 100000 cap). The 7 pruning helpers are duplicated from Hive because fe-connector-hudi depends on fe-connector-hms, not fe-connector-hive; consolidate during the P7 Hive migration (same precedent as T02's toHiveTypeString). Test: HudiPartitionPruningTest -- 8 cases (EQ/IN/AND prune, non-partition predicate ignored, predicate matching all/none, unpartitioned). Module test 19 green, checkstyle 0 violations, import-gate pass. Dormant branch behind the closed gate (SPI_READY_TYPES excludes hms/hudi), zero live caller -> zero live-path risk. listPartitions* override and MVCC (T06) deferred per DV-007; full snapshot/incremental support stays in batch E. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 301fe38 commit 10b72d4

3 files changed

Lines changed: 540 additions & 6 deletions

File tree

‎fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java‎

Lines changed: 143 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@
2424
import org.apache.doris.connector.api.ConnectorType;
2525
import org.apache.doris.connector.api.handle.ConnectorColumnHandle;
2626
import org.apache.doris.connector.api.handle.ConnectorTableHandle;
27+
import org.apache.doris.connector.api.pushdown.ConnectorAnd;
28+
import org.apache.doris.connector.api.pushdown.ConnectorComparison;
29+
import org.apache.doris.connector.api.pushdown.ConnectorExpression;
2730
import org.apache.doris.connector.api.pushdown.ConnectorFilterConstraint;
31+
import org.apache.doris.connector.api.pushdown.ConnectorIn;
32+
import org.apache.doris.connector.api.pushdown.ConnectorLiteral;
2833
import org.apache.doris.connector.api.pushdown.FilterApplicationResult;
2934
import org.apache.doris.connector.hms.HmsClient;
3035
import org.apache.doris.connector.hms.HmsClientException;
@@ -39,10 +44,12 @@
3944

4045
import java.util.ArrayList;
4146
import java.util.Collections;
47+
import java.util.HashMap;
4248
import java.util.LinkedHashMap;
4349
import java.util.List;
4450
import java.util.Map;
4551
import java.util.Optional;
52+
import java.util.Set;
4653
import java.util.stream.Collectors;
4754

4855
/**
@@ -150,17 +157,38 @@ public Optional<FilterApplicationResult<ConnectorTableHandle>> applyFilter(
150157
return Optional.empty();
151158
}
152159

153-
// List all partition names from HMS (e.g. "year=2024/month=01")
154-
// These are relative paths that double as partition identifiers
155-
List<String> partitionNames = hmsClient.listPartitionNames(
160+
// Extract equality/IN predicates on partition columns from the expression.
161+
// No partition predicate -> leave the handle untouched so resolvePartitions
162+
// falls back to Hudi's own metadata listing (HoodieTableMetadata.getAllPartitionPaths).
163+
Map<String, List<String>> partitionPredicates = extractPartitionPredicates(
164+
constraint.getExpression(), partKeyNames);
165+
if (partitionPredicates.isEmpty()) {
166+
return Optional.empty();
167+
}
168+
169+
// List candidate partition names from HMS (e.g. "year=2024/month=01"). These
170+
// relative paths double as partition identifiers consumed by HudiScanPlanProvider.
171+
// Keep maxParts=-1 (unlimited): no silent partition truncation.
172+
List<String> allPartNames = hmsClient.listPartitionNames(
156173
hudiHandle.getDbName(), hudiHandle.getTableName(), -1);
157-
if (partitionNames == null || partitionNames.isEmpty()) {
174+
if (allPartNames == null || allPartNames.isEmpty()) {
175+
return Optional.empty();
176+
}
177+
178+
List<String> matchedPartNames = prunePartitionNames(
179+
allPartNames, partKeyNames, partitionPredicates);
180+
if (matchedPartNames.size() == allPartNames.size()) {
181+
// No pruning effect
158182
return Optional.empty();
159183
}
160184

161-
// Build updated handle with partition paths for scan planning
185+
LOG.info("Partition pruning: {}.{} all={} pruned={}",
186+
hudiHandle.getDbName(), hudiHandle.getTableName(),
187+
allPartNames.size(), matchedPartNames.size());
188+
189+
// Build updated handle carrying only the matched partition paths for scan planning.
162190
HudiTableHandle updatedHandle = hudiHandle.toBuilder()
163-
.prunedPartitionPaths(partitionNames)
191+
.prunedPartitionPaths(matchedPartNames)
164192
.build();
165193

166194
return Optional.of(new FilterApplicationResult<>(updatedHandle, constraint.getExpression(), false));
@@ -303,4 +331,113 @@ private Configuration buildHadoopConf() {
303331
}
304332
return conf;
305333
}
334+
335+
// ========== Partition pruning helpers ==========
336+
// Mirrors HiveConnectorMetadata's EQ/IN partition pruning. Duplicated rather than
337+
// shared because fe-connector-hudi depends on fe-connector-hms, not fe-connector-hive;
338+
// consolidate during the Hive (P7) migration. See P3-T05 design.
339+
340+
/**
341+
* Extracts equality predicates on partition columns from the expression tree.
342+
* Supports: col = 'value', col IN ('v1', 'v2', ...), AND combinations.
343+
*/
344+
private Map<String, List<String>> extractPartitionPredicates(
345+
ConnectorExpression expr, List<String> partKeyNames) {
346+
Set<String> partKeySet = partKeyNames.stream().collect(Collectors.toSet());
347+
Map<String, List<String>> result = new HashMap<>();
348+
extractPredicatesRecursive(expr, partKeySet, result);
349+
return result;
350+
}
351+
352+
private void extractPredicatesRecursive(ConnectorExpression expr,
353+
Set<String> partKeySet, Map<String, List<String>> result) {
354+
if (expr instanceof ConnectorAnd) {
355+
for (ConnectorExpression child : ((ConnectorAnd) expr).getConjuncts()) {
356+
extractPredicatesRecursive(child, partKeySet, result);
357+
}
358+
} else if (expr instanceof ConnectorComparison) {
359+
ConnectorComparison cmp = (ConnectorComparison) expr;
360+
if (cmp.getOperator() == ConnectorComparison.Operator.EQ) {
361+
String colName = extractColumnName(cmp.getLeft());
362+
String value = extractLiteralValue(cmp.getRight());
363+
if (colName != null && value != null && partKeySet.contains(colName)) {
364+
result.computeIfAbsent(colName, k -> new ArrayList<>()).add(value);
365+
}
366+
}
367+
} else if (expr instanceof ConnectorIn) {
368+
ConnectorIn inExpr = (ConnectorIn) expr;
369+
if (!inExpr.isNegated()) {
370+
String colName = extractColumnName(inExpr.getValue());
371+
if (colName != null && partKeySet.contains(colName)) {
372+
List<String> values = new ArrayList<>();
373+
for (ConnectorExpression item : inExpr.getInList()) {
374+
String val = extractLiteralValue(item);
375+
if (val != null) {
376+
values.add(val);
377+
}
378+
}
379+
if (!values.isEmpty()) {
380+
result.computeIfAbsent(colName, k -> new ArrayList<>()).addAll(values);
381+
}
382+
}
383+
}
384+
}
385+
}
386+
387+
private String extractColumnName(ConnectorExpression expr) {
388+
if (expr instanceof org.apache.doris.connector.api.pushdown.ConnectorColumnRef) {
389+
return ((org.apache.doris.connector.api.pushdown.ConnectorColumnRef) expr).getColumnName();
390+
}
391+
return null;
392+
}
393+
394+
private String extractLiteralValue(ConnectorExpression expr) {
395+
if (expr instanceof ConnectorLiteral) {
396+
Object val = ((ConnectorLiteral) expr).getValue();
397+
return val != null ? String.valueOf(val) : null;
398+
}
399+
return null;
400+
}
401+
402+
/**
403+
* Prunes partition names based on extracted equality predicates.
404+
* Partition names follow the Hive convention: key1=val1/key2=val2
405+
*/
406+
private List<String> prunePartitionNames(List<String> allPartNames,
407+
List<String> partKeyNames, Map<String, List<String>> predicates) {
408+
List<String> matched = new ArrayList<>();
409+
for (String partName : allPartNames) {
410+
Map<String, String> partValues = parsePartitionName(partName, partKeyNames);
411+
if (matchesPredicates(partValues, predicates)) {
412+
matched.add(partName);
413+
}
414+
}
415+
return matched;
416+
}
417+
418+
private Map<String, String> parsePartitionName(String partName,
419+
List<String> partKeyNames) {
420+
Map<String, String> values = new HashMap<>();
421+
String[] parts = partName.split("/");
422+
for (String part : parts) {
423+
int eq = part.indexOf('=');
424+
if (eq > 0) {
425+
values.put(part.substring(0, eq), part.substring(eq + 1));
426+
}
427+
}
428+
return values;
429+
}
430+
431+
private boolean matchesPredicates(Map<String, String> partValues,
432+
Map<String, List<String>> predicates) {
433+
for (Map.Entry<String, List<String>> entry : predicates.entrySet()) {
434+
String colName = entry.getKey();
435+
List<String> allowedValues = entry.getValue();
436+
String actualValue = partValues.get(colName);
437+
if (actualValue == null || !allowedValues.contains(actualValue)) {
438+
return false;
439+
}
440+
}
441+
return true;
442+
}
306443
}

0 commit comments

Comments
 (0)