|
24 | 24 | import org.apache.doris.connector.api.ConnectorType; |
25 | 25 | import org.apache.doris.connector.api.handle.ConnectorColumnHandle; |
26 | 26 | import org.apache.doris.connector.api.handle.ConnectorTableHandle; |
| 27 | +import org.apache.doris.connector.api.pushdown.ConnectorAnd; |
| 28 | +import org.apache.doris.connector.api.pushdown.ConnectorComparison; |
| 29 | +import org.apache.doris.connector.api.pushdown.ConnectorExpression; |
27 | 30 | import org.apache.doris.connector.api.pushdown.ConnectorFilterConstraint; |
| 31 | +import org.apache.doris.connector.api.pushdown.ConnectorIn; |
| 32 | +import org.apache.doris.connector.api.pushdown.ConnectorLiteral; |
28 | 33 | import org.apache.doris.connector.api.pushdown.FilterApplicationResult; |
29 | 34 | import org.apache.doris.connector.hms.HmsClient; |
30 | 35 | import org.apache.doris.connector.hms.HmsClientException; |
|
39 | 44 |
|
40 | 45 | import java.util.ArrayList; |
41 | 46 | import java.util.Collections; |
| 47 | +import java.util.HashMap; |
42 | 48 | import java.util.LinkedHashMap; |
43 | 49 | import java.util.List; |
44 | 50 | import java.util.Map; |
45 | 51 | import java.util.Optional; |
| 52 | +import java.util.Set; |
46 | 53 | import java.util.stream.Collectors; |
47 | 54 |
|
48 | 55 | /** |
@@ -150,17 +157,38 @@ public Optional<FilterApplicationResult<ConnectorTableHandle>> applyFilter( |
150 | 157 | return Optional.empty(); |
151 | 158 | } |
152 | 159 |
|
153 | | - // List all partition names from HMS (e.g. "year=2024/month=01") |
154 | | - // These are relative paths that double as partition identifiers |
155 | | - List<String> partitionNames = hmsClient.listPartitionNames( |
| 160 | + // Extract equality/IN predicates on partition columns from the expression. |
| 161 | + // No partition predicate -> leave the handle untouched so resolvePartitions |
| 162 | + // falls back to Hudi's own metadata listing (HoodieTableMetadata.getAllPartitionPaths). |
| 163 | + Map<String, List<String>> partitionPredicates = extractPartitionPredicates( |
| 164 | + constraint.getExpression(), partKeyNames); |
| 165 | + if (partitionPredicates.isEmpty()) { |
| 166 | + return Optional.empty(); |
| 167 | + } |
| 168 | + |
| 169 | + // List candidate partition names from HMS (e.g. "year=2024/month=01"). These |
| 170 | + // relative paths double as partition identifiers consumed by HudiScanPlanProvider. |
| 171 | + // Keep maxParts=-1 (unlimited): no silent partition truncation. |
| 172 | + List<String> allPartNames = hmsClient.listPartitionNames( |
156 | 173 | hudiHandle.getDbName(), hudiHandle.getTableName(), -1); |
157 | | - if (partitionNames == null || partitionNames.isEmpty()) { |
| 174 | + if (allPartNames == null || allPartNames.isEmpty()) { |
| 175 | + return Optional.empty(); |
| 176 | + } |
| 177 | + |
| 178 | + List<String> matchedPartNames = prunePartitionNames( |
| 179 | + allPartNames, partKeyNames, partitionPredicates); |
| 180 | + if (matchedPartNames.size() == allPartNames.size()) { |
| 181 | + // No pruning effect |
158 | 182 | return Optional.empty(); |
159 | 183 | } |
160 | 184 |
|
161 | | - // Build updated handle with partition paths for scan planning |
| 185 | + LOG.info("Partition pruning: {}.{} all={} pruned={}", |
| 186 | + hudiHandle.getDbName(), hudiHandle.getTableName(), |
| 187 | + allPartNames.size(), matchedPartNames.size()); |
| 188 | + |
| 189 | + // Build updated handle carrying only the matched partition paths for scan planning. |
162 | 190 | HudiTableHandle updatedHandle = hudiHandle.toBuilder() |
163 | | - .prunedPartitionPaths(partitionNames) |
| 191 | + .prunedPartitionPaths(matchedPartNames) |
164 | 192 | .build(); |
165 | 193 |
|
166 | 194 | return Optional.of(new FilterApplicationResult<>(updatedHandle, constraint.getExpression(), false)); |
@@ -303,4 +331,113 @@ private Configuration buildHadoopConf() { |
303 | 331 | } |
304 | 332 | return conf; |
305 | 333 | } |
| 334 | + |
| 335 | + // ========== Partition pruning helpers ========== |
| 336 | + // Mirrors HiveConnectorMetadata's EQ/IN partition pruning. Duplicated rather than |
| 337 | + // shared because fe-connector-hudi depends on fe-connector-hms, not fe-connector-hive; |
| 338 | + // consolidate during the Hive (P7) migration. See P3-T05 design. |
| 339 | + |
| 340 | + /** |
| 341 | + * Extracts equality predicates on partition columns from the expression tree. |
| 342 | + * Supports: col = 'value', col IN ('v1', 'v2', ...), AND combinations. |
| 343 | + */ |
| 344 | + private Map<String, List<String>> extractPartitionPredicates( |
| 345 | + ConnectorExpression expr, List<String> partKeyNames) { |
| 346 | + Set<String> partKeySet = partKeyNames.stream().collect(Collectors.toSet()); |
| 347 | + Map<String, List<String>> result = new HashMap<>(); |
| 348 | + extractPredicatesRecursive(expr, partKeySet, result); |
| 349 | + return result; |
| 350 | + } |
| 351 | + |
| 352 | + private void extractPredicatesRecursive(ConnectorExpression expr, |
| 353 | + Set<String> partKeySet, Map<String, List<String>> result) { |
| 354 | + if (expr instanceof ConnectorAnd) { |
| 355 | + for (ConnectorExpression child : ((ConnectorAnd) expr).getConjuncts()) { |
| 356 | + extractPredicatesRecursive(child, partKeySet, result); |
| 357 | + } |
| 358 | + } else if (expr instanceof ConnectorComparison) { |
| 359 | + ConnectorComparison cmp = (ConnectorComparison) expr; |
| 360 | + if (cmp.getOperator() == ConnectorComparison.Operator.EQ) { |
| 361 | + String colName = extractColumnName(cmp.getLeft()); |
| 362 | + String value = extractLiteralValue(cmp.getRight()); |
| 363 | + if (colName != null && value != null && partKeySet.contains(colName)) { |
| 364 | + result.computeIfAbsent(colName, k -> new ArrayList<>()).add(value); |
| 365 | + } |
| 366 | + } |
| 367 | + } else if (expr instanceof ConnectorIn) { |
| 368 | + ConnectorIn inExpr = (ConnectorIn) expr; |
| 369 | + if (!inExpr.isNegated()) { |
| 370 | + String colName = extractColumnName(inExpr.getValue()); |
| 371 | + if (colName != null && partKeySet.contains(colName)) { |
| 372 | + List<String> values = new ArrayList<>(); |
| 373 | + for (ConnectorExpression item : inExpr.getInList()) { |
| 374 | + String val = extractLiteralValue(item); |
| 375 | + if (val != null) { |
| 376 | + values.add(val); |
| 377 | + } |
| 378 | + } |
| 379 | + if (!values.isEmpty()) { |
| 380 | + result.computeIfAbsent(colName, k -> new ArrayList<>()).addAll(values); |
| 381 | + } |
| 382 | + } |
| 383 | + } |
| 384 | + } |
| 385 | + } |
| 386 | + |
| 387 | + private String extractColumnName(ConnectorExpression expr) { |
| 388 | + if (expr instanceof org.apache.doris.connector.api.pushdown.ConnectorColumnRef) { |
| 389 | + return ((org.apache.doris.connector.api.pushdown.ConnectorColumnRef) expr).getColumnName(); |
| 390 | + } |
| 391 | + return null; |
| 392 | + } |
| 393 | + |
| 394 | + private String extractLiteralValue(ConnectorExpression expr) { |
| 395 | + if (expr instanceof ConnectorLiteral) { |
| 396 | + Object val = ((ConnectorLiteral) expr).getValue(); |
| 397 | + return val != null ? String.valueOf(val) : null; |
| 398 | + } |
| 399 | + return null; |
| 400 | + } |
| 401 | + |
| 402 | + /** |
| 403 | + * Prunes partition names based on extracted equality predicates. |
| 404 | + * Partition names follow the Hive convention: key1=val1/key2=val2 |
| 405 | + */ |
| 406 | + private List<String> prunePartitionNames(List<String> allPartNames, |
| 407 | + List<String> partKeyNames, Map<String, List<String>> predicates) { |
| 408 | + List<String> matched = new ArrayList<>(); |
| 409 | + for (String partName : allPartNames) { |
| 410 | + Map<String, String> partValues = parsePartitionName(partName, partKeyNames); |
| 411 | + if (matchesPredicates(partValues, predicates)) { |
| 412 | + matched.add(partName); |
| 413 | + } |
| 414 | + } |
| 415 | + return matched; |
| 416 | + } |
| 417 | + |
| 418 | + private Map<String, String> parsePartitionName(String partName, |
| 419 | + List<String> partKeyNames) { |
| 420 | + Map<String, String> values = new HashMap<>(); |
| 421 | + String[] parts = partName.split("/"); |
| 422 | + for (String part : parts) { |
| 423 | + int eq = part.indexOf('='); |
| 424 | + if (eq > 0) { |
| 425 | + values.put(part.substring(0, eq), part.substring(eq + 1)); |
| 426 | + } |
| 427 | + } |
| 428 | + return values; |
| 429 | + } |
| 430 | + |
| 431 | + private boolean matchesPredicates(Map<String, String> partValues, |
| 432 | + Map<String, List<String>> predicates) { |
| 433 | + for (Map.Entry<String, List<String>> entry : predicates.entrySet()) { |
| 434 | + String colName = entry.getKey(); |
| 435 | + List<String> allowedValues = entry.getValue(); |
| 436 | + String actualValue = partValues.get(colName); |
| 437 | + if (actualValue == null || !allowedValues.contains(actualValue)) { |
| 438 | + return false; |
| 439 | + } |
| 440 | + } |
| 441 | + return true; |
| 442 | + } |
306 | 443 | } |
0 commit comments