|
71 | 71 | import com.google.common.collect.Maps; |
72 | 72 | import com.google.common.collect.Sets; |
73 | 73 |
|
| 74 | +import java.util.ArrayList; |
74 | 75 | import java.util.Arrays; |
75 | 76 | import java.util.Collections; |
76 | 77 | import java.util.HashSet; |
| 78 | +import java.util.LinkedHashMap; |
77 | 79 | import java.util.List; |
78 | 80 | import java.util.Map; |
79 | 81 | import java.util.Objects; |
@@ -440,53 +442,78 @@ public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOper |
440 | 442 | return PhysicalProperties.GATHER; |
441 | 443 | } |
442 | 444 |
|
443 | | - // TODO: open comment when support `enable_local_shuffle_planner` |
444 | | - // int distributeToChildIndex |
445 | | - // = setOperation.<Integer>getMutableState(PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX).orElse(-1); |
446 | | - // if (distributeToChildIndex >= 0 |
447 | | - // && childrenDistribution.get(distributeToChildIndex) instanceof DistributionSpecHash) { |
448 | | - // DistributionSpecHash childDistribution |
449 | | - // = (DistributionSpecHash) childrenDistribution.get(distributeToChildIndex); |
450 | | - // List<SlotReference> childToIndex = setOperation.getRegularChildrenOutputs().get(distributeToChildIndex); |
451 | | - // Map<ExprId, Integer> idToOutputIndex = new LinkedHashMap<>(); |
452 | | - // for (int j = 0; j < childToIndex.size(); j++) { |
453 | | - // idToOutputIndex.put(childToIndex.get(j).getExprId(), j); |
454 | | - // } |
455 | | - // |
456 | | - // List<ExprId> orderedShuffledColumns = childDistribution.getOrderedShuffledColumns(); |
457 | | - // List<ExprId> setOperationDistributeColumnIds = new ArrayList<>(); |
458 | | - // for (ExprId tableDistributeColumnId : orderedShuffledColumns) { |
459 | | - // Integer index = idToOutputIndex.get(tableDistributeColumnId); |
460 | | - // if (index == null) { |
461 | | - // break; |
462 | | - // } |
463 | | - // setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId()); |
464 | | - // } |
465 | | - // // check whether the set operation output all distribution columns of the child |
466 | | - // if (setOperationDistributeColumnIds.size() == orderedShuffledColumns.size()) { |
467 | | - // boolean isUnion = setOperation instanceof Union; |
468 | | - // boolean shuffleToRight = distributeToChildIndex > 0; |
469 | | - // if (!isUnion && shuffleToRight) { |
470 | | - // return new PhysicalProperties( |
471 | | - // new DistributionSpecHash( |
472 | | - // setOperationDistributeColumnIds, |
473 | | - // ShuffleType.EXECUTION_BUCKETED |
474 | | - // ) |
475 | | - // ); |
476 | | - // } else { |
477 | | - // // keep the distribution as the child |
478 | | - // return new PhysicalProperties( |
479 | | - // new DistributionSpecHash( |
480 | | - // setOperationDistributeColumnIds, |
481 | | - // childDistribution.getShuffleType(), |
482 | | - // childDistribution.getTableId(), |
483 | | - // childDistribution.getSelectedIndexId(), |
484 | | - // childDistribution.getPartitionIds() |
485 | | - // ) |
486 | | - // ); |
487 | | - // } |
488 | | - // } |
489 | | - // } |
| 445 | + // After ChildrenPropertiesRegulator the children distributions are already legal, so the |
| 446 | + // output is derived by describing what the children provide: |
| 447 | + // 1. one or more NATURAL children: output the first NATURAL child's distribution |
| 448 | + // (several NATURAL children behave like colocate); |
| 449 | + // 2. no NATURAL but some STORAGE_BUCKETED child: output its STORAGE_BUCKETED distribution; |
| 450 | + // 3. all EXECUTION_BUCKETED children: output the execution hash (the generic loop below); |
| 451 | + // 4. anything else (e.g. random): output a non-specific property (also below). |
| 452 | + // When the basic child does not directly output its shuffle columns, or the children do |
| 453 | + // not agree, this falls through to the generic loop below, which is equivalence-set aware |
| 454 | + // and checks that every child maps its shuffle columns to the same set-operation output |
| 455 | + // positions before it claims a bucketed output. The basic child is recomputed from the |
| 456 | + // children distributions instead of being carried as mutable planner state, because mutable |
| 457 | + // state does not survive the with-copies in chooseBestPlan() and the |
| 458 | + // RecomputePhysicalPropertiesPostProcessor re-derivation, while this recomputation is |
| 459 | + // deterministic on any copy of the plan. |
| 460 | + int distributeToChildIndex = -1; |
| 461 | + int firstStorageBucketedIndex = -1; |
| 462 | + for (int i = 0; i < childrenDistribution.size(); i++) { |
| 463 | + if (childrenDistribution.get(i) instanceof DistributionSpecHash) { |
| 464 | + ShuffleType childShuffleType |
| 465 | + = ((DistributionSpecHash) childrenDistribution.get(i)).getShuffleType(); |
| 466 | + if (childShuffleType == ShuffleType.NATURAL) { |
| 467 | + distributeToChildIndex = i; |
| 468 | + break; |
| 469 | + } else if (childShuffleType == ShuffleType.STORAGE_BUCKETED |
| 470 | + && firstStorageBucketedIndex < 0) { |
| 471 | + firstStorageBucketedIndex = i; |
| 472 | + } |
| 473 | + } |
| 474 | + } |
| 475 | + if (distributeToChildIndex < 0) { |
| 476 | + distributeToChildIndex = firstStorageBucketedIndex; |
| 477 | + } |
| 478 | + if (distributeToChildIndex >= 0) { |
| 479 | + DistributionSpecHash childDistribution |
| 480 | + = (DistributionSpecHash) childrenDistribution.get(distributeToChildIndex); |
| 481 | + List<SlotReference> childToIndex = setOperation.getRegularChildrenOutputs().get(distributeToChildIndex); |
| 482 | + Map<ExprId, Integer> idToOutputIndex = new LinkedHashMap<>(); |
| 483 | + for (int j = 0; j < childToIndex.size(); j++) { |
| 484 | + idToOutputIndex.put(childToIndex.get(j).getExprId(), j); |
| 485 | + } |
| 486 | + |
| 487 | + List<ExprId> orderedShuffledColumns = childDistribution.getOrderedShuffledColumns(); |
| 488 | + List<ExprId> setOperationDistributeColumnIds = new ArrayList<>(); |
| 489 | + for (ExprId tableDistributeColumnId : orderedShuffledColumns) { |
| 490 | + Integer index = idToOutputIndex.get(tableDistributeColumnId); |
| 491 | + if (index == null) { |
| 492 | + break; |
| 493 | + } |
| 494 | + setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId()); |
| 495 | + } |
| 496 | + // check whether the set operation output all distribution columns of the child |
| 497 | + if (setOperationDistributeColumnIds.size() == orderedShuffledColumns.size()) { |
| 498 | + // Keep the basic child's specific storage layout as the set operation output. When |
| 499 | + // the basic child is on the right (shuffleToRight) the output rows are physically |
| 500 | + // placed by the right child's storage bucket function, so advertising that layout is |
| 501 | + // truthful: a parent can co-locate this set operation against the same layout, while |
| 502 | + // a sibling on a different layout is re-aligned rather than wrongly co-located. (The |
| 503 | + // earlier "Can not find tablet ... in the bucket" failure came from advertising a |
| 504 | + // layout-less EXECUTION_BUCKETED here, which erased the table id so two different- |
| 505 | + // layout set operations looked co-locatable; preserving the layout fixes that.) |
| 506 | + return new PhysicalProperties( |
| 507 | + new DistributionSpecHash( |
| 508 | + setOperationDistributeColumnIds, |
| 509 | + childDistribution.getShuffleType(), |
| 510 | + childDistribution.getTableId(), |
| 511 | + childDistribution.getSelectedIndexId(), |
| 512 | + childDistribution.getPartitionIds() |
| 513 | + ) |
| 514 | + ); |
| 515 | + } |
| 516 | + } |
490 | 517 |
|
491 | 518 | for (int i = 0; i < childrenDistribution.size(); i++) { |
492 | 519 | DistributionSpec childDistribution = childrenDistribution.get(i); |
|
0 commit comments