@Override public PlanNodeCostEstimate visitJoin(JoinNode node, Void context) { return calculateJoinCost( node, node.getLeft(), node.getRight(), Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED))); }
@Override public PlanNodeCostEstimate visitJoin(JoinNode node, Void context) { return calculateJoinCost( node, node.getLeft(), node.getRight(), Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED))); }
@Override public PlanNodeCostEstimate visitJoin(JoinNode node, Void context) { return calculateJoinExchangeCost( node.getLeft(), node.getRight(), stats, types, Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED)), taskCountEstimator.estimateSourceDistributedTaskCount()); }
@Override public PlanNodeCostEstimate visitJoin(JoinNode node, Void context) { return calculateJoinExchangeCost( node.getLeft(), node.getRight(), stats, types, Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED)), taskCountEstimator.estimateSourceDistributedTaskCount()); }
private void flattenNode(PlanNode node, int limit) { PlanNode resolved = lookup.resolve(node); // (limit - 2) because you need to account for adding left and right side if (!(resolved instanceof JoinNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } JoinNode joinNode = (JoinNode) resolved; if (joinNode.getType() != INNER || !isDeterministic(joinNode.getFilter().orElse(TRUE_LITERAL)) || joinNode.getDistributionType().isPresent()) { sources.add(node); return; } // we set the left limit to limit - 1 to account for the node on the right flattenNode(joinNode.getLeft(), limit - 1); flattenNode(joinNode.getRight(), limit); joinNode.getCriteria().stream() .map(EquiJoinClause::toExpression) .forEach(filters::add); joinNode.getFilter().ifPresent(filters::add); }
private void flattenNode(PlanNode node, int limit) { PlanNode resolved = lookup.resolve(node); // (limit - 2) because you need to account for adding left and right side if (!(resolved instanceof JoinNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } JoinNode joinNode = (JoinNode) resolved; if (joinNode.getType() != INNER || !isDeterministic(joinNode.getFilter().orElse(TRUE_LITERAL)) || joinNode.getDistributionType().isPresent()) { sources.add(node); return; } // we set the left limit to limit - 1 to account for the node on the right flattenNode(joinNode.getLeft(), limit - 1); flattenNode(joinNode.getRight(), limit); joinNode.getCriteria().stream() .map(EquiJoinClause::toExpression) .forEach(filters::add); joinNode.getFilter().ifPresent(filters::add); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { Optional<PlanNode> newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs); Optional<PlanNode> newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs); if (!newLeft.isPresent() && !newRight.isPresent()) { return Optional.empty(); } ImmutableList.Builder<Symbol> outputSymbolBuilder = ImmutableList.builder(); outputSymbolBuilder.addAll(newLeft.orElse(joinNode.getLeft()).getOutputSymbols()); outputSymbolBuilder.addAll(newRight.orElse(joinNode.getRight()).getOutputSymbols()); return Optional.of(new JoinNode( idAllocator.getNextId(), joinNode.getType(), newLeft.orElse(joinNode.getLeft()), newRight.orElse(joinNode.getRight()), joinNode.getCriteria(), outputSymbolBuilder.build(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { Optional<PlanNode> newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs); Optional<PlanNode> newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs); if (!newLeft.isPresent() && !newRight.isPresent()) { return Optional.empty(); } ImmutableList.Builder<Symbol> outputSymbolBuilder = ImmutableList.builder(); outputSymbolBuilder.addAll(newLeft.orElse(joinNode.getLeft()).getOutputSymbols()); outputSymbolBuilder.addAll(newRight.orElse(joinNode.getRight()).getOutputSymbols()); return Optional.of(new JoinNode( idAllocator.getNextId(), joinNode.getType(), newLeft.orElse(joinNode.getLeft()), newRight.orElse(joinNode.getRight()), joinNode.getCriteria(), outputSymbolBuilder.build(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List<Symbol> leftSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getLeft) .collect(toImmutableList()); List<Symbol> rightSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableList()); JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); if (distributionType == JoinNode.DistributionType.REPLICATED) { PlanWithProperties left = node.getLeft().accept(this, PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) if (!node.getCriteria().isEmpty() && left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { return planPartitionedJoin(node, leftSymbols, rightSymbols, left); } return planReplicatedJoin(node, left); } else { return planPartitionedJoin(node, leftSymbols, rightSymbols); } }
@Override public PlanWithProperties visitJoin(JoinNode node, PreferredProperties preferredProperties) { List<Symbol> leftSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getLeft) .collect(toImmutableList()); List<Symbol> rightSymbols = node.getCriteria().stream() .map(JoinNode.EquiJoinClause::getRight) .collect(toImmutableList()); JoinNode.DistributionType distributionType = node.getDistributionType().orElseThrow(() -> new IllegalArgumentException("distributionType not yet set")); if (distributionType == JoinNode.DistributionType.REPLICATED) { PlanWithProperties left = node.getLeft().accept(this, PreferredProperties.any()); // use partitioned join if probe side is naturally partitioned on join symbols (e.g: because of aggregation) if (!node.getCriteria().isEmpty() && left.getProperties().isNodePartitionedOn(leftSymbols) && !left.getProperties().isSingleNode()) { return planPartitionedJoin(node, leftSymbols, rightSymbols, left); } return planReplicatedJoin(node, left); } else { return planPartitionedJoin(node, leftSymbols, rightSymbols); } }
private PlanNode pushPartialToJoin( AggregationNode aggregation, JoinNode child, PlanNode leftChild, PlanNode rightChild, Context context) { JoinNode joinNode = new JoinNode( child.getId(), child.getType(), leftChild, rightChild, child.getCriteria(), ImmutableList.<Symbol>builder() .addAll(leftChild.getOutputSymbols()) .addAll(rightChild.getOutputSymbols()) .build(), child.getFilter(), child.getLeftHashSymbol(), child.getRightHashSymbol(), child.getDistributionType()); return restrictOutputs(context.getIdAllocator(), joinNode, ImmutableSet.copyOf(aggregation.getOutputSymbols())).orElse(joinNode); } }
private PlanNode pushPartialToJoin( AggregationNode aggregation, JoinNode child, PlanNode leftChild, PlanNode rightChild, Context context) { JoinNode joinNode = new JoinNode( child.getId(), child.getType(), leftChild, rightChild, child.getCriteria(), ImmutableList.<Symbol>builder() .addAll(leftChild.getOutputSymbols()) .addAll(rightChild.getOutputSymbols()) .build(), child.getFilter(), child.getLeftHashSymbol(), child.getRightHashSymbol(), child.getDistributionType()); return restrictOutputs(context.getIdAllocator(), joinNode, ImmutableSet.copyOf(aggregation.getOutputSymbols())).orElse(joinNode); } }
@Override public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) { PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); List<JoinNode.EquiJoinClause> canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); Optional<Expression> canonicalFilter = node.getFilter().map(this::canonicalize); Optional<Symbol> canonicalLeftHashSymbol = canonicalize(node.getLeftHashSymbol()); Optional<Symbol> canonicalRightHashSymbol = canonicalize(node.getRightHashSymbol()); if (node.getType().equals(INNER)) { canonicalCriteria.stream() .filter(clause -> types.get(clause.getLeft()).equals(types.get(clause.getRight()))) .filter(clause -> node.getOutputSymbols().contains(clause.getLeft())) .forEach(clause -> map(clause.getRight(), clause.getLeft())); } return new JoinNode(node.getId(), node.getType(), left, right, canonicalCriteria, canonicalizeAndDistinct(node.getOutputSymbols()), canonicalFilter, canonicalLeftHashSymbol, canonicalRightHashSymbol, node.getDistributionType()); }
@Override public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) { PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); List<JoinNode.EquiJoinClause> canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); Optional<Expression> canonicalFilter = node.getFilter().map(this::canonicalize); Optional<Symbol> canonicalLeftHashSymbol = canonicalize(node.getLeftHashSymbol()); Optional<Symbol> canonicalRightHashSymbol = canonicalize(node.getRightHashSymbol()); if (node.getType().equals(INNER)) { canonicalCriteria.stream() .filter(clause -> types.get(clause.getLeft()).equals(types.get(clause.getRight()))) .filter(clause -> node.getOutputSymbols().contains(clause.getLeft())) .forEach(clause -> map(clause.getRight(), clause.getLeft())); } return new JoinNode(node.getId(), node.getType(), left, right, canonicalCriteria, canonicalizeAndDistinct(node.getOutputSymbols()), canonicalFilter, canonicalLeftHashSymbol, canonicalRightHashSymbol, node.getDistributionType()); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { return Optional.of( new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), filteredCopy(joinNode.getOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { return Optional.of( new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), filteredCopy(joinNode.getOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override public Result apply(JoinNode joinNode, Captures captures, Context context) { Optional<Expression> filter = joinNode.getFilter().map(x -> rewriter.rewrite(x, context)); if (!joinNode.getFilter().equals(filter)) { return Result.ofPlanNode(new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), joinNode.getOutputSymbols(), filter, joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } return Result.empty(); } }
@Override public Result apply(JoinNode joinNode, Captures captures, Context context) { Optional<Expression> filter = joinNode.getFilter().map(x -> rewriter.rewrite(x, context)); if (!joinNode.getFilter().equals(filter)) { return Result.ofPlanNode(new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), joinNode.getOutputSymbols(), filter, joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } return Result.empty(); } }
@Override public Void visitJoin(JoinNode node, Integer indent) { JoinNode.DistributionType distributionType = node.getDistributionType() .orElseThrow(() -> new VerifyException("Expected distribution type to be set")); if (node.isCrossJoin()) { checkState(node.getType() == INNER && distributionType == REPLICATED, "Expected CROSS JOIN to be INNER REPLICATED"); output(indent, "cross join:"); } else { output(indent, "join (%s, %s):", node.getType(), distributionType); } return visitPlan(node, indent + 1); }
private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheritedPredicate) { checkArgument(EnumSet.of(INNER, RIGHT, LEFT, FULL).contains(node.getType()), "Unsupported join type: %s", node.getType()); if (node.getType() == JoinNode.Type.INNER) { return node; } if (node.getType() == JoinNode.Type.FULL) { boolean canConvertToLeftJoin = canConvertOuterToInner(node.getLeft().getOutputSymbols(), inheritedPredicate); boolean canConvertToRightJoin = canConvertOuterToInner(node.getRight().getOutputSymbols(), inheritedPredicate); if (!canConvertToLeftJoin && !canConvertToRightJoin) { return node; } if (canConvertToLeftJoin && canConvertToRightJoin) { return new JoinNode(node.getId(), INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); } else { return new JoinNode(node.getId(), canConvertToLeftJoin ? LEFT : RIGHT, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); } } if (node.getType() == JoinNode.Type.LEFT && !canConvertOuterToInner(node.getRight().getOutputSymbols(), inheritedPredicate) || node.getType() == JoinNode.Type.RIGHT && !canConvertOuterToInner(node.getLeft().getOutputSymbols(), inheritedPredicate)) { return node; } return new JoinNode(node.getId(), JoinNode.Type.INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); }