private JoinGraph joinWith(JoinGraph other, List<JoinNode.EquiJoinClause> joinClauses, Context context, PlanNodeId newRoot) { for (PlanNode node : other.nodes) { checkState(!edges.containsKey(node.getId()), format("Node [%s] appeared in two JoinGraphs", node)); } List<PlanNode> nodes = ImmutableList.<PlanNode>builder() .addAll(this.nodes) .addAll(other.nodes) .build(); ImmutableMultimap.Builder<PlanNodeId, Edge> edges = ImmutableMultimap.<PlanNodeId, Edge>builder() .putAll(this.edges) .putAll(other.edges); List<Expression> joinedFilters = ImmutableList.<Expression>builder() .addAll(this.filters) .addAll(other.filters) .build(); for (JoinNode.EquiJoinClause edge : joinClauses) { Symbol leftSymbol = edge.getLeft(); Symbol rightSymbol = edge.getRight(); checkState(context.containsSymbol(leftSymbol)); checkState(context.containsSymbol(rightSymbol)); PlanNode left = context.getSymbolSource(leftSymbol); PlanNode right = context.getSymbolSource(rightSymbol); edges.put(left.getId(), new Edge(right, leftSymbol, rightSymbol)); edges.put(right.getId(), new Edge(left, rightSymbol, leftSymbol)); } return new JoinGraph(nodes, edges.build(), newRoot, joinedFilters, Optional.empty()); }
public static Optional<Symbol> filterOrRewrite(Collection<Symbol> columns, Collection<JoinNode.EquiJoinClause> equalities, Symbol column) { // symbol is exposed directly, so no translation needed if (columns.contains(column)) { return Optional.of(column); } // if the column is part of the equality conditions and its counterpart // is exposed, use that, instead for (JoinNode.EquiJoinClause equality : equalities) { if (equality.getLeft().equals(column) && columns.contains(equality.getRight())) { return Optional.of(equality.getRight()); } else if (equality.getRight().equals(column) && columns.contains(equality.getLeft())) { return Optional.of(equality.getLeft()); } } return Optional.empty(); }
private List<JoinNode.EquiJoinClause> canonicalizeJoinCriteria(List<JoinNode.EquiJoinClause> criteria) { ImmutableList.Builder<JoinNode.EquiJoinClause> builder = ImmutableList.builder(); for (JoinNode.EquiJoinClause clause : criteria) { builder.add(new JoinNode.EquiJoinClause(canonicalize(clause.getLeft()), canonicalize(clause.getRight()))); } return builder.build(); }
for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinConjuncts.add(new ComparisonExpression(ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getLeft().toQualifiedName()), new QualifiedNameReference(clause.getRight().toQualifiedName())));
@Override public Void visitJoin(JoinNode node, Void context) { node.getLeft().accept(this, context); node.getRight().accept(this, context); verifyUniqueId(node); Set<Symbol> leftInputs = ImmutableSet.copyOf(node.getLeft().getOutputSymbols()); Set<Symbol> rightInputs = ImmutableSet.copyOf(node.getRight().getOutputSymbols()); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(leftInputs.contains(clause.getLeft()), "Symbol from join clause (%s) not in left source (%s)", clause.getLeft(), node.getLeft().getOutputSymbols()); checkArgument(rightInputs.contains(clause.getRight()), "Symbol from join clause (%s) not in right source (%s)", clause.getRight(), node.getRight().getOutputSymbols()); } return null; }
@Override public Void visitJoin(JoinNode node, Void context) { List<Expression> joinExpressions = new ArrayList<>(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getLeft().toQualifiedName()), new QualifiedNameReference(clause.getRight().toQualifiedName()))); } String criteria = Joiner.on(" AND ").join(joinExpressions); printNode(node, node.getType().getJoinLabel(), criteria, NODE_COLORS.get(NodeType.JOIN)); node.getLeft().accept(this, context); node.getRight().accept(this, context); return null; }
@Override public Void visitJoin(JoinNode node, Integer indent) { List<Expression> joinExpressions = new ArrayList<>(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getLeft().toQualifiedName()), new QualifiedNameReference(clause.getRight().toQualifiedName()))); } print(indent, "- %s[%s] => [%s]", node.getType().getJoinLabel(), Joiner.on(" AND ").join(joinExpressions), formatOutputs(node.getOutputSymbols())); node.getLeft().accept(this, indent + 1); node.getRight().accept(this, indent + 1); return null; }
private List<JoinNode.EquiJoinClause> canonicalizeJoinCriteria(List<JoinNode.EquiJoinClause> criteria) { ImmutableList.Builder<JoinNode.EquiJoinClause> builder = ImmutableList.builder(); for (JoinNode.EquiJoinClause clause : criteria) { builder.add(new JoinNode.EquiJoinClause(canonicalize(clause.getLeft()), canonicalize(clause.getRight()))); } return builder.build(); }
private static Expression extractJoinPredicate(JoinNode joinNode) { ImmutableList.Builder<Expression> builder = ImmutableList.builder(); for (JoinNode.EquiJoinClause equiJoinClause : joinNode.getCriteria()) { builder.add(equalsExpression(equiJoinClause.getLeft(), equiJoinClause.getRight())); } return combineConjuncts(builder.build()); }
for (Symbol column : nodePartitioning.getColumns()) { for (JoinNode.EquiJoinClause equality : node.getCriteria()) { if (equality.getLeft().equals(column) || equality.getRight().equals(column)) { coalesceExpressions.add(new CoalesceExpression(ImmutableList.of(equality.getLeft().toSymbolReference(), equality.getRight().toSymbolReference())));
SymbolStatsEstimate leftColumnStats = leftStats.getSymbolStatistics(drivingClause.getLeft()); SymbolStatsEstimate rightColumnStats = rightStats.getSymbolStatistics(drivingClause.getRight()); double scaleFactor = nonMatchingLeftValuesFraction + leftColumnStats.getNullsFraction(); double newLeftNullsFraction = leftColumnStats.getNullsFraction() / scaleFactor; result = result.mapSymbolColumnStatistics(drivingClause.getLeft(), columnStats -> SymbolStatsEstimate.buildFrom(columnStats) .setLowValue(leftColumnStats.getLowValue()) result = result.mapSymbolColumnStatistics(drivingClause.getLeft(), columnStats -> SymbolStatsEstimate.buildFrom(columnStats) .setLowValue(NaN)
private PlanNodeStatsEstimate filterByAuxiliaryClause(PlanNodeStatsEstimate stats, EquiJoinClause clause, TypeProvider types) { // we just clear null fraction and adjust ranges here // selectivity is mostly handled by driving clause. We just scale heuristically by UNKNOWN_FILTER_COEFFICIENT here. SymbolStatsEstimate leftStats = stats.getSymbolStatistics(clause.getLeft()); SymbolStatsEstimate rightStats = stats.getSymbolStatistics(clause.getRight()); StatisticRange leftRange = StatisticRange.from(leftStats); StatisticRange rightRange = StatisticRange.from(rightStats); StatisticRange intersect = leftRange.intersect(rightRange); double leftFilterValue = firstNonNaN(leftRange.overlapPercentWith(intersect), 1); double rightFilterValue = firstNonNaN(rightRange.overlapPercentWith(intersect), 1); double leftNdvInRange = leftFilterValue * leftRange.getDistinctValuesCount(); double rightNdvInRange = rightFilterValue * rightRange.getDistinctValuesCount(); double retainedNdv = MoreMath.min(leftNdvInRange, rightNdvInRange); SymbolStatsEstimate newLeftStats = buildFrom(leftStats) .setNullsFraction(0) .setStatisticsRange(intersect) .setDistinctValuesCount(retainedNdv) .build(); SymbolStatsEstimate newRightStats = buildFrom(rightStats) .setNullsFraction(0) .setStatisticsRange(intersect) .setDistinctValuesCount(retainedNdv) .build(); PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.buildFrom(stats) .setOutputRowCount(stats.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT) .addSymbolStatistics(clause.getLeft(), newLeftStats) .addSymbolStatistics(clause.getRight(), newRightStats); return normalizer.normalize(result.build(), types); }
@Override public Void visitJoin(JoinNode node, Set<Symbol> boundSymbols) { node.getLeft().accept(this, boundSymbols); node.getRight().accept(this, boundSymbols); Set<Symbol> leftInputs = createInputs(node.getLeft(), boundSymbols); Set<Symbol> rightInputs = createInputs(node.getRight(), boundSymbols); Set<Symbol> allInputs = ImmutableSet.<Symbol>builder() .addAll(leftInputs) .addAll(rightInputs) .build(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(leftInputs.contains(clause.getLeft()), "Symbol from join clause (%s) not in left source (%s)", clause.getLeft(), node.getLeft().getOutputSymbols()); checkArgument(rightInputs.contains(clause.getRight()), "Symbol from join clause (%s) not in right source (%s)", clause.getRight(), node.getRight().getOutputSymbols()); } node.getFilter().ifPresent(predicate -> { Set<Symbol> predicateSymbols = SymbolsExtractor.extractUnique(predicate); checkArgument( allInputs.containsAll(predicateSymbols), "Symbol from filter (%s) not in sources (%s)", predicateSymbols, allInputs); }); checkLeftOutputSymbolsBeforeRight(node.getLeft().getOutputSymbols(), node.getOutputSymbols()); return null; }
@Override public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) { PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); List<JoinNode.EquiJoinClause> canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); Optional<Expression> canonicalFilter = node.getFilter().map(this::canonicalize); Optional<Symbol> canonicalLeftHashSymbol = canonicalize(node.getLeftHashSymbol()); Optional<Symbol> canonicalRightHashSymbol = canonicalize(node.getRightHashSymbol()); if (node.getType().equals(INNER)) { canonicalCriteria.stream() .filter(clause -> types.get(clause.getLeft()).equals(types.get(clause.getRight()))) .filter(clause -> node.getOutputSymbols().contains(clause.getLeft())) .forEach(clause -> map(clause.getRight(), clause.getLeft())); } return new JoinNode(node.getId(), node.getType(), left, right, canonicalCriteria, canonicalizeAndDistinct(node.getOutputSymbols()), canonicalFilter, canonicalLeftHashSymbol, canonicalRightHashSymbol, node.getDistributionType()); }
private PlanNodeStatsEstimate filterByEquiJoinClauses( PlanNodeStatsEstimate stats, EquiJoinClause drivingClause, Collection<EquiJoinClause> remainingClauses, Session session, TypeProvider types) { ComparisonExpression drivingPredicate = new ComparisonExpression(EQUAL, drivingClause.getLeft().toSymbolReference(), drivingClause.getRight().toSymbolReference()); PlanNodeStatsEstimate filteredStats = filterStatsCalculator.filterStats(stats, drivingPredicate, session, types); for (EquiJoinClause clause : remainingClauses) { filteredStats = filterByAuxiliaryClause(filteredStats, clause, types); } return filteredStats; }