/** * This method creates a mapping from each index source lookup symbol (directly applied to the index) * to the corresponding probe key Input */ private SetMultimap<Symbol, Integer> mapIndexSourceLookupSymbolToProbeKeyInput(IndexJoinNode node, Map<Symbol, Integer> probeKeyLayout) { Set<Symbol> indexJoinSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); // Trace the index join symbols to the index source lookup symbols // Map: Index join symbol => Index source lookup symbol Map<Symbol, Symbol> indexKeyTrace = IndexJoinOptimizer.IndexKeyTracer.trace(node.getIndexSource(), indexJoinSymbols); // Map the index join symbols to the probe key Input Multimap<Symbol, Integer> indexToProbeKeyInput = HashMultimap.create(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { indexToProbeKeyInput.put(clause.getIndex(), probeKeyLayout.get(clause.getProbe())); } // Create the mapping from index source look up symbol to probe key Input ImmutableSetMultimap.Builder<Symbol, Integer> builder = ImmutableSetMultimap.builder(); for (Map.Entry<Symbol, Symbol> entry : indexKeyTrace.entrySet()) { Symbol indexJoinSymbol = entry.getKey(); Symbol indexLookupSymbol = entry.getValue(); builder.putAll(indexLookupSymbol, indexToProbeKeyInput.get(indexJoinSymbol)); } return builder.build(); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> probeInputsBuilder = ImmutableSet.builder(); probeInputsBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe)); if (node.getProbeHashSymbol().isPresent()) { probeInputsBuilder.add(node.getProbeHashSymbol().get()); } Set<Symbol> probeInputs = probeInputsBuilder.build(); ImmutableSet.Builder<Symbol> indexInputBuilder = ImmutableSet.builder(); indexInputBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getIndex)); if (node.getIndexHashSymbol().isPresent()) { indexInputBuilder.add(node.getIndexHashSymbol().get()); } Set<Symbol> indexInputs = indexInputBuilder.build(); PlanNode probeSource = context.rewrite(node.getProbeSource(), probeInputs); PlanNode indexSource = context.rewrite(node.getIndexSource(), indexInputs); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { List<Expression> joinExpressions = new ArrayList<>(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Operator.EQUAL, clause.getProbe().toSymbolReference(), clause.getIndex().toSymbolReference())); } String criteria = Joiner.on(" AND ").join(joinExpressions); String joinLabel = format("%sIndexJoin", node.getType().getJoinLabel()); printNode(node, joinLabel, criteria, NODE_COLORS.get(NodeType.JOIN)); node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); return null; }
@Override public Void visitIndexJoin(IndexJoinNode node, Set<Symbol> boundSymbols) { node.getProbeSource().accept(this, boundSymbols); node.getIndexSource().accept(this, boundSymbols); Set<Symbol> probeInputs = createInputs(node.getProbeSource(), boundSymbols); Set<Symbol> indexSourceInputs = createInputs(node.getIndexSource(), boundSymbols); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(probeInputs.contains(clause.getProbe()), "Probe symbol from index join clause (%s) not in probe source (%s)", clause.getProbe(), node.getProbeSource().getOutputSymbols()); checkArgument(indexSourceInputs.contains(clause.getIndex()), "Index symbol from index join clause (%s) not in index source (%s)", clause.getIndex(), node.getIndexSource().getOutputSymbols()); } Set<Symbol> lookupSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); Map<Symbol, Symbol> trace = IndexKeyTracer.trace(node.getIndexSource(), lookupSymbols); checkArgument(!trace.isEmpty() && lookupSymbols.containsAll(trace.keySet()), "Index lookup symbols are not traceable to index source: %s", lookupSymbols); return null; }
@Override public PhysicalOperation visitIndexJoin(IndexJoinNode node, LocalExecutionPlanContext context) List<IndexJoinNode.EquiJoinClause> clauses = node.getCriteria(); Set<Integer> lookupSourceInputChannels = node.getCriteria().stream() .filter(equiJoinClause -> indexSymbolsNeededBySource.contains(equiJoinClause.getIndex())) .map(IndexJoinNode.EquiJoinClause::getProbe) int[] nonLookupInputChannels = Ints.toArray(node.getCriteria().stream() .filter(equiJoinClause -> !indexSymbolsNeededBySource.contains(equiJoinClause.getIndex())) .map(IndexJoinNode.EquiJoinClause::getProbe) .map(probeKeyLayout::get) .collect(toImmutableList())); int[] nonLookupOutputChannels = Ints.toArray(node.getCriteria().stream() .filter(equiJoinClause -> !indexSymbolsNeededBySource.contains(equiJoinClause.getIndex())) .map(IndexJoinNode.EquiJoinClause::getIndex)
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, HashComputationSet parentPreference) List<IndexJoinNode.EquiJoinClause> clauses = node.getCriteria(); probe.getNode(), index.getNode(), node.getCriteria(), Optional.of(probeHashSymbol), Optional.of(indexHashSymbol)),
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through the probe side of an index join Set<Symbol> probeLookupSymbols = context.get().getLookupSymbols().stream() .filter(node.getProbeSource().getOutputSymbols()::contains) .collect(toImmutableSet()); if (probeLookupSymbols.isEmpty()) { return node; } PlanNode rewrittenProbeSource = context.rewrite(node.getProbeSource(), new Context(probeLookupSymbols, context.get().getSuccess())); PlanNode source = node; if (rewrittenProbeSource != node.getProbeSource()) { source = new IndexJoinNode(node.getId(), node.getType(), rewrittenProbeSource, node.getIndexSource(), node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); } return source; }
@Override public Void visitIndexJoin(IndexJoinNode node, Integer indent) { List<Expression> joinExpressions = new ArrayList<>(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Operator.EQUAL, clause.getProbe().toSymbolReference(), clause.getIndex().toSymbolReference())); } print(indent, "- %sIndexJoin[%s]%s => [%s]", node.getType().getJoinLabel(), Joiner.on(" AND ").join(joinExpressions), formatHash(node.getProbeHashSymbol(), node.getIndexHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); node.getProbeSource().accept(this, indent + 1); node.getIndexSource().accept(this, indent + 1); return null; }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Void> context) { PlanNode probeSource = context.rewrite(node.getProbeSource()); PlanNode indexSource = context.rewrite(node.getIndexSource()); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, canonicalizeIndexJoinCriteria(node.getCriteria()), canonicalize(node.getProbeHashSymbol()), canonicalize(node.getIndexHashSymbol())); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, List<PlanNode> newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); return new IndexJoinNode(node.getId(), node.getType(), newChildren.get(0), newChildren.get(1), node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); }
/** * This method creates a mapping from each index source lookup symbol (directly applied to the index) * to the corresponding probe key Input */ private SetMultimap<Symbol, Integer> mapIndexSourceLookupSymbolToProbeKeyInput(IndexJoinNode node, Map<Symbol, Integer> probeKeyLayout) { Set<Symbol> indexJoinSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); // Trace the index join symbols to the index source lookup symbols // Map: Index join symbol => Index source lookup symbol Map<Symbol, Symbol> indexKeyTrace = IndexJoinOptimizer.IndexKeyTracer.trace(node.getIndexSource(), indexJoinSymbols); // Map the index join symbols to the probe key Input Multimap<Symbol, Integer> indexToProbeKeyInput = HashMultimap.create(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { indexToProbeKeyInput.put(clause.getIndex(), probeKeyLayout.get(clause.getProbe())); } // Create the mapping from index source look up symbol to probe key Input ImmutableSetMultimap.Builder<Symbol, Integer> builder = ImmutableSetMultimap.builder(); for (Map.Entry<Symbol, Symbol> entry : indexKeyTrace.entrySet()) { Symbol indexJoinSymbol = entry.getKey(); Symbol indexLookupSymbol = entry.getValue(); builder.putAll(indexLookupSymbol, indexToProbeKeyInput.get(indexJoinSymbol)); } return builder.build(); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> probeInputsBuilder = ImmutableSet.builder(); probeInputsBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe)); if (node.getProbeHashSymbol().isPresent()) { probeInputsBuilder.add(node.getProbeHashSymbol().get()); } Set<Symbol> probeInputs = probeInputsBuilder.build(); ImmutableSet.Builder<Symbol> indexInputBuilder = ImmutableSet.builder(); indexInputBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getIndex)); if (node.getIndexHashSymbol().isPresent()) { indexInputBuilder.add(node.getIndexHashSymbol().get()); } Set<Symbol> indexInputs = indexInputBuilder.build(); PlanNode probeSource = context.rewrite(node.getProbeSource(), probeInputs); PlanNode indexSource = context.rewrite(node.getIndexSource(), indexInputs); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Void> context) { PlanNode rewrittenIndex = context.rewrite(node.getIndexSource(), null); PlanNode rewrittenProbe = context.rewrite(node.getProbeSource(), null); Symbol indexHashSymbol = symbolAllocator.newHashSymbol(); Symbol probeHashSymbol = symbolAllocator.newHashSymbol(); List<IndexJoinNode.EquiJoinClause> clauses = node.getCriteria(); List<Symbol> indexSymbols = Lists.transform(clauses, IndexJoinNode.EquiJoinClause::getIndex); List<Symbol> probeSymbols = Lists.transform(clauses, IndexJoinNode.EquiJoinClause::getProbe); PlanNode indexHashProjectNode = getHashProjectNode(idAllocator, rewrittenIndex, indexHashSymbol, indexSymbols); PlanNode probeHashProjectNode = getHashProjectNode(idAllocator, rewrittenProbe, probeHashSymbol, probeSymbols); return new IndexJoinNode(idAllocator.getNextId(), node.getType(), probeHashProjectNode, indexHashProjectNode, node.getCriteria(), Optional.of(probeHashSymbol), Optional.of(indexHashSymbol)); }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); verifyUniqueId(node); Set<Symbol> probeInputs = ImmutableSet.copyOf(node.getProbeSource().getOutputSymbols()); Set<Symbol> indexSourceInputs = ImmutableSet.copyOf(node.getIndexSource().getOutputSymbols()); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(probeInputs.contains(clause.getProbe()), "Probe symbol from index join clause (%s) not in probe source (%s)", clause.getProbe(), node.getProbeSource().getOutputSymbols()); checkArgument(indexSourceInputs.contains(clause.getIndex()), "Index symbol from index join clause (%s) not in index source (%s)", clause.getIndex(), node.getIndexSource().getOutputSymbols()); } Set<Symbol> lookupSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); Map<Symbol, Symbol> trace = IndexKeyTracer.trace(node.getIndexSource(), lookupSymbols); checkArgument(!trace.isEmpty() && lookupSymbols.containsAll(trace.keySet()), "Index lookup symbols are not traceable to index source: %s", lookupSymbols); return null; }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { List<Expression> joinExpressions = new ArrayList<>(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getProbe().toQualifiedName()), new QualifiedNameReference(clause.getIndex().toQualifiedName()))); } String criteria = Joiner.on(" AND ").join(joinExpressions); String joinLabel = format("%sIndexJoin", node.getType().getJoinLabel()); printNode(node, joinLabel, criteria, NODE_COLORS.get(NodeType.JOIN)); node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); return null; }
@Override public Void visitIndexJoin(IndexJoinNode node, Integer indent) { List<Expression> joinExpressions = new ArrayList<>(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getProbe().toQualifiedName()), new QualifiedNameReference(clause.getIndex().toQualifiedName()))); } print(indent, "- %sIndexJoin[%s] => [%s]", node.getType().getJoinLabel(), Joiner.on(" AND ").join(joinExpressions), formatOutputs(node.getOutputSymbols())); node.getProbeSource().accept(this, indent + 1); node.getIndexSource().accept(this, indent + 1); return null; }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Context> context) { // Lookup symbols can only be passed through the probe side of an index join Set<Symbol> probeLookupSymbols = context.get().getLookupSymbols().stream() .filter(node.getProbeSource().getOutputSymbols()::contains) .collect(toImmutableSet()); if (probeLookupSymbols.isEmpty()) { return node; } PlanNode rewrittenProbeSource = context.rewrite(node.getProbeSource(), new Context(probeLookupSymbols, context.get().getSuccess())); PlanNode source = node; if (rewrittenProbeSource != node.getProbeSource()) { source = new IndexJoinNode(node.getId(), node.getType(), rewrittenProbeSource, node.getIndexSource(), node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); } return source; }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, Context context) { List<Symbol> joinColumns = Lists.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = context.getPreferredProperties().getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(joinColumns), desiredLocalProperties))); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, context.getPreferredProperties(), probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Void> context) { PlanNode probeSource = context.rewrite(node.getProbeSource()); PlanNode indexSource = context.rewrite(node.getIndexSource()); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, canonicalizeIndexJoinCriteria(node.getCriteria()), canonicalize(node.getProbeHashSymbol()), canonicalize(node.getIndexHashSymbol())); }