@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Void> context) { PlanNode probeSource = context.rewrite(node.getProbeSource()); PlanNode indexSource = context.rewrite(node.getIndexSource()); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, canonicalizeIndexJoinCriteria(node.getCriteria()), canonicalize(node.getProbeHashSymbol()), canonicalize(node.getIndexHashSymbol())); }
@Override public PlanNode replaceChildren(List<PlanNode> newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); return new IndexJoinNode(getId(), type, newChildren.get(0), newChildren.get(1), criteria, probeHashSymbol, indexHashSymbol); }
@Override public Set<PlanFragmentId> visitIndexJoin(IndexJoinNode node, PlanFragmentId currentFragmentId) { return processJoin(node.getIndexSource(), node.getProbeSource(), currentFragmentId); }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { List<Expression> joinExpressions = new ArrayList<>(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(new ComparisonExpression(ComparisonExpression.Operator.EQUAL, clause.getProbe().toSymbolReference(), clause.getIndex().toSymbolReference())); } String criteria = Joiner.on(" AND ").join(joinExpressions); String joinLabel = format("%sIndexJoin", node.getType().getJoinLabel()); printNode(node, joinLabel, criteria, NODE_COLORS.get(NodeType.JOIN)); node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); return null; }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, List<PlanNode> newChildren) { checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); return new IndexJoinNode(node.getId(), node.getType(), newChildren.get(0), newChildren.get(1), node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public Void visitIndexJoin(IndexJoinNode node, Set<Symbol> boundSymbols) { node.getProbeSource().accept(this, boundSymbols); node.getIndexSource().accept(this, boundSymbols); Set<Symbol> probeInputs = createInputs(node.getProbeSource(), boundSymbols); Set<Symbol> indexSourceInputs = createInputs(node.getIndexSource(), boundSymbols); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(probeInputs.contains(clause.getProbe()), "Probe symbol from index join clause (%s) not in probe source (%s)", clause.getProbe(), node.getProbeSource().getOutputSymbols()); checkArgument(indexSourceInputs.contains(clause.getIndex()), "Index symbol from index join clause (%s) not in index source (%s)", clause.getIndex(), node.getIndexSource().getOutputSymbols()); } Set<Symbol> lookupSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); Map<Symbol, Symbol> trace = IndexKeyTracer.trace(node.getIndexSource(), lookupSymbols); checkArgument(!trace.isEmpty() && lookupSymbols.containsAll(trace.keySet()), "Index lookup symbols are not traceable to index source: %s", lookupSymbols); return null; }
private static PlanNode createIndexJoinWithExpectedOutputs(List<Symbol> expectedOutputs, IndexJoinNode.Type type, PlanNode probe, PlanNode index, List<IndexJoinNode.EquiJoinClause> equiJoinClause, PlanNodeIdAllocator idAllocator) { PlanNode result = new IndexJoinNode(idAllocator.getNextId(), type, probe, index, equiJoinClause, Optional.empty(), Optional.empty()); if (!result.getOutputSymbols().equals(expectedOutputs)) { result = new ProjectNode( idAllocator.getNextId(), result, Assignments.identity(expectedOutputs)); } return result; }
/** * This method creates a mapping from each index source lookup symbol (directly applied to the index) * to the corresponding probe key Input */ private SetMultimap<Symbol, Integer> mapIndexSourceLookupSymbolToProbeKeyInput(IndexJoinNode node, Map<Symbol, Integer> probeKeyLayout) { Set<Symbol> indexJoinSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); // Trace the index join symbols to the index source lookup symbols // Map: Index join symbol => Index source lookup symbol Map<Symbol, Symbol> indexKeyTrace = IndexJoinOptimizer.IndexKeyTracer.trace(node.getIndexSource(), indexJoinSymbols); // Map the index join symbols to the probe key Input Multimap<Symbol, Integer> indexToProbeKeyInput = HashMultimap.create(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { indexToProbeKeyInput.put(clause.getIndex(), probeKeyLayout.get(clause.getProbe())); } // Create the mapping from index source look up symbol to probe key Input ImmutableSetMultimap.Builder<Symbol, Integer> builder = ImmutableSetMultimap.builder(); for (Map.Entry<Symbol, Symbol> entry : indexKeyTrace.entrySet()) { Symbol indexJoinSymbol = entry.getKey(); Symbol indexLookupSymbol = entry.getValue(); builder.putAll(indexLookupSymbol, indexToProbeKeyInput.get(indexJoinSymbol)); } return builder.build(); }
@Override public Map<PlanNodeId, SplitSource> visitIndexJoin(IndexJoinNode node, Void context) { return node.getProbeSource().accept(this, context); }
@Override public StreamProperties visitIndexJoin(IndexJoinNode node, List<StreamProperties> inputProperties) { StreamProperties probeProperties = inputProperties.get(0); switch (node.getType()) { case INNER: return probeProperties; case SOURCE_OUTER: // the probe can contain nulls in any stream so we can't say anything about the // partitioning but the other properties of the probe will be maintained. return probeProperties.withUnspecifiedPartitioning(); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); } }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, HashComputationSet parentPreference) List<IndexJoinNode.EquiJoinClause> clauses = node.getCriteria(); node.getProbeSource(), new HashComputationSet(probeHashComputation), true, PlanWithProperties index = planAndEnforce(node.getIndexSource(), requiredHashes, true, requiredHashes); Symbol indexHashSymbol = index.getRequiredHashSymbol(indexHashComputation.get()); if (node.getType() == IndexJoinNode.Type.INNER) { allHashSymbols.putAll(probe.getHashSymbols()); new IndexJoinNode( node.getId(), node.getType(), probe.getNode(), index.getNode(), node.getCriteria(), Optional.of(probeHashSymbol), Optional.of(indexHashSymbol)),
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, Context context) { List<Symbol> joinColumns = Lists.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = context.getPreferredProperties().getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(joinColumns), desiredLocalProperties))); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, context.getPreferredProperties(), probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); return null; }
@Override public Void visitIndexJoin(IndexJoinNode node, Void context) { node.getProbeSource().accept(this, context); node.getIndexSource().accept(this, context); verifyUniqueId(node); Set<Symbol> probeInputs = ImmutableSet.copyOf(node.getProbeSource().getOutputSymbols()); Set<Symbol> indexSourceInputs = ImmutableSet.copyOf(node.getIndexSource().getOutputSymbols()); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { checkArgument(probeInputs.contains(clause.getProbe()), "Probe symbol from index join clause (%s) not in probe source (%s)", clause.getProbe(), node.getProbeSource().getOutputSymbols()); checkArgument(indexSourceInputs.contains(clause.getIndex()), "Index symbol from index join clause (%s) not in index source (%s)", clause.getIndex(), node.getIndexSource().getOutputSymbols()); } Set<Symbol> lookupSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); Map<Symbol, Symbol> trace = IndexKeyTracer.trace(node.getIndexSource(), lookupSymbols); checkArgument(!trace.isEmpty() && lookupSymbols.containsAll(trace.keySet()), "Index lookup symbols are not traceable to index source: %s", lookupSymbols); return null; }
public PlanNode indexJoin(IndexJoinNode.Type type, TableScanNode probe, TableScanNode index) { return new IndexJoinNode( idAllocator.getNextId(), type, probe, index, emptyList(), Optional.empty(), Optional.empty()); }
/** * This method creates a mapping from each index source lookup symbol (directly applied to the index) * to the corresponding probe key Input */ private SetMultimap<Symbol, Integer> mapIndexSourceLookupSymbolToProbeKeyInput(IndexJoinNode node, Map<Symbol, Integer> probeKeyLayout) { Set<Symbol> indexJoinSymbols = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getIndex) .collect(toImmutableSet()); // Trace the index join symbols to the index source lookup symbols // Map: Index join symbol => Index source lookup symbol Map<Symbol, Symbol> indexKeyTrace = IndexJoinOptimizer.IndexKeyTracer.trace(node.getIndexSource(), indexJoinSymbols); // Map the index join symbols to the probe key Input Multimap<Symbol, Integer> indexToProbeKeyInput = HashMultimap.create(); for (IndexJoinNode.EquiJoinClause clause : node.getCriteria()) { indexToProbeKeyInput.put(clause.getIndex(), probeKeyLayout.get(clause.getProbe())); } // Create the mapping from index source look up symbol to probe key Input ImmutableSetMultimap.Builder<Symbol, Integer> builder = ImmutableSetMultimap.builder(); for (Map.Entry<Symbol, Symbol> entry : indexKeyTrace.entrySet()) { Symbol indexJoinSymbol = entry.getKey(); Symbol indexLookupSymbol = entry.getValue(); builder.putAll(indexLookupSymbol, indexToProbeKeyInput.get(indexJoinSymbol)); } return builder.build(); }
@Override public Map<Symbol, Symbol> visitIndexJoin(IndexJoinNode node, Set<Symbol> lookupSymbols) { Set<Symbol> probeLookupSymbols = lookupSymbols.stream() .filter(node.getProbeSource().getOutputSymbols()::contains) .collect(toImmutableSet()); checkState(!probeLookupSymbols.isEmpty(), "No lookup symbols were able to pass through the index join probe source"); return node.getProbeSource().accept(this, probeLookupSymbols); }
@Override public ActualProperties visitIndexJoin(IndexJoinNode node, List<ActualProperties> inputProperties) { // TODO: include all equivalent columns in partitioning properties ActualProperties probeProperties = inputProperties.get(0); ActualProperties indexProperties = inputProperties.get(1); switch (node.getType()) { case INNER: return ActualProperties.builderFrom(probeProperties) .constants(ImmutableMap.<Symbol, NullableValue>builder() .putAll(probeProperties.getConstants()) .putAll(indexProperties.getConstants()) .build()) .build(); case SOURCE_OUTER: return ActualProperties.builderFrom(probeProperties) .constants(probeProperties.getConstants()) .build(); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); } }
@Override public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> probeInputsBuilder = ImmutableSet.builder(); probeInputsBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe)); if (node.getProbeHashSymbol().isPresent()) { probeInputsBuilder.add(node.getProbeHashSymbol().get()); } Set<Symbol> probeInputs = probeInputsBuilder.build(); ImmutableSet.Builder<Symbol> indexInputBuilder = ImmutableSet.builder(); indexInputBuilder.addAll(context.get()) .addAll(Iterables.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getIndex)); if (node.getIndexHashSymbol().isPresent()) { indexInputBuilder.add(node.getIndexHashSymbol().get()); } Set<Symbol> indexInputs = indexInputBuilder.build(); PlanNode probeSource = context.rewrite(node.getProbeSource(), probeInputs); PlanNode indexSource = context.rewrite(node.getIndexSource(), indexInputs); return new IndexJoinNode(node.getId(), node.getType(), probeSource, indexSource, node.getCriteria(), node.getProbeHashSymbol(), node.getIndexHashSymbol()); }