@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, MarkDistinctNode markDistinctNode, Set<Symbol> referencedOutputs) { if (!referencedOutputs.contains(markDistinctNode.getMarkerSymbol())) { return Optional.of(markDistinctNode.getSource()); } Set<Symbol> requiredInputs = Streams.concat( referencedOutputs.stream() .filter(symbol -> !symbol.equals(markDistinctNode.getMarkerSymbol())), markDistinctNode.getDistinctSymbols().stream(), markDistinctNode.getHashSymbol().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); return restrictChildOutputs(idAllocator, markDistinctNode, requiredInputs); } }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, MarkDistinctNode markDistinctNode, Set<Symbol> referencedOutputs) { if (!referencedOutputs.contains(markDistinctNode.getMarkerSymbol())) { return Optional.of(markDistinctNode.getSource()); } Set<Symbol> requiredInputs = Streams.concat( referencedOutputs.stream() .filter(symbol -> !symbol.equals(markDistinctNode.getMarkerSymbol())), markDistinctNode.getDistinctSymbols().stream(), markDistinctNode.getHashSymbol().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); return restrictChildOutputs(idAllocator, markDistinctNode, requiredInputs); } }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Set<Symbol>> context) { if (!context.get().contains(node.getMarkerSymbol())) { return context.rewrite(node.getSource(), context.get()); } ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getDistinctSymbols()) .addAll(context.get().stream() .filter(symbol -> !symbol.equals(node.getMarkerSymbol())) .collect(toImmutableList())); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new MarkDistinctNode(node.getId(), source, node.getMarkerSymbol(), node.getDistinctSymbols(), node.getHashSymbol()); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Set<Symbol>> context) { if (!context.get().contains(node.getMarkerSymbol())) { return context.rewrite(node.getSource(), context.get()); } ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getDistinctSymbols()) .addAll(context.get().stream() .filter(symbol -> !symbol.equals(node.getMarkerSymbol())) .collect(toImmutableList())); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new MarkDistinctNode(node.getId(), source, node.getMarkerSymbol(), node.getDistinctSymbols(), node.getHashSymbol()); }
ImmutableList.of(), SINGLE, originalNode.getHashSymbol(), Optional.empty());
ImmutableList.of(), SINGLE, originalNode.getHashSymbol(), Optional.empty());
@Override public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); MarkDistinctNode markDistinctNode = (MarkDistinctNode) node; if (!markDistinctNode.getHashSymbol().equals(hashSymbol.map(alias -> alias.toSymbol(symbolAliases)))) { return NO_MATCH; } if (!ImmutableSet.copyOf(markDistinctNode.getDistinctSymbols()) .equals(distinctSymbols.stream().map(alias -> alias.toSymbol(symbolAliases)).collect(toImmutableSet()))) { return NO_MATCH; } return match(markerSymbol.toString(), markDistinctNode.getMarkerSymbol().toSymbolReference()); }
@Override public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) { checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); MarkDistinctNode markDistinctNode = (MarkDistinctNode) node; if (!markDistinctNode.getHashSymbol().equals(hashSymbol.map(alias -> alias.toSymbol(symbolAliases)))) { return NO_MATCH; } if (!ImmutableSet.copyOf(markDistinctNode.getDistinctSymbols()) .equals(distinctSymbols.stream().map(alias -> alias.toSymbol(symbolAliases)).collect(toImmutableSet()))) { return NO_MATCH; } return match(markerSymbol.toString(), markDistinctNode.getMarkerSymbol().toSymbolReference()); }
@Override public PhysicalOperation visitMarkDistinct(MarkDistinctNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); List<Integer> channels = getChannelsForSymbols(node.getDistinctSymbols(), source.getLayout()); Optional<Integer> hashChannel = node.getHashSymbol().map(channelGetter(source)); MarkDistinctOperatorFactory operator = new MarkDistinctOperatorFactory(context.getNextOperatorId(), node.getId(), source.getTypes(), channels, hashChannel, joinCompiler); return new PhysicalOperation(operator, makeLayout(node), context, source); }
@Override public PhysicalOperation visitMarkDistinct(MarkDistinctNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); List<Integer> channels = getChannelsForSymbols(node.getDistinctSymbols(), source.getLayout()); Optional<Integer> hashChannel = node.getHashSymbol().map(channelGetter(source)); MarkDistinctOperatorFactory operator = new MarkDistinctOperatorFactory(context.getNextOperatorId(), node.getId(), source.getTypes(), channels, hashChannel, joinCompiler); return new PhysicalOperation(operator, makeLayout(node), context, source); }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Integer indent) { print(indent, "- MarkDistinct[distinct=%s marker=%s]%s => [%s]", formatOutputs(node.getDistinctSymbols()), node.getMarkerSymbol(), formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return processChildren(node, indent + 1); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); List<Symbol> symbols = canonicalizeAndDistinct(node.getDistinctSymbols()); return new MarkDistinctNode(node.getId(), source, canonicalize(node.getMarkerSymbol()), symbols, canonicalize(node.getHashSymbol())); }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Integer indent) { print(indent, "- MarkDistinct[distinct=%s marker=%s]%s => [%s]", formatOutputs(node.getDistinctSymbols()), node.getMarkerSymbol(), formatHash(node.getHashSymbol()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return processChildren(node, indent + 1); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); List<Symbol> symbols = canonicalizeAndDistinct(node.getDistinctSymbols()); return new MarkDistinctNode(node.getId(), source, canonicalize(node.getMarkerSymbol()), symbols, canonicalize(node.getHashSymbol())); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, StreamPreferredProperties parentPreferences) { // mark distinct requires that all data partitioned StreamPreferredProperties childRequirements = parentPreferences .constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce(node.getSource(), childRequirements, childRequirements); MarkDistinctNode result = new MarkDistinctNode( node.getId(), child.getNode(), node.getMarkerSymbol(), pruneMarkDistinctSymbols(node, child.getProperties().getLocalProperties()), node.getHashSymbol()); return deriveProperties(result, child.getProperties()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, StreamPreferredProperties parentPreferences) { // mark distinct requires that all data partitioned StreamPreferredProperties childRequirements = parentPreferences .constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce(node.getSource(), childRequirements, childRequirements); MarkDistinctNode result = new MarkDistinctNode( node.getId(), child.getNode(), node.getMarkerSymbol(), pruneMarkDistinctSymbols(node, child.getProperties().getLocalProperties()), node.getHashSymbol()); return deriveProperties(result, child.getProperties()); }