@Override public Map<PlanNodeId, SplitSource> visitMarkDistinct(MarkDistinctNode node, Void context) { return node.getSource().accept(this, context); }
@Override public Map<PlanNodeId, SplitSource> visitMarkDistinct(MarkDistinctNode node, Void context) { return node.getSource().accept(this, context); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, MarkDistinctNode markDistinctNode, Set<Symbol> referencedOutputs) { if (!referencedOutputs.contains(markDistinctNode.getMarkerSymbol())) { return Optional.of(markDistinctNode.getSource()); } Set<Symbol> requiredInputs = Streams.concat( referencedOutputs.stream() .filter(symbol -> !symbol.equals(markDistinctNode.getMarkerSymbol())), markDistinctNode.getDistinctSymbols().stream(), markDistinctNode.getHashSymbol().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); return restrictChildOutputs(idAllocator, markDistinctNode, requiredInputs); } }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Void context) { printNode(node, format("MarkDistinct[%s]", node.getMarkerSymbol()), format("%s => %s", node.getDistinctSymbols(), node.getMarkerSymbol()), NODE_COLORS.get(NodeType.MARK_DISTINCT)); return node.getSource().accept(this, context); }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Void context) { printNode(node, format("MarkDistinct[%s]", node.getMarkerSymbol()), format("%s => %s", node.getDistinctSymbols(), node.getMarkerSymbol()), NODE_COLORS.get(NodeType.MARK_DISTINCT)); return node.getSource().accept(this, context); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, MarkDistinctNode markDistinctNode, Set<Symbol> referencedOutputs) { if (!referencedOutputs.contains(markDistinctNode.getMarkerSymbol())) { return Optional.of(markDistinctNode.getSource()); } Set<Symbol> requiredInputs = Streams.concat( referencedOutputs.stream() .filter(symbol -> !symbol.equals(markDistinctNode.getMarkerSymbol())), markDistinctNode.getDistinctSymbols().stream(), markDistinctNode.getHashSymbol().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); return restrictChildOutputs(idAllocator, markDistinctNode, requiredInputs); } }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Set<Symbol>> context) { if (!context.get().contains(node.getMarkerSymbol())) { return context.rewrite(node.getSource(), context.get()); } ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getDistinctSymbols()) .addAll(context.get().stream() .filter(symbol -> !symbol.equals(node.getMarkerSymbol())) .collect(toImmutableList())); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new MarkDistinctNode(node.getId(), source, node.getMarkerSymbol(), node.getDistinctSymbols(), node.getHashSymbol()); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Set<Symbol>> context) { if (!context.get().contains(node.getMarkerSymbol())) { return context.rewrite(node.getSource(), context.get()); } ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getDistinctSymbols()) .addAll(context.get().stream() .filter(symbol -> !symbol.equals(node.getMarkerSymbol())) .collect(toImmutableList())); if (node.getHashSymbol().isPresent()) { expectedInputs.add(node.getHashSymbol().get()); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new MarkDistinctNode(node.getId(), source, node.getMarkerSymbol(), node.getDistinctSymbols(), node.getHashSymbol()); }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Set<Symbol> boundSymbols) { PlanNode source = node.getSource(); source.accept(this, boundSymbols); // visit child checkDependencies(source.getOutputSymbols(), node.getDistinctSymbols(), "Invalid node. Mark distinct symbols (%s) not in source plan output (%s)", node.getDistinctSymbols(), source.getOutputSymbols()); return null; }
@Override public Void visitMarkDistinct(MarkDistinctNode node, Set<Symbol> boundSymbols) { PlanNode source = node.getSource(); source.accept(this, boundSymbols); // visit child checkDependencies(source.getOutputSymbols(), node.getDistinctSymbols(), "Invalid node. Mark distinct symbols (%s) not in source plan output (%s)", node.getDistinctSymbols(), source.getOutputSymbols()); return null; }
@Override public PhysicalOperation visitMarkDistinct(MarkDistinctNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); List<Integer> channels = getChannelsForSymbols(node.getDistinctSymbols(), source.getLayout()); Optional<Integer> hashChannel = node.getHashSymbol().map(channelGetter(source)); MarkDistinctOperatorFactory operator = new MarkDistinctOperatorFactory(context.getNextOperatorId(), node.getId(), source.getTypes(), channels, hashChannel, joinCompiler); return new PhysicalOperation(operator, makeLayout(node), context, source); }
@Override public PhysicalOperation visitMarkDistinct(MarkDistinctNode node, LocalExecutionPlanContext context) { PhysicalOperation source = node.getSource().accept(this, context); List<Integer> channels = getChannelsForSymbols(node.getDistinctSymbols(), source.getLayout()); Optional<Integer> hashChannel = node.getHashSymbol().map(channelGetter(source)); MarkDistinctOperatorFactory operator = new MarkDistinctOperatorFactory(context.getNextOperatorId(), node.getId(), source.getTypes(), channels, hashChannel, joinCompiler); return new PhysicalOperation(operator, makeLayout(node), context, source); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); List<Symbol> symbols = canonicalizeAndDistinct(node.getDistinctSymbols()); return new MarkDistinctNode(node.getId(), source, canonicalize(node.getMarkerSymbol()), symbols, canonicalize(node.getHashSymbol())); }
@Override public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext<Void> context) { PlanNode source = context.rewrite(node.getSource()); List<Symbol> symbols = canonicalizeAndDistinct(node.getDistinctSymbols()); return new MarkDistinctNode(node.getId(), source, canonicalize(node.getMarkerSymbol()), symbols, canonicalize(node.getHashSymbol())); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, StreamPreferredProperties parentPreferences) { // mark distinct requires that all data partitioned StreamPreferredProperties childRequirements = parentPreferences .constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce(node.getSource(), childRequirements, childRequirements); MarkDistinctNode result = new MarkDistinctNode( node.getId(), child.getNode(), node.getMarkerSymbol(), pruneMarkDistinctSymbols(node, child.getProperties().getLocalProperties()), node.getHashSymbol()); return deriveProperties(result, child.getProperties()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, StreamPreferredProperties parentPreferences) { // mark distinct requires that all data partitioned StreamPreferredProperties childRequirements = parentPreferences .constrainTo(node.getSource().getOutputSymbols()) .withDefaultParallelism(session) .withPartitioning(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce(node.getSource(), childRequirements, childRequirements); MarkDistinctNode result = new MarkDistinctNode( node.getId(), child.getNode(), node.getMarkerSymbol(), pruneMarkDistinctSymbols(node, child.getProperties().getLocalProperties()), node.getHashSymbol()); return deriveProperties(result, child.getProperties()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, HashComputationSet parentPreference) { // skip hash symbol generation for single bigint if (canSkipHashGeneration(node.getDistinctSymbols())) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new MarkDistinctNode(node.getId(), child.getNode(), node.getMarkerSymbol(), node.getDistinctSymbols(), Optional.of(hashSymbol)), child.getHashSymbols()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, HashComputationSet parentPreference) { // skip hash symbol generation for single bigint if (canSkipHashGeneration(node.getDistinctSymbols())) { return planSimpleNodeWithProperties(node, parentPreference); } Optional<HashComputation> hashComputation = computeHash(node.getDistinctSymbols()); PlanWithProperties child = planAndEnforce( node.getSource(), new HashComputationSet(hashComputation), false, parentPreference.withHashComputation(node, hashComputation)); Symbol hashSymbol = child.getRequiredHashSymbol(hashComputation.get()); return new PlanWithProperties( new MarkDistinctNode(node.getId(), child.getNode(), node.getMarkerSymbol(), node.getDistinctSymbols(), Optional.of(hashSymbol)), child.getHashSymbols()); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }