public boolean isStreamPartitionedOn(Collection<Symbol> columns) { return isStreamPartitionedOn(columns, false); }
public boolean isStreamPartitionedOn(Collection<Symbol> columns, boolean nullsAndAnyReplicated) { return global.isStreamPartitionedOn(columns, constants.keySet(), nullsAndAnyReplicated); }
private boolean shouldRepartitionForIndexJoin(List<Symbol> joinColumns, PreferredProperties parentPreferredProperties, ActualProperties probeProperties) { // See if distributed index joins are enabled if (!distributedIndexJoins) { return false; } // No point in repartitioning if the plan is not distributed if (probeProperties.isSingleNode()) { return false; } Optional<PreferredProperties.PartitioningProperties> parentPartitioningPreferences = parentPreferredProperties.getGlobalProperties() .flatMap(PreferredProperties.Global::getPartitioningProperties); // Disable repartitioning if it would disrupt a parent's partitioning preference when streaming is enabled boolean parentAlreadyPartitionedOnChild = parentPartitioningPreferences .map(partitioning -> probeProperties.isStreamPartitionedOn(partitioning.getPartitioningColumns())) .orElse(false); if (preferStreamingOperators && parentAlreadyPartitionedOnChild) { return false; } // Otherwise, repartition if we need to align with the join columns if (!probeProperties.isStreamPartitionedOn(joinColumns)) { return true; } // If we are already partitioned on the join columns because the data has been forced effectively into one stream, // then we should repartition if that would make a difference (from the single stream state). return probeProperties.isEffectivelySingleStream() && probeProperties.isStreamRepartitionEffective(joinColumns); }
private static boolean meetsPartitioningRequirements(PreferredProperties preferred, ActualProperties actual) { if (!preferred.getGlobalProperties().isPresent()) { return true; } PreferredProperties.Global preferredGlobal = preferred.getGlobalProperties().get(); if (!preferredGlobal.isDistributed()) { return actual.isSingleNode(); } if (!preferredGlobal.getPartitioningProperties().isPresent()) { return !actual.isSingleNode(); } return actual.isStreamPartitionedOn(preferredGlobal.getPartitioningProperties().get().getPartitioningColumns()); }
@Override public PlanWithProperties visitWindow(WindowNode node, PreferredProperties preferredProperties) { List<LocalProperty<Symbol>> desiredProperties = new ArrayList<>(); if (!node.getPartitionBy().isEmpty()) { desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> orderingScheme.getOrderBy().stream() .map(symbol -> new SortingProperty<>(symbol, orderingScheme.getOrdering(symbol))) .forEach(desiredProperties::add)); PlanWithProperties child = planChild( node, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties) .mergeWithParent(preferredProperties)); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } } return rebaseAndDeriveProperties(node, child); }
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, PreferredProperties preferredProperties) { PreferredProperties preferredChildProperties = PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())) .mergeWithParent(preferredProperties); PlanWithProperties child = node.getSource().accept(this, preferredChildProperties); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) {
@Override public PlanWithProperties visitAggregation(AggregationNode node, PreferredProperties parentPreferredProperties) { Set<Symbol> partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionRegistry()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { preferredProperties = PreferredProperties.partitionedWithLocal(partitioningRequirement, grouped(node.getGroupingKeys())) .mergeWithParent(parentPreferredProperties); } PlanWithProperties child = planChild(node, preferredProperties); if (child.getProperties().isSingleNode()) { // If already unpartitioned, just drop the single aggregation back on return rebaseAndDeriveProperties(node, child); } if (preferSingleNode) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), REMOTE, child.getNode()), child.getProperties()); } else if (!child.getProperties().isStreamPartitionedOn(partitioningRequirement) && !child.getProperties().isNodePartitionedOn(partitioningRequirement)) { child = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, child.getNode(), node.getGroupingKeys(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy()) && !child.getProperties().isNodePartitionedOn(node.getPartitionBy())) { child = withDerivedProperties(
private boolean shouldRepartitionForIndexJoin(List<Symbol> joinColumns, PreferredProperties parentPreferredProperties, ActualProperties probeProperties) { // See if distributed index joins are enabled if (!distributedIndexJoins) { return false; } // No point in repartitioning if the plan is not distributed if (probeProperties.isSingleNode()) { return false; } Optional<PreferredProperties.Partitioning> parentPartitioningPreferences = parentPreferredProperties.getGlobalProperties() .flatMap(PreferredProperties.Global::getPartitioningProperties); // Disable repartitioning if it would disrupt a parent's partitioning preference when streaming is enabled boolean parentAlreadyPartitionedOnChild = parentPartitioningPreferences .map(partitioning -> probeProperties.isStreamPartitionedOn(partitioning.getPartitioningColumns())) .orElse(false); if (preferStreamingOperators && parentAlreadyPartitionedOnChild) { return false; } // Otherwise, repartition if we need to align with the join columns if (!probeProperties.isStreamPartitionedOn(joinColumns)) { return true; } // If we are already partitioned on the join columns because the data has been forced effectively into one stream, // then we should repartition if that would make a difference (from the single stream state). return probeProperties.isEffectivelySingleStream() && probeProperties.isStreamRepartitionEffective(joinColumns); }
private static boolean meetsPartitioningRequirements(PreferredProperties preferred, ActualProperties actual) { if (!preferred.getGlobalProperties().isPresent()) { return true; } PreferredProperties.Global preferredGlobal = preferred.getGlobalProperties().get(); if (!preferredGlobal.isDistributed()) { return actual.isSingleNode(); } if (!preferredGlobal.getPartitioningProperties().isPresent()) { return !actual.isSingleNode(); } return actual.isStreamPartitionedOn(preferredGlobal.getPartitioningProperties().get().getPartitioningColumns()); }
context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(node.getPartitionBy()), desiredProperties))); if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy())) { if (node.getPartitionBy().isEmpty()) { child = withDerivedProperties(
@Override public PlanWithProperties visitMarkDistinct(MarkDistinctNode node, Context context) { PreferredProperties preferredChildProperties = PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(node.getDistinctSymbols()), Optional.of(node.getDistinctSymbols()), grouped(node.getDistinctSymbols())); PlanWithProperties child = node.getSource().accept(this, context.withPreferredProperties(preferredChildProperties)); if (child.getProperties().isSingleNode() || !child.getProperties().isStreamPartitionedOn(node.getDistinctSymbols())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), child.getNode(), node.getDistinctSymbols(), node.getHashSymbol()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); }
if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy())) {
@Override public PlanWithProperties visitRowNumber(RowNumberNode node, Context context) { if (node.getPartitionBy().isEmpty()) { PlanWithProperties child = planChild(node, context.withPreferredProperties(PreferredProperties.undistributed())); if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( gatheringExchange(idAllocator.getNextId(), child.getNode()), child.getProperties()); } return rebaseAndDeriveProperties(node, child); } PlanWithProperties child = planChild(node, context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(node.getPartitionBy()), grouped(node.getPartitionBy())))); // TODO: add config option/session property to force parallel plan if child is unpartitioned and window has a PARTITION BY clause if (!child.getProperties().isStreamPartitionedOn(node.getPartitionBy())) { child = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), child.getNode(), node.getPartitionBy(), node.getHashSymbol()), child.getProperties()); } // TODO: streaming return rebaseAndDeriveProperties(node, child); }