public static StreamPreferredProperties exactlyPartitionedOn(Collection<Symbol> partitionSymbols) { if (partitionSymbols.isEmpty()) { return singleStream(); } // this must be the exact partitioning symbols, in the exact order return new StreamPreferredProperties(Optional.of(FIXED), true, Optional.of(ImmutableList.copyOf(partitionSymbols)), false); }
@Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, StreamPreferredProperties parentPreferences) { // Although explain analyze discards all output, we want to maintain the behavior // of a normal output node, so declare the node to be order sensitive return planAndEnforceChildren( node, singleStream().withOrderSensitivity(), singleStream().withOrderSensitivity()); }
public StreamPreferredProperties withPartitioning(Collection<Symbol> partitionSymbols) { if (partitionSymbols.isEmpty()) { return singleStream(); } Iterable<Symbol> desiredPartitioning = partitionSymbols; if (partitioningColumns.isPresent()) { if (exactColumnOrder) { if (partitioningColumns.get().equals(desiredPartitioning)) { return this; } } else { // If there are common columns between our requirements and the desired partitionSymbols, both can be satisfied in one shot Set<Symbol> common = Sets.intersection(ImmutableSet.copyOf(desiredPartitioning), ImmutableSet.copyOf(partitioningColumns.get())); // If we find common partitioning columns, use them, else use child's partitioning columns if (!common.isEmpty()) { desiredPartitioning = common; } } } return new StreamPreferredProperties(distribution, Optional.of(desiredPartitioning), false); }
@Override public PlanWithProperties visitEnforceSingleRow(EnforceSingleRowNode node, StreamPreferredProperties parentPreferences) { return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); }
@Override public PlanWithProperties visitTableFinish(TableFinishNode node, StreamPreferredProperties parentPreferences) { // table commit requires that all data be in one stream // this node changes the input organization completely, so we do not pass through parent preferences return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); }
@Override public PlanWithProperties visitSpatialJoin(SpatialJoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties probe = planAndEnforce( node.getLeft(), defaultParallelism(session), parentPreferences.constrainTo(node.getLeft().getOutputSymbols()) .withDefaultParallelism(session)); PlanWithProperties build = planAndEnforce(node.getRight(), singleStream(), singleStream()); return rebaseAndDeriveProperties(node, ImmutableList.of(probe, build)); }
@Override public PlanWithProperties visitSemiJoin(SemiJoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties source = planAndEnforce( node.getSource(), defaultParallelism(session), parentPreferences.constrainTo(node.getSource().getOutputSymbols()).withDefaultParallelism(session)); // this filter source consumes the input completely, so we do not pass through parent preferences PlanWithProperties filteringSource = planAndEnforce(node.getFilteringSource(), singleStream(), singleStream()); return rebaseAndDeriveProperties(node, ImmutableList.of(source, filteringSource)); }
@Override public PlanWithProperties visitTableWriter(TableWriterNode node, StreamPreferredProperties parentPreferences) { StreamPreferredProperties requiredProperties; StreamPreferredProperties preferredProperties; if (getTaskWriterCount(session) > 1) { requiredProperties = fixedParallelism(); preferredProperties = fixedParallelism(); } else { requiredProperties = singleStream(); preferredProperties = defaultParallelism(session); } return planAndEnforceChildren(node, requiredProperties, preferredProperties); }
@Override public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, StreamPreferredProperties parentPreferences) { // final limit requires that all data be in one stream StreamPreferredProperties requiredProperties; StreamPreferredProperties preferredProperties; if (node.isPartial()) { requiredProperties = parentPreferences.withoutPreference().withDefaultParallelism(session); preferredProperties = parentPreferences.withDefaultParallelism(session); } else { // a final changes the input organization completely, so we do not pass through parent preferences requiredProperties = singleStream(); preferredProperties = defaultParallelism(session); } return planAndEnforceChildren(node, requiredProperties, preferredProperties); }
@Override public PlanWithProperties visitLimit(LimitNode node, StreamPreferredProperties parentPreferences) { if (node.isPartial()) { return planAndEnforceChildren( node, parentPreferences.withoutPreference().withDefaultParallelism(session), parentPreferences.withDefaultParallelism(session)); } // final limit requires that all data be in one stream // also, a final changes the input organization completely, so we do not pass through parent preferences return planAndEnforceChildren( node, singleStream(), defaultParallelism(session)); }
@Override public PlanWithProperties visitTopN(TopNNode node, StreamPreferredProperties parentPreferences) { if (node.getStep().equals(TopNNode.Step.PARTIAL)) { return planAndEnforceChildren( node, parentPreferences.withoutPreference().withDefaultParallelism(session), parentPreferences.withDefaultParallelism(session)); } // final topN requires that all data be in one stream // also, a final changes the input organization completely, so we do not pass through parent preferences return planAndEnforceChildren( node, singleStream(), defaultParallelism(session)); }
@Override public PlanWithProperties visitJoin(JoinNode node, StreamPreferredProperties parentPreferences) { PlanWithProperties probe; if (isSpillEnabled(session)) { probe = planAndEnforce( node.getLeft(), fixedParallelism(), parentPreferences.constrainTo(node.getLeft().getOutputSymbols()).withFixedParallelism()); } else { probe = planAndEnforce( node.getLeft(), defaultParallelism(session), parentPreferences.constrainTo(node.getLeft().getOutputSymbols()).withDefaultParallelism(session)); } // this build consumes the input completely, so we do not pass through parent preferences List<Symbol> buildHashSymbols = Lists.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight); StreamPreferredProperties buildPreference; if (getTaskConcurrency(session) > 1) { buildPreference = exactlyPartitionedOn(buildHashSymbols); } else { buildPreference = singleStream(); } PlanWithProperties build = planAndEnforce(node.getRight(), buildPreference, buildPreference); return rebaseAndDeriveProperties(node, ImmutableList.of(probe, build)); }
return planAndEnforceChildren(node, singleStream(), defaultParallelism(session));
@Override public PlanWithProperties visitSort(SortNode node, StreamPreferredProperties parentPreferences) { if (isDistributedSortEnabled(session)) { PlanWithProperties sortPlan = planAndEnforceChildren(node, fixedParallelism(), fixedParallelism()); if (!sortPlan.getProperties().isSingleStream()) { return deriveProperties( mergingExchange( idAllocator.getNextId(), LOCAL, sortPlan.getNode(), node.getOrderingScheme()), sortPlan.getProperties()); } return sortPlan; } // sort requires that all data be in one stream // this node changes the input organization completely, so we do not pass through parent preferences return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); }