public PartitionFunction getPartitionFunction( Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) { Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition(); checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); BucketFunction bucketFunction; if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) { checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created"); return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction( partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get()); } else { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length); checkArgument(bucketFunction != null, "No function %s", partitioningHandle); } return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get()); }
public PartitionFunction getPartitionFunction( Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) { Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition(); checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); BucketFunction bucketFunction; if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) { checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created"); return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction( partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get()); } else { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length); checkArgument(bucketFunction != null, "No function %s", partitioningHandle); } return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get()); }
private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( scheme.getPartitioning().translate(this::map), mapAndDistinct(source.getOutputSymbols()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), scheme.getBucketToPartition()); }
private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( scheme.getPartitioning().translate(this::map), mapAndDistinct(source.getOutputSymbols()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), scheme.getBucketToPartition()); }
public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set<SqlStageExecution> children) { this.currentStageFragmentId = fragmentId; this.parent = parent; this.childOutputBufferManagers = children.stream() .map(childStage -> { PartitioningHandle partitioningHandle = childStage.getFragment().getPartitioningScheme().getPartitioning().getHandle(); if (partitioningHandle.equals(FIXED_BROADCAST_DISTRIBUTION)) { return new BroadcastOutputBufferManager(childStage::setOutputBuffers); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { return new ScaledOutputBufferManager(childStage::setOutputBuffers); } else { int partitionCount = Ints.max(childStage.getFragment().getPartitioningScheme().getBucketToPartition().get()) + 1; return new PartitionedOutputBufferManager(partitioningHandle, partitionCount, childStage::setOutputBuffers); } }) .collect(toImmutableSet()); this.childStageIds = children.stream() .map(SqlStageExecution::getStageId) .collect(toImmutableSet()); }
canonicalize(node.getPartitioningScheme().getHashColumn()), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
canonicalize(node.getPartitioningScheme().getHashColumn()), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
node.getPartitioningScheme().getHashColumn(), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
node.getPartitioningScheme().getHashColumn(), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set<SqlStageExecution> children) { this.currentStageFragmentId = fragmentId; this.parent = parent; this.childOutputBufferManagers = children.stream() .map(childStage -> { PartitioningHandle partitioningHandle = childStage.getFragment().getPartitioningScheme().getPartitioning().getHandle(); if (partitioningHandle.equals(FIXED_BROADCAST_DISTRIBUTION)) { return new BroadcastOutputBufferManager(childStage::setOutputBuffers); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { return new ScaledOutputBufferManager(childStage::setOutputBuffers); } else { int partitionCount = Ints.max(childStage.getFragment().getPartitioningScheme().getBucketToPartition().get()) + 1; return new PartitionedOutputBufferManager(partitioningHandle, partitionCount, childStage::setOutputBuffers); } }) .collect(toImmutableSet()); this.childStageIds = children.stream() .map(SqlStageExecution::getStageId) .collect(toImmutableSet()); }
partitionSymbols.map(newHashSymbols::get), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition());
partitionSymbols.map(newHashSymbols::get), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition());
exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
outputPartitioningScheme.getHashColumn(), outputPartitioningScheme.isReplicateNullsAndAny(), outputPartitioningScheme.getBucketToPartition()), fragment.getStageExecutionStrategy(), fragment.getStatsAndCosts());
outputPartitioningScheme.getHashColumn(), outputPartitioningScheme.isReplicateNullsAndAny(), outputPartitioningScheme.getBucketToPartition()), fragment.getStageExecutionDescriptor(), fragment.getStatsAndCosts());