public PartitionFunction getPartitionFunction( Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) { Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition(); checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); BucketFunction bucketFunction; if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) { checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created"); return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction( partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get()); } else { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length); checkArgument(bucketFunction != null, "No function %s", partitioningHandle); } return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get()); }
public PartitionFunction getPartitionFunction( Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) { Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition(); checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); BucketFunction bucketFunction; if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) { checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created"); return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction( partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get()); } else { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length); checkArgument(bucketFunction != null, "No function %s", partitioningHandle); } return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get()); }
node.getPartitioningScheme().getHashColumn().ifPresent(expectedOutputSymbols::add); node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getPartitioningScheme().getPartitioning(), newOutputSymbols, node.getPartitioningScheme().getHashColumn(), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( scheme.getPartitioning().translate(this::map), mapAndDistinct(source.getOutputSymbols()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), scheme.getBucketToPartition()); }
node.getPartitioningScheme().getHashColumn().ifPresent(expectedOutputSymbols::add); node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getPartitioningScheme().getPartitioning(), newOutputSymbols, node.getPartitioningScheme().getHashColumn(), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( scheme.getPartitioning().translate(this::map), mapAndDistinct(source.getOutputSymbols()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), scheme.getBucketToPartition()); }
node.getPartitioningScheme().getPartitioning().translate(this::canonicalize), outputs.build(), canonicalize(node.getPartitioningScheme().getHashColumn()), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
node.getPartitioningScheme().getPartitioning().translate(this::canonicalize), outputs.build(), canonicalize(node.getPartitioningScheme().getHashColumn()), node.getPartitioningScheme().isReplicateNullsAndAny(), node.getPartitioningScheme().getBucketToPartition());
@Override public PlanNode visitTableWriter(TableWriterNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getColumns()); if (node.getPartitioningScheme().isPresent()) { PartitioningScheme partitioningScheme = node.getPartitioningScheme().get(); partitioningScheme.getPartitioning().getColumns().forEach(expectedInputs::add); partitioningScheme.getHashColumn().ifPresent(expectedInputs::add); } if (node.getStatisticsAggregation().isPresent()) { StatisticAggregations aggregations = node.getStatisticsAggregation().get(); expectedInputs.addAll(aggregations.getGroupingSymbols()); aggregations.getAggregations().values().forEach(aggregation -> expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall()))); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new TableWriterNode( node.getId(), source, node.getTarget(), node.getRowCountSymbol(), node.getFragmentSymbol(), node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), node.getStatisticsAggregation(), node.getStatisticsAggregationDescriptor()); }
partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn()))); partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
@Override public PlanNode visitTableWriter(TableWriterNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getColumns()); if (node.getPartitioningScheme().isPresent()) { PartitioningScheme partitioningScheme = node.getPartitioningScheme().get(); partitioningScheme.getPartitioning().getColumns().forEach(expectedInputs::add); partitioningScheme.getHashColumn().ifPresent(expectedInputs::add); } if (node.getStatisticsAggregation().isPresent()) { StatisticAggregations aggregations = node.getStatisticsAggregation().get(); expectedInputs.addAll(aggregations.getGroupingSymbols()); aggregations.getAggregations().values().forEach(aggregation -> expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall()))); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new TableWriterNode( node.getId(), source, node.getTarget(), node.getRowCountSymbol(), node.getFragmentSymbol(), node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), node.getStatisticsAggregation(), node.getStatisticsAggregationDescriptor()); }
node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols()));
node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols()));
exchange.getPartitioningScheme().getPartitioning(), aggregation.getOutputSymbols(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
exchange.getPartitioningScheme().getPartitioning(), aggregation.getOutputSymbols(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
if (aggregationNode.getHashSymbol().isPresent() || exchangeNode.getPartitioningScheme().getHashColumn().isPresent()) { return Result.empty();
newOutputPartitioning, outputPartitioningScheme.getOutputLayout(), outputPartitioningScheme.getHashColumn(), outputPartitioningScheme.isReplicateNullsAndAny(), outputPartitioningScheme.getBucketToPartition()),
newOutputPartitioning, outputPartitioningScheme.getOutputLayout(), outputPartitioningScheme.getHashColumn(), outputPartitioningScheme.isReplicateNullsAndAny(), outputPartitioningScheme.getBucketToPartition()),