private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( scheme.getPartitioning().translate(this::map), mapAndDistinct(source.getOutputSymbols()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), scheme.getBucketToPartition()); }
checkArgument(inputs.stream().allMatch(inputSymbols -> inputSymbols.size() == partitioningScheme.getOutputLayout().size()), "Input symbols do not match output symbols"); checkArgument(inputs.size() == sources.size(), "Must have same number of input lists as sources"); for (int i = 0; i < inputs.size(); i++) { checkArgument(scope != LOCAL || partitioningScheme.getPartitioning().getArguments().stream().allMatch(ArgumentBinding::isVariable), "local exchanges do not support constant partition function arguments"); checkArgument(scope != REMOTE || type == Type.REPARTITION || !partitioningScheme.isReplicateNullsAndAny(), "Only REPARTITION can replicate remotely"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); checkArgument(scope != REMOTE || partitioningHandle.equals(SINGLE_DISTRIBUTION), "remote merging exchange requires single distribution"); checkArgument(scope != LOCAL || partitioningHandle.equals(FIXED_PASSTHROUGH_DISTRIBUTION), "local merging exchange requires passthrough distribution"); checkArgument(partitioningScheme.getOutputLayout().containsAll(ordering.getOrderBy()), "Partitioning scheme does not supply all required ordering symbols"); }); this.type = type;
public PlanFragment withBucketToPartition(Optional<int[]> bucketToPartition) { return new PlanFragment(id, root, symbols, partitioning, partitionedSources, partitioningScheme.withBucketToPartition(bucketToPartition), stageExecutionDescriptor, statsAndCosts); }
public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanNode child, PartitioningScheme partitioningScheme) { if (partitioningScheme.getPartitioning().getHandle().isSingleNode()) { return gatheringExchange(id, scope, child); } return new ExchangeNode( id, ExchangeNode.Type.REPARTITION, scope, partitioningScheme, ImmutableList.of(child), ImmutableList.of(partitioningScheme.getOutputLayout()).asList(), Optional.empty()); }
builder.append(indentString(1)) .append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout()))); boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny(); List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { if (replicateNullsAndAny) { builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn()))); partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
public PartitioningScheme withBucketToPartition(Optional<int[]> bucketToPartition) { return new PartitioningScheme(partitioning, outputLayout, hashColumn, replicateNullsAndAny, bucketToPartition); }
node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols()));
@Override public List<Symbol> getOutputSymbols() { return partitioningScheme.getOutputLayout(); }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(Symbol::toSymbolReference) .map(ArgumentBinding::expressionBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
@Override public PlanNode visitTableWriter(TableWriterNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getColumns()); if (node.getPartitioningScheme().isPresent()) { PartitioningScheme partitioningScheme = node.getPartitioningScheme().get(); partitioningScheme.getPartitioning().getColumns().forEach(expectedInputs::add); partitioningScheme.getHashColumn().ifPresent(expectedInputs::add); } if (node.getStatisticsAggregation().isPresent()) { StatisticAggregations aggregations = node.getStatisticsAggregation().get(); expectedInputs.addAll(aggregations.getGroupingSymbols()); aggregations.getAggregations().values().forEach(aggregation -> expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall()))); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new TableWriterNode( node.getId(), source, node.getTarget(), node.getRowCountSymbol(), node.getFragmentSymbol(), node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), node.getStatisticsAggregation(), node.getStatisticsAggregationDescriptor()); }
@Override public PlanWithProperties visitTableWriter(TableWriterNode node, PreferredProperties preferredProperties) { PlanWithProperties source = node.getSource().accept(this, preferredProperties); Optional<PartitioningScheme> partitioningScheme = node.getPartitioningScheme(); if (!partitioningScheme.isPresent()) { if (scaleWriters) { partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(SCALED_WRITER_DISTRIBUTION, ImmutableList.of()), source.getNode().getOutputSymbols())); } else if (redistributeWrites) { partitioningScheme = Optional.of(new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), source.getNode().getOutputSymbols())); } } if (partitioningScheme.isPresent() && !source.getProperties().isCompatibleTablePartitioningWith(partitioningScheme.get().getPartitioning(), false, metadata, session)) { source = withDerivedProperties( partitionedExchange( idAllocator.getNextId(), REMOTE, source.getNode(), partitioningScheme.get()), source.getProperties()); } return rebaseAndDeriveProperties(node, source); }
public PartitionFunction getPartitionFunction( Session session, PartitioningScheme partitioningScheme, List<Type> partitionChannelTypes) { Optional<int[]> bucketToPartition = partitioningScheme.getBucketToPartition(); checkArgument(bucketToPartition.isPresent(), "Bucket to partition must be set before a partition function can be created"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); BucketFunction bucketFunction; if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) { checkArgument(partitioningScheme.getBucketToPartition().isPresent(), "Bucket to partition must be set before a partition function can be created"); return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getPartitionFunction( partitionChannelTypes, partitioningScheme.getHashColumn().isPresent(), partitioningScheme.getBucketToPartition().get()); } else { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle(), partitionChannelTypes, bucketToPartition.get().length); checkArgument(bucketFunction != null, "No function %s", partitioningHandle); } return new BucketPartitionFunction(bucketFunction, partitioningScheme.getBucketToPartition().get()); }
public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set<SqlStageExecution> children) { this.currentStageFragmentId = fragmentId; this.parent = parent; this.childOutputBufferManagers = children.stream() .map(childStage -> { PartitioningHandle partitioningHandle = childStage.getFragment().getPartitioningScheme().getPartitioning().getHandle(); if (partitioningHandle.equals(FIXED_BROADCAST_DISTRIBUTION)) { return new BroadcastOutputBufferManager(childStage::setOutputBuffers); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { return new ScaledOutputBufferManager(childStage::setOutputBuffers); } else { int partitionCount = Ints.max(childStage.getFragment().getPartitioningScheme().getBucketToPartition().get()) + 1; return new PartitionedOutputBufferManager(partitioningHandle, partitionCount, childStage::setOutputBuffers); } }) .collect(toImmutableSet()); this.childStageIds = children.stream() .map(SqlStageExecution::getStageId) .collect(toImmutableSet()); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
boolean coordinatorOnly = node.getPartitioningScheme().getPartitioning().getHandle().isCoordinatorOnly(); return ActualProperties.builder() .global(coordinatorOnly ? coordinatorSingleStreamPartition() : singleStreamPartition()) return ActualProperties.builder() .global(partitionedOn( node.getPartitioningScheme().getPartitioning(), Optional.of(node.getPartitioningScheme().getPartitioning())) .withReplicatedNulls(node.getPartitioningScheme().isReplicateNullsAndAny())) .constants(constants) .build();
builder.append(indentString(1)) .append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout()))); boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny(); List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { if (replicateNullsAndAny) { builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn()))); partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
public PartitioningScheme withBucketToPartition(Optional<int[]> bucketToPartition) { return new PartitioningScheme(partitioning, outputLayout, hashColumn, replicateNullsAndAny, bucketToPartition); }
node.getPartitioningScheme().getPartitioning().getHandle(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), Joiner.on(", ").join(node.getPartitioningScheme().getPartitioning().getArguments()), formatOutputs(node.getOutputSymbols())); UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, node.getScope().toString()), node.getType(), node.getPartitioningScheme().isReplicateNullsAndAny() ? " - REPLICATE NULLS AND ANY" : "", formatHash(node.getPartitioningScheme().getHashColumn()), formatOutputs(node.getOutputSymbols()));
@Override public List<Symbol> getOutputSymbols() { return partitioningScheme.getOutputLayout(); }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(ArgumentBinding::columnBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }