public static Global partitionedOn(PartitioningHandle nodePartitioningHandle, List<Symbol> nodePartitioning, Optional<List<Symbol>> streamPartitioning) { return new Global( Optional.of(Partitioning.create(nodePartitioningHandle, nodePartitioning)), streamPartitioning.map(columns -> Partitioning.create(SOURCE_DISTRIBUTION, columns)), false); }
/** * @return true if the plan will only execute on a single node */ private boolean isSingleNode() { if (!nodePartitioning.isPresent()) { return false; } return nodePartitioning.get().getHandle().isSingleNode(); }
public Partitioning withAlternativePartitiongingHandle(PartitioningHandle partitiongingHandle) { return new Partitioning(partitiongingHandle, this.arguments); }
checkArgument(scope != LOCAL || partitioningScheme.getPartitioning().getArguments().stream().allMatch(ArgumentBinding::isVariable), "local exchanges do not support constant partition function arguments"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); checkArgument(scope != REMOTE || partitioningHandle.equals(SINGLE_DISTRIBUTION), "remote merging exchange requires single distribution"); checkArgument(scope != LOCAL || partitioningHandle.equals(FIXED_PASSTHROUGH_DISTRIBUTION), "local merging exchange requires passthrough distribution");
if (partitioningScheme.getPartitioning().getHandle().equals(FIXED_BROADCAST_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(SCALED_WRITER_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(SINGLE_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(COORDINATOR_DISTRIBUTION)) { return plan(taskContext, stageExecutionStrategy, plan, outputLayout, types, partitionedSourceOrder, new TaskOutputFactory(outputBuffer)); partitionChannels = partitioningScheme.getPartitioning().getArguments().stream() .map(ArgumentBinding::getColumn) .map(outputLayout::indexOf) .collect(toImmutableList()); partitionConstants = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { }) .collect(toImmutableList()); partitionChannelTypes = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { Set<Symbol> partitioningColumns = partitioningScheme.getPartitioning().getColumns();
Partitioning nodePartitioning = probeProperties.getNodePartitioning().get(); ImmutableList.Builder<Expression> coalesceExpressions = ImmutableList.builder(); for (Symbol column : nodePartitioning.getColumns()) { for (JoinNode.EquiJoinClause equality : node.getCriteria()) { if (equality.getLeft().equals(column) || equality.getRight().equals(column)) { .global(partitionedOn(Partitioning.createWithExpressions(nodePartitioning.getHandle(), coalesceExpressions.build()), Optional.empty())) .unordered(unordered) .build();
if (outputPartitioningScheme.getPartitioning().getHandle().getConnectorId().isPresent()) { newOutputPartitioning = newOutputPartitioning.withAlternativePartitiongingHandle(newOutputPartitioningHandle);
private Global(Optional<Partitioning> nodePartitioning, Optional<Partitioning> streamPartitioning, boolean nullsAndAnyReplicated) { checkArgument(!nodePartitioning.isPresent() || !streamPartitioning.isPresent() || nodePartitioning.get().getColumns().containsAll(streamPartitioning.get().getColumns()) || streamPartitioning.get().getColumns().containsAll(nodePartitioning.get().getColumns()), "Global stream partitioning columns should match node partitioning columns"); this.nodePartitioning = requireNonNull(nodePartitioning, "nodePartitioning is null"); this.streamPartitioning = requireNonNull(streamPartitioning, "streamPartitioning is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(Symbol::toSymbolReference) .map(ArgumentBinding::expressionBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
public boolean isEffectivelySinglePartition(Set<Symbol> knownConstants) { return isPartitionedOn(ImmutableSet.of(), knownConstants); }
/** * @return true if all the data will effectively land in a single stream */ private boolean isEffectivelySingleStream(Set<Symbol> constants) { return streamPartitioning.isPresent() && streamPartitioning.get().isEffectivelySinglePartition(constants) && !nullsAndAnyReplicated; }
private boolean isCompatibleTablePartitioningWith( Global other, Function<Symbol, Set<Symbol>> symbolMappings, Function<Symbol, Optional<NullableValue>> leftConstantMapping, Function<Symbol, Optional<NullableValue>> rightConstantMapping, Metadata metadata, Session session) { return nodePartitioning.isPresent() && other.nodePartitioning.isPresent() && nodePartitioning.get().isCompatibleWith( other.nodePartitioning.get(), symbolMappings, leftConstantMapping, rightConstantMapping, metadata, session) && nullsAndAnyReplicated == other.nullsAndAnyReplicated; }
public boolean isCompatibleWith( Partitioning right, Function<Symbol, Set<Symbol>> leftToRightMappings, Function<Symbol, Optional<NullableValue>> leftConstantMapping, Function<Symbol, Optional<NullableValue>> rightConstantMapping, Metadata metadata, Session session) { if (!handle.equals(right.handle) && !metadata.getCommonPartitioning(session, handle, right.handle).isPresent()) { return false; } if (arguments.size() != right.arguments.size()) { return false; } for (int i = 0; i < arguments.size(); i++) { ArgumentBinding leftArgument = arguments.get(i); ArgumentBinding rightArgument = right.arguments.get(i); if (!isPartitionedWith(leftArgument, leftConstantMapping, rightArgument, rightConstantMapping, leftToRightMappings)) { return false; } } return true; }
checkArgument(scope != LOCAL || partitioningScheme.getPartitioning().getArguments().stream().allMatch(ArgumentBinding::isVariable), "local exchanges do not support constant partition function arguments"); PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); checkArgument(scope != REMOTE || partitioningHandle.equals(SINGLE_DISTRIBUTION), "remote merging exchange requires single distribution"); checkArgument(scope != LOCAL || partitioningHandle.equals(FIXED_PASSTHROUGH_DISTRIBUTION), "local merging exchange requires passthrough distribution");
if (partitioningScheme.getPartitioning().getHandle().equals(FIXED_BROADCAST_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(SCALED_WRITER_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(SINGLE_DISTRIBUTION) || partitioningScheme.getPartitioning().getHandle().equals(COORDINATOR_DISTRIBUTION)) { return plan(taskContext, stageExecutionDescriptor, plan, outputLayout, types, partitionedSourceOrder, new TaskOutputFactory(outputBuffer)); partitionChannels = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { }) .collect(toImmutableList()); partitionConstants = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { }) .collect(toImmutableList()); partitionChannelTypes = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { if (argument.isConstant()) { Set<Symbol> partitioningColumns = partitioningScheme.getPartitioning().getColumns();
if (outputPartitioningScheme.getPartitioning().getHandle().getConnectorId().isPresent()) { newOutputPartitioning = newOutputPartitioning.withAlternativePartitiongingHandle(newOutputPartitioningHandle);
private Global(Optional<Partitioning> nodePartitioning, Optional<Partitioning> streamPartitioning, boolean nullsAndAnyReplicated) { checkArgument(!nodePartitioning.isPresent() || !streamPartitioning.isPresent() || nodePartitioning.get().getColumns().containsAll(streamPartitioning.get().getColumns()) || streamPartitioning.get().getColumns().containsAll(nodePartitioning.get().getColumns()), "Global stream partitioning columns should match node partitioning columns"); this.nodePartitioning = requireNonNull(nodePartitioning, "nodePartitioning is null"); this.streamPartitioning = requireNonNull(streamPartitioning, "streamPartitioning is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; }
@Override public Void visitExchange(ExchangeNode node, Void context) { List<ArgumentBinding> symbols = node.getOutputSymbols().stream() .map(ArgumentBinding::columnBinding) .collect(toImmutableList()); if (node.getType() == REPARTITION) { symbols = node.getPartitioningScheme().getPartitioning().getArguments(); } String columns = Joiner.on(", ").join(symbols); printNode(node, format("ExchangeNode[%s]", node.getType()), columns, NODE_COLORS.get(NodeType.EXCHANGE)); for (PlanNode planNode : node.getSources()) { planNode.accept(this, context); } return null; }
public boolean isEffectivelySinglePartition(Set<Symbol> knownConstants) { return isPartitionedOn(ImmutableSet.of(), knownConstants); }
/** * @return true if all the data will effectively land in a single stream */ private boolean isEffectivelySingleStream(Set<Symbol> constants) { return streamPartitioning.isPresent() && streamPartitioning.get().isEffectivelySinglePartition(constants) && !nullsAndAnyReplicated; }