private Global(Optional<Partitioning> nodePartitioning, Optional<Partitioning> streamPartitioning, boolean nullsAndAnyReplicated) { checkArgument(!nodePartitioning.isPresent() || !streamPartitioning.isPresent() || nodePartitioning.get().getColumns().containsAll(streamPartitioning.get().getColumns()) || streamPartitioning.get().getColumns().containsAll(nodePartitioning.get().getColumns()), "Global stream partitioning columns should match node partitioning columns"); this.nodePartitioning = requireNonNull(nodePartitioning, "nodePartitioning is null"); this.streamPartitioning = requireNonNull(streamPartitioning, "streamPartitioning is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; }
private Global(Optional<Partitioning> nodePartitioning, Optional<Partitioning> streamPartitioning, boolean nullsAndAnyReplicated) { checkArgument(!nodePartitioning.isPresent() || !streamPartitioning.isPresent() || nodePartitioning.get().getColumns().containsAll(streamPartitioning.get().getColumns()) || streamPartitioning.get().getColumns().containsAll(nodePartitioning.get().getColumns()), "Global stream partitioning columns should match node partitioning columns"); this.nodePartitioning = requireNonNull(nodePartitioning, "nodePartitioning is null"); this.streamPartitioning = requireNonNull(streamPartitioning, "streamPartitioning is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; }
private PartitioningProperties(Set<Symbol> partitioningColumns, Optional<Partitioning> partitioning, boolean nullsAndAnyReplicated) { this.partitioningColumns = ImmutableSet.copyOf(requireNonNull(partitioningColumns, "partitioningColumns is null")); this.partitioning = requireNonNull(partitioning, "function is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; checkArgument(!partitioning.isPresent() || partitioning.get().getColumns().equals(partitioningColumns), "Partitioning input must match partitioningColumns"); }
private PartitioningProperties(Set<Symbol> partitioningColumns, Optional<Partitioning> partitioning, boolean nullsAndAnyReplicated) { this.partitioningColumns = ImmutableSet.copyOf(requireNonNull(partitioningColumns, "partitioningColumns is null")); this.partitioning = requireNonNull(partitioning, "function is null"); this.nullsAndAnyReplicated = nullsAndAnyReplicated; checkArgument(!partitioning.isPresent() || partitioning.get().getColumns().equals(partitioningColumns), "Partitioning input must match partitioningColumns"); }
public static PartitioningProperties partitioned(Partitioning partitioning) { return new PartitioningProperties(partitioning.getColumns(), Optional.of(partitioning), false); }
public static PartitioningProperties partitioned(Partitioning partitioning) { return new PartitioningProperties(partitioning.getColumns(), Optional.of(partitioning), false); }
@JsonCreator public PartitioningScheme( @JsonProperty("partitioning") Partitioning partitioning, @JsonProperty("outputLayout") List<Symbol> outputLayout, @JsonProperty("hashColumn") Optional<Symbol> hashColumn, @JsonProperty("replicateNullsAndAny") boolean replicateNullsAndAny, @JsonProperty("bucketToPartition") Optional<int[]> bucketToPartition) { this.partitioning = requireNonNull(partitioning, "partitioning is null"); this.outputLayout = ImmutableList.copyOf(requireNonNull(outputLayout, "outputLayout is null")); Set<Symbol> columns = partitioning.getColumns(); checkArgument(ImmutableSet.copyOf(outputLayout).containsAll(columns), "Output layout (%s) don't include all partition columns (%s)", outputLayout, columns); this.hashColumn = requireNonNull(hashColumn, "hashColumn is null"); hashColumn.ifPresent(column -> checkArgument(outputLayout.contains(column), "Output layout (%s) don't include hash column (%s)", outputLayout, column)); checkArgument(!replicateNullsAndAny || columns.size() <= 1, "Must have at most one partitioning column when nullPartition is REPLICATE."); this.replicateNullsAndAny = replicateNullsAndAny; this.bucketToPartition = requireNonNull(bucketToPartition, "bucketToPartition is null"); }
@JsonCreator public PartitioningScheme( @JsonProperty("partitioning") Partitioning partitioning, @JsonProperty("outputLayout") List<Symbol> outputLayout, @JsonProperty("hashColumn") Optional<Symbol> hashColumn, @JsonProperty("replicateNullsAndAny") boolean replicateNullsAndAny, @JsonProperty("bucketToPartition") Optional<int[]> bucketToPartition) { this.partitioning = requireNonNull(partitioning, "partitioning is null"); this.outputLayout = ImmutableList.copyOf(requireNonNull(outputLayout, "outputLayout is null")); Set<Symbol> columns = partitioning.getColumns(); checkArgument(ImmutableSet.copyOf(outputLayout).containsAll(columns), "Output layout (%s) don't include all partition columns (%s)", outputLayout, columns); this.hashColumn = requireNonNull(hashColumn, "hashColumn is null"); hashColumn.ifPresent(column -> checkArgument(outputLayout.contains(column), "Output layout (%s) don't include hash column (%s)", outputLayout, column)); checkArgument(!replicateNullsAndAny || columns.size() <= 1, "Must have at most one partitioning column when nullPartition is REPLICATE."); this.replicateNullsAndAny = replicateNullsAndAny; this.bucketToPartition = requireNonNull(bucketToPartition, "bucketToPartition is null"); }
public static ActualProperties deriveProperties(PlanNode node, List<ActualProperties> inputProperties, Metadata metadata, Session session, TypeProvider types, SqlParser parser) { ActualProperties output = node.accept(new Visitor(metadata, session, types, parser), inputProperties); output.getNodePartitioning().ifPresent(partitioning -> verify(node.getOutputSymbols().containsAll(partitioning.getColumns()), "Node-level partitioning properties contain columns not present in node's output")); verify(node.getOutputSymbols().containsAll(output.getConstants().keySet()), "Node-level constant properties contain columns not present in node's output"); Set<Symbol> localPropertyColumns = output.getLocalProperties().stream() .flatMap(property -> property.getColumns().stream()) .collect(Collectors.toSet()); verify(node.getOutputSymbols().containsAll(localPropertyColumns), "Node-level local properties contain columns not present in node's output"); return output; }
public static ActualProperties deriveProperties(PlanNode node, List<ActualProperties> inputProperties, Metadata metadata, Session session, TypeProvider types, SqlParser parser) { ActualProperties output = node.accept(new Visitor(metadata, session, types, parser), inputProperties); output.getNodePartitioning().ifPresent(partitioning -> verify(node.getOutputSymbols().containsAll(partitioning.getColumns()), "Node-level partitioning properties contain columns not present in node's output")); verify(node.getOutputSymbols().containsAll(output.getConstants().keySet()), "Node-level constant properties contain columns not present in node's output"); Set<Symbol> localPropertyColumns = output.getLocalProperties().stream() .flatMap(property -> property.getColumns().stream()) .collect(Collectors.toSet()); verify(node.getOutputSymbols().containsAll(localPropertyColumns), "Node-level local properties contain columns not present in node's output"); return output; }
node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getOrderingScheme().ifPresent(orderingScheme -> expectedOutputSymbols.addAll(orderingScheme.getOrderBy()));
node.getPartitioningScheme().getPartitioning().getColumns().stream() .forEach(expectedOutputSymbols::add); node.getOrderingScheme().ifPresent(orderingScheme -> expectedOutputSymbols.addAll(orderingScheme.getOrderBy()));
@Override public PlanNode visitTableWriter(TableWriterNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getColumns()); if (node.getPartitioningScheme().isPresent()) { PartitioningScheme partitioningScheme = node.getPartitioningScheme().get(); partitioningScheme.getPartitioning().getColumns().forEach(expectedInputs::add); partitioningScheme.getHashColumn().ifPresent(expectedInputs::add); } if (node.getStatisticsAggregation().isPresent()) { StatisticAggregations aggregations = node.getStatisticsAggregation().get(); expectedInputs.addAll(aggregations.getGroupingSymbols()); aggregations.getAggregations().values().forEach(aggregation -> expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall()))); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new TableWriterNode( node.getId(), source, node.getTarget(), node.getRowCountSymbol(), node.getFragmentSymbol(), node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), node.getStatisticsAggregation(), node.getStatisticsAggregationDescriptor()); }
@Override public PlanNode visitTableWriter(TableWriterNode node, RewriteContext<Set<Symbol>> context) { ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(node.getColumns()); if (node.getPartitioningScheme().isPresent()) { PartitioningScheme partitioningScheme = node.getPartitioningScheme().get(); partitioningScheme.getPartitioning().getColumns().forEach(expectedInputs::add); partitioningScheme.getHashColumn().ifPresent(expectedInputs::add); } if (node.getStatisticsAggregation().isPresent()) { StatisticAggregations aggregations = node.getStatisticsAggregation().get(); expectedInputs.addAll(aggregations.getGroupingSymbols()); aggregations.getAggregations().values().forEach(aggregation -> expectedInputs.addAll(SymbolsExtractor.extractUnique(aggregation.getCall()))); } PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new TableWriterNode( node.getId(), source, node.getTarget(), node.getRowCountSymbol(), node.getFragmentSymbol(), node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), node.getStatisticsAggregation(), node.getStatisticsAggregationDescriptor()); }
Set<Symbol> partitioningColumns = exchange.getPartitioningScheme().getPartitioning().getColumns();
Set<Symbol> partitioningColumns = exchange.getPartitioningScheme().getPartitioning().getColumns();
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
@Override public Result apply(ExchangeNode node, Captures captures, Context context) { checkArgument(!node.getOrderingScheme().isPresent(), "Merge exchange over AssignUniqueId not supported"); AssignUniqueId assignUniqueId = captures.get(ASSIGN_UNIQUE_ID); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getColumns().contains(assignUniqueId.getIdColumn())) { // The column produced by the AssignUniqueId is used in the partitioning scheme of the exchange. // Hence, AssignUniqueId node has to stay below the exchange node. return Result.empty(); } return Result.ofPlanNode(new AssignUniqueId( assignUniqueId.getId(), new ExchangeNode( node.getId(), node.getType(), node.getScope(), new PartitioningScheme( partitioningScheme.getPartitioning(), removeSymbol(partitioningScheme.getOutputLayout(), assignUniqueId.getIdColumn()), partitioningScheme.getHashColumn(), partitioningScheme.isReplicateNullsAndAny(), partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeSymbol(getOnlyElement(node.getInputs()), assignUniqueId.getIdColumn())), Optional.empty()), assignUniqueId.getIdColumn())); }
Set<Symbol> partitioningColumns = partitioningScheme.getPartitioning().getColumns();
Partitioning nodePartitioning = probeProperties.getNodePartitioning().get(); ImmutableList.Builder<Expression> coalesceExpressions = ImmutableList.builder(); for (Symbol column : nodePartitioning.getColumns()) { for (JoinNode.EquiJoinClause equality : node.getCriteria()) { if (equality.getLeft().equals(column) || equality.getRight().equals(column)) {