private DistributionTrait convertDist(DistributionTrait srcDist, Map<Integer, Integer> inToOut) { List<DistributionField> newFields = Lists.newArrayList(); for (DistributionField field : srcDist.getFields()) { if (inToOut.containsKey(field.getFieldId())) { newFields.add(new DistributionField(inToOut.get(field.getFieldId()))); } } // After the projection, if the new distribution fields is empty, or new distribution fields is a subset of // original distribution field, we should replace with either SINGLETON or RANDOM_DISTRIBUTED. if (newFields.isEmpty() || newFields.size() < srcDist.getFields().size()) { if (srcDist.getType() != DistributionType.SINGLETON) { return DistributionTrait.ANY; } else { return DistributionTrait.SINGLETON; } } else { return new DistributionTrait(srcDist.getType(), ImmutableList.copyOf(newFields)); } }
@Override public boolean satisfies(RelTrait trait) { if (trait instanceof DistributionTrait) { DistributionType requiredDist = ((DistributionTrait) trait).getType(); if (requiredDist == DistributionType.ANY) { return true; } if (this.type == DistributionType.HASH_DISTRIBUTED) { if (requiredDist == DistributionType.HASH_DISTRIBUTED) { // A subset of the required distribution columns can satisfy (subsume) the requirement // e.g: required distribution: {a, b, c} // Following can satisfy the requirements: {a}, {b}, {c}, {a, b}, {b, c}, {a, c} or {a, b, c} // New: Use equals for subsumes check of hash distribution. If we uses subsumes, // a join may end up with hash-distributions using different keys. This would // cause incorrect query result. return this.equals(trait); } } } return this.equals(trait); }
if(distribution.getType() != DistributionType.HASH_DISTRIBUTED){ throw UserException.planError().message("Tried to plan a distribution writer but distribution was incorrect.").build(logger); if(distribution.getFields().size() != options.getDistributionColumns().size()){ Prel project = HashPrelUtil.addHashProject(distribution.getFields(), input, options.getRingCount());
private static DistributionTrait hashDistributedOn(final List<String> columns, final RelDataType inputRowType) { return new DistributionTrait(DistributionType.HASH_DISTRIBUTED, FluentIterable.from(WriterUpdater.getFieldIndices(columns, inputRowType)) .transform(new Function<Integer, DistributionField>() { @Override public DistributionField apply(Integer input) { return new DistributionField(input); } }).toList()); } }
@Override public RelNode visit(final RelNode other) { if (other instanceof WriterPrel) { for (RelTrait trait : other.getTraitSet()) { if (trait instanceof DistributionTrait) { List<DistributionField> distributionFieldList = ((DistributionTrait) trait).getFields(); if (distributionFieldList.size() != 1) { continue; } int fieldId = distributionFieldList.get(0).getFieldId(); String fieldName = ((WriterPrel) other).getInput().getRowType().getFieldNames().get(fieldId); if ("position".equals(fieldName)) { hashDistributedWriter.set(true); } } } } return super.visit(other); } });
private boolean isDefaultDist(RelNode n) { return n.getTraitSet().getTrait(DistributionTraitDef.INSTANCE).equals(DistributionTrait.DEFAULT); }
@Override public T read(final Kryo kryo, final Input input, final Class<T> type) { final boolean isKnown = kryo.readObject(input, Boolean.class); final T result; if (isKnown) { final DistributionTrait.DistributionType kind = kryo.readObject(input, DistributionTrait.DistributionType.class); result = (T)distributionMap.get(kind); } else { result = super.read(kryo, input, type); } final T normalized = (T) result.getTraitDef().canonize(result); kryo.reference(normalized); return normalized; } }
@Override public RelNode convert( RelOptPlanner planner, RelNode rel, DistributionTrait toDist, boolean allowInfiniteCostConverters) { switch(toDist.getType()){ // UnionExchange, HashToRandomExchange, OrderedPartitionExchange and BroadcastExchange destroy the ordering property, // therefore RelCollation is set to default, which is EMPTY. case SINGLETON: return new UnionExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.PHYSICAL).plus(toDist), rel); case HASH_DISTRIBUTED: return new HashToRandomExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.PHYSICAL).plus(toDist), rel, toDist.getFields()); case RANGE_DISTRIBUTED: return new OrderedPartitionExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.PHYSICAL).plus(toDist), rel); case BROADCAST_DISTRIBUTED: return new BroadcastExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.PHYSICAL).plus(toDist), rel); case ROUND_ROBIN_DISTRIBUTED: return new RoundRobinExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.PHYSICAL).plus(toDist), rel); case ANY: // If target is "any", any input would satisfy "any". Return input directly. return rel; default: return null; } }
protected static enum PhysicalJoinType {HASH_JOIN, MERGE_JOIN, NESTEDLOOP_JOIN};
&& ! left.getTraitSet().getTrait(DistributionTraitDef.INSTANCE).equals(DistributionTrait.SINGLETON) && (join.getJoinType() == JoinRelType.INNER || join.getJoinType() == JoinRelType.LEFT)) {
@Override public boolean canConvert( RelOptPlanner planner, DistributionTrait fromTrait, DistributionTrait toTrait, RelNode fromRel) { if (fromTrait.equals(toTrait)) { return true; } // Source trait is "ANY", which is abstract type of distribution. // We do not want to convert from "ANY", since it's abstract. // Source trait should be concrete type: SINGLETON, HASH_DISTRIBUTED, etc. if (fromTrait.equals(DistributionTrait.DEFAULT) && !(fromRel instanceof RelSubset) ) { return false; } // It is only possible to apply a distribution trait to a PHYSICAL convention. if (fromRel.getConvention() != Prel.PHYSICAL) { return false; } if (fromTrait.getType() == DistributionType.HASH_DISTRIBUTED && toTrait.getType() == DistributionType.BROADCAST_DISTRIBUTED) { return false; } if (fromTrait.getType() == DistributionType.BROADCAST_DISTRIBUTED && toTrait.getType() == DistributionType.HASH_DISTRIBUTED) { return false; } return true; }
DistributionTrait hashChild = new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(childDistFields)); traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.PHYSICAL).plus(hashChild);
new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(windowBase)));
new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true /* get all grouping keys */))); new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, false /* get single grouping key */)));
@Override public void onMatch(RelOptRuleCall call) { final SortRel sort = (SortRel) call.rel(0); final RelNode input = sort.getInput(); final PlannerSettings plannerSettings = PrelUtil.getPlannerSettings(call.getPlanner()); final RelTraitSet inputTraits; if (plannerSettings.getOptions().getOption(PlannerSettings.ENABLE_SORT_ROUND_ROBIN)) { // Keep the collation in logical sort just make its input round robin round robin. inputTraits = sort.getTraitSet().plus(Prel.PHYSICAL).plus(DistributionTrait.ROUND_ROBIN); } else { // Keep the collation in logical sort. Convert input into a RelNode with 1) this collation, 2) Physical, 3) hash distributed on DistributionTrait hashDistribution = new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(sort))); inputTraits = sort.getTraitSet().plus(Prel.PHYSICAL).plus(hashDistribution); } final RelNode convertedInput = convert(input, inputTraits); if(isSingleMode(call)){ call.transformTo(convertedInput); }else{ RelNode exch = new SingleMergeExchangePrel(sort.getCluster(), sort.getTraitSet().plus(Prel.PHYSICAL).plus(DistributionTrait.SINGLETON), convertedInput, sort.getCollation()); call.transformTo(exch); // transform logical "sort" into "SingleMergeExchange". } }
new DistributionTrait(DistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true)));