/** * Create a distribution hash expression. * * @param fields Distribution fields * @param rowType Row type * @return */ public static LogicalExpression getHashExpression(List<DistributionField> fields, RelDataType rowType) { assert fields.size() > 0; final List<String> childFields = rowType.getFieldNames(); // If we already included a field with hash - no need to calculate hash further down if ( childFields.contains(HASH_EXPR_NAME)) { return new FieldReference(HASH_EXPR_NAME); } final List<LogicalExpression> expressions = new ArrayList<LogicalExpression>(childFields.size()); for(int i =0; i < fields.size(); i++){ expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()))); } return createHashBasedPartitionExpression(expressions, HASH_HELPER_LOGICALEXPRESSION); } }
private DistributionTrait convertDist(DistributionTrait srcDist, Map<Integer, Integer> inToOut) { List<DistributionField> newFields = Lists.newArrayList(); for (DistributionField field : srcDist.getFields()) { if (inToOut.containsKey(field.getFieldId())) { newFields.add(new DistributionField(inToOut.get(field.getFieldId()))); } } // After the projection, if the new distribution fields is empty, or new distribution fields is a subset of // original distribution field, we should replace with either SINGLETON or RANDOM_DISTRIBUTED. if (newFields.isEmpty() || newFields.size() < srcDist.getFields().size()) { if (srcDist.getType() != DistributionType.SINGLETON) { return DistributionTrait.ANY; } else { return DistributionTrait.SINGLETON; } } else { return new DistributionTrait(srcDist.getType(), ImmutableList.copyOf(newFields)); } }
childDistFields.add(new DistributionField(f.getIndex()));
public static ProjectPrel addHashProject(List<DistributionField> distFields, Prel input, Integer ringCount){ // Insert Project SqlOperatorImpl with new column that will be a hash for HashToRandomExchange fields final List<String> outputFieldNames = Lists.newArrayList(input.getRowType().getFieldNames()); final String fieldName = ringCount == null ? HashPrelUtil.HASH_EXPR_NAME : WriterPrel.BUCKET_NUMBER_FIELD; outputFieldNames.add(fieldName); final RexBuilder rexBuilder = input.getCluster().getRexBuilder(); final List<RelDataTypeField> childRowTypeFields = input.getRowType().getFieldList(); // create new hashed field. final HashExpressionCreatorHelper<RexNode> hashHelper = new RexNodeBasedHashExpressionCreatorHelper(rexBuilder); final List<RexNode> distFieldRefs = Lists.newArrayListWithExpectedSize(distFields.size()); for(int i = 0; i < distFields.size(); i++) { final int fieldId = distFields.get(i).getFieldId(); distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId)); } final List <RexNode> updatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size() + 1); for ( RelDataTypeField field : childRowTypeFields) { RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex()); updatedExpr.add(rex); } RexNode hashExpression = HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, hashHelper); if(ringCount != null){ RelDataType intType = input.getCluster().getTypeFactory().createSqlType(SqlTypeName.INTEGER); hashExpression = rexBuilder.makeCall(SqlStdOperatorTable.MOD, ImmutableList.of(hashExpression, rexBuilder.makeExactLiteral(BigDecimal.valueOf(ringCount), intType))); hashExpression = rexBuilder.makeCall(SqlStdOperatorTable.ABS, Collections.singletonList(hashExpression)); } updatedExpr.add(hashExpression); RelDataType rowType = RexUtil.createStructType(input.getCluster().getTypeFactory(), updatedExpr, outputFieldNames); ProjectPrel addColumnprojectPrel = new ProjectPrel(input.getCluster(), input.getTraitSet(), input, updatedExpr, rowType); return addColumnprojectPrel; }
@Override public RelNode visit(final RelNode other) { if (other instanceof WriterPrel) { for (RelTrait trait : other.getTraitSet()) { if (trait instanceof DistributionTrait) { List<DistributionField> distributionFieldList = ((DistributionTrait) trait).getFields(); if (distributionFieldList.size() != 1) { continue; } int fieldId = distributionFieldList.get(0).getFieldId(); String fieldName = ((WriterPrel) other).getInput().getRowType().getFieldNames().get(fieldId); if ("position".equals(fieldName)) { hashDistributedWriter.set(true); } } } } return super.visit(other); } });
protected List<DistributionField> getDistributionField(AggregateRel rel, boolean allFields) { List<DistributionField> groupByFields = Lists.newArrayList(); for (int i = 0; i < rel.getGroupSet().cardinality(); i++) { DistributionField field = new DistributionField(i); groupByFields.add(field); if (!allFields && groupByFields.size() == 1) { // if we are only interested in 1 grouping field, pick the first one for now.. // but once we have num distinct values (NDV) statistics, we should pick the one // with highest NDV. break; } } return groupByFields; }
@Override public Integer apply(DistributionField input) { return input.getFieldId(); } }));
@Override public DistributionField apply(Integer input) { return new DistributionField(input); } }).toList());
private List<DistributionTrait.DistributionField> getDistributionFields(Window.Group window) { List<DistributionTrait.DistributionField> groupByFields = Lists.newArrayList(); for (int group : BitSets.toIter(window.keys)) { DistributionTrait.DistributionField field = new DistributionTrait.DistributionField(group); groupByFields.add(field); } return groupByFields; }
private List<DistributionField> getDistributionField(SortRel rel) { List<DistributionField> distFields = Lists.newArrayList(); for (RelFieldCollation relField : rel.getCollation().getFieldCollations()) { DistributionField field = new DistributionField(relField.getFieldIndex()); distFields.add(field); } return distFields; } }
protected List<DistributionField> getDistributionField(List<Integer> keys) { List<DistributionField> distFields = Lists.newArrayList(); for (int key : keys) { distFields.add(new DistributionField(key)); } return distFields; }