@Override public boolean satisfies(RelTrait trait) { if (trait == this) { return true; } switch (((RelDistribution)trait).getType()) { case HASH_DISTRIBUTED : return this.getKeys().equals(((RelDistribution)trait).getKeys()); default: throw new RuntimeException("Other distributions are not used yet."); } }
OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException { OpAttr inputOpAf = dispatch(exchangeRel.getInput()); String tabAlias = inputOpAf.tabAlias; if (tabAlias == null || tabAlias.length() == 0) { tabAlias = getHiveDerivedTableAlias(); } if (LOG.isDebugEnabled()) { LOG.debug("Translating operator rel#" + exchangeRel.getId() + ":" + exchangeRel.getRelTypeName() + " with row type: [" + exchangeRel.getRowType() + "]"); } RelDistribution distribution = exchangeRel.getDistribution(); if (distribution.getType() != Type.HASH_DISTRIBUTED) { throw new SemanticException("Only hash distribution supported for LogicalExchange"); } ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getJoinKeys().size()]; for (int index = 0; index < exchangeRel.getJoinKeys().size(); index++) { expressions[index] = convertToExprNode(exchangeRel.getJoinKeys().get(index), exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf); } exchangeRel.setJoinExpressions(expressions); ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions, -1, -1, Operation.NOT_ACID, hiveConf); return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, rsOp); }
public boolean isBucketedInput() { return RelMetadataQuery.instance().distribution(this.getInput()).getKeys(). containsAll(groupSet.asList()); }
/** Helper method to determine a {@link Project}'s collation. */ public static RelDistribution project(RelMetadataQuery mq, RelNode input, List<? extends RexNode> projects) { final RelDistribution inputDistribution = mq.distribution(input); final Mappings.TargetMapping mapping = Project.getPartialMapping(input.getRowType().getFieldCount(), projects); return inputDistribution.apply(mapping); }
@Test public void testRelDistributionSatisfy() { RelDistribution distribution1 = RelDistributions.hash(ImmutableList.of(0)); RelDistribution distribution2 = RelDistributions.hash(ImmutableList.of(1)); RelTraitSet traitSet = RelTraitSet.createEmpty(); RelTraitSet simpleTrait1 = traitSet.plus(distribution1); RelTraitSet simpleTrait2 = traitSet.plus(distribution2); RelTraitSet compositeTrait = traitSet.replace(RelDistributionTraitDef.INSTANCE, ImmutableList.of(distribution1, distribution2)); assertThat(compositeTrait.satisfies(simpleTrait1), is(true)); assertThat(compositeTrait.satisfies(simpleTrait2), is(true)); assertThat(distribution1.compareTo(distribution2), is(-1)); assertThat(distribution2.compareTo(distribution1), is(1)); //noinspection EqualsWithItself assertThat(distribution2.compareTo(distribution2), is(0)); } }
@Override public T read(final Kryo kryo, final Input input, final Class<T> type) { final boolean isKnown = kryo.readObject(input, Boolean.class); final T result; if (isKnown) { final RelDistribution.Type kind = kryo.readObject(input, RelDistribution.Type.class); result = (T)distributionMap.get(kind); } else { result = super.read(kryo, input, type); } final T normalized = (T) result.getTraitDef().canonize(result); kryo.reference(normalized); return normalized; } }
OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException { OpAttr inputOpAf = dispatch(exchangeRel.getInput()); String tabAlias = inputOpAf.tabAlias; if (tabAlias == null || tabAlias.length() == 0) { tabAlias = getHiveDerivedTableAlias(); } if (LOG.isDebugEnabled()) { LOG.debug("Translating operator rel#" + exchangeRel.getId() + ":" + exchangeRel.getRelTypeName() + " with row type: [" + exchangeRel.getRowType() + "]"); } RelDistribution distribution = exchangeRel.getDistribution(); if (distribution.getType() != Type.HASH_DISTRIBUTED) { throw new SemanticException("Only hash distribution supported for LogicalExchange"); } ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getJoinKeys().size()]; for (int index = 0; index < exchangeRel.getJoinKeys().size(); index++) { expressions[index] = convertToExprNode(exchangeRel.getJoinKeys().get(index), exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf); } exchangeRel.setJoinExpressions(expressions); ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions, -1, -1, Operation.NOT_ACID, hiveConf); return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, rsOp); }
public boolean isBucketedInput() { final RelMetadataQuery mq = this.getInput().getCluster().getMetadataQuery(); return mq.distribution(this).getKeys(). containsAll(groupSet.asList()); }
/** Helper method to determine a {@link Project}'s collation. */ public static RelDistribution project(RelMetadataQuery mq, RelNode input, List<? extends RexNode> projects) { final RelDistribution inputDistribution = mq.distribution(input); final Mappings.TargetMapping mapping = Project.getPartialMapping(input.getRowType().getFieldCount(), projects); return inputDistribution.apply(mapping); }
@Test public void testRelDistributionSatisfy() { RelDistribution distribution1 = RelDistributions.hash(ImmutableList.of(0)); RelDistribution distribution2 = RelDistributions.hash(ImmutableList.of(1)); RelTraitSet traitSet = RelTraitSet.createEmpty(); RelTraitSet simpleTrait1 = traitSet.plus(distribution1); RelTraitSet simpleTrait2 = traitSet.plus(distribution2); RelTraitSet compositeTrait = traitSet.replace(RelDistributionTraitDef.INSTANCE, ImmutableList.of(distribution1, distribution2)); assertThat(compositeTrait.satisfies(simpleTrait1), is(true)); assertThat(compositeTrait.satisfies(simpleTrait2), is(true)); assertThat(distribution1.compareTo(distribution2), is(-1)); assertThat(distribution2.compareTo(distribution1), is(1)); //noinspection EqualsWithItself assertThat(distribution2.compareTo(distribution2), is(0)); } }
@Override public boolean satisfies(RelTrait trait) { if (trait == this) { return true; } switch (((RelDistribution)trait).getType()) { case HASH_DISTRIBUTED : return this.getKeys().equals(((RelDistribution)trait).getKeys()); default: throw new RuntimeException("Other distributions are not used yet."); } }
public boolean isBucketedInput() { return RelMetadataQuery.distribution(this.getInput()).getKeys(). containsAll(groupSet.asList()); }
@Override public int compareTo(RelMultipleTrait o) { final RelDistribution distribution = (RelDistribution) o; if (type == distribution.getType() && (type == Type.HASH_DISTRIBUTED || type == Type.RANGE_DISTRIBUTED)) { return ORDERING.compare(getKeys(), distribution.getKeys()); } return type.compareTo(distribution.getType()); }
private void calculateSingle(ExecNode<?, ?> singleNode) { calculateInputs(singleNode); ExecNode<?, ?> inputNode = singleNode.getInputNodes().get(0); if (inputNode instanceof BatchExecExchange) { if (((BatchExecExchange) inputNode).getDistribution().getType() == RelDistribution.Type.SINGLETON) { finalParallelismNodeMap.put(singleNode, 1); } } }
private void calculateExchange(BatchExecExchange execExchange) { if (execExchange.getDistribution().getType() == RelDistribution.Type.RANGE_DISTRIBUTED) { setDefaultRes(execExchange.getResource()); } }
private void calculateExchange(StreamExecExchange execExchange) { if (execExchange.getDistribution().getType() == RelDistribution.Type.SINGLETON) { execExchange.getResource().setParallelism(1); } else { calculateDefault(execExchange); } }