@Override public boolean satisfies(RelTrait trait) { if (trait == this) { return true; } switch (((RelDistribution)trait).getType()) { case HASH_DISTRIBUTED : return this.getKeys().equals(((RelDistribution)trait).getKeys()); default: throw new RuntimeException("Other distributions are not used yet."); } }
@Override public boolean satisfies(RelTrait trait) { if (trait == this) { return true; } switch (((RelDistribution)trait).getType()) { case HASH_DISTRIBUTED : return this.getKeys().equals(((RelDistribution)trait).getKeys()); default: throw new RuntimeException("Other distributions are not used yet."); } }
@Override public int compareTo(RelMultipleTrait o) { final RelDistribution distribution = (RelDistribution) o; if (type == distribution.getType() && (type == Type.HASH_DISTRIBUTED || type == Type.RANGE_DISTRIBUTED)) { return ORDERING.compare(getKeys(), distribution.getKeys()); } return type.compareTo(distribution.getType()); }
if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false;
if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false;
OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException { OpAttr inputOpAf = dispatch(exchangeRel.getInput()); String tabAlias = inputOpAf.tabAlias; if (tabAlias == null || tabAlias.length() == 0) { tabAlias = getHiveDerivedTableAlias(); } if (LOG.isDebugEnabled()) { LOG.debug("Translating operator rel#" + exchangeRel.getId() + ":" + exchangeRel.getRelTypeName() + " with row type: [" + exchangeRel.getRowType() + "]"); } RelDistribution distribution = exchangeRel.getDistribution(); if (distribution.getType() != Type.HASH_DISTRIBUTED) { throw new SemanticException("Only hash distribution supported for LogicalExchange"); } ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getJoinKeys().size()]; for (int index = 0; index < exchangeRel.getJoinKeys().size(); index++) { expressions[index] = convertToExprNode(exchangeRel.getJoinKeys().get(index), exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf); } exchangeRel.setJoinExpressions(expressions); ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions, -1, -1, Operation.NOT_ACID, hiveConf); return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, rsOp); }
if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false;
if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false;
OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException { OpAttr inputOpAf = dispatch(exchangeRel.getInput()); String tabAlias = inputOpAf.tabAlias; if (tabAlias == null || tabAlias.length() == 0) { tabAlias = getHiveDerivedTableAlias(); } if (LOG.isDebugEnabled()) { LOG.debug("Translating operator rel#" + exchangeRel.getId() + ":" + exchangeRel.getRelTypeName() + " with row type: [" + exchangeRel.getRowType() + "]"); } RelDistribution distribution = exchangeRel.getDistribution(); if (distribution.getType() != Type.HASH_DISTRIBUTED) { throw new SemanticException("Only hash distribution supported for LogicalExchange"); } ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getJoinKeys().size()]; for (int index = 0; index < exchangeRel.getJoinKeys().size(); index++) { expressions[index] = convertToExprNode(exchangeRel.getJoinKeys().get(index), exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf); } exchangeRel.setJoinExpressions(expressions); ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions, -1, -1, Operation.NOT_ACID, hiveConf); return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, rsOp); }
@Override public boolean satisfies(RelTrait trait) { if (trait == this) { return true; } switch (((RelDistribution)trait).getType()) { case HASH_DISTRIBUTED : return this.getKeys().equals(((RelDistribution)trait).getKeys()); default: throw new RuntimeException("Other distributions are not used yet."); } }
@Override public int compareTo(@Nonnull RelMultipleTrait o) { final RelDistribution distribution = (RelDistribution) o; if (type == distribution.getType() && (type == Type.HASH_DISTRIBUTED || type == Type.RANGE_DISTRIBUTED)) { return ORDERING.compare(getKeys(), distribution.getKeys()); } return type.compareTo(distribution.getType()); } }
@Override public int compareTo(@Nonnull RelMultipleTrait o) { final RelDistribution distribution = (RelDistribution) o; if (type == distribution.getType() && (type == Type.HASH_DISTRIBUTED || type == Type.RANGE_DISTRIBUTED)) { return ORDERING.compare(getKeys(), distribution.getKeys()); } return type.compareTo(distribution.getType()); } }
private void calculateSingle(ExecNode<?, ?> singleNode) { calculateInputs(singleNode); ExecNode<?, ?> inputNode = singleNode.getInputNodes().get(0); if (inputNode instanceof BatchExecExchange) { if (((BatchExecExchange) inputNode).getDistribution().getType() == RelDistribution.Type.SINGLETON) { finalParallelismNodeMap.put(singleNode, 1); } } }
private void calculateExchange(BatchExecExchange execExchange) { if (execExchange.getDistribution().getType() == RelDistribution.Type.RANGE_DISTRIBUTED) { setDefaultRes(execExchange.getResource()); } }
private void calculateExchange(StreamExecExchange execExchange) { if (execExchange.getDistribution().getType() == RelDistribution.Type.SINGLETON) { execExchange.getResource().setParallelism(1); } else { calculateDefault(execExchange); } }
private void buildShuffleStages(ExecNode<?, ?> execNode) { if (nodeShuffleStageMap.containsKey(execNode)) { return; } for (ExecNode<?, ?> input : execNode.getInputNodes()) { buildShuffleStages((input)); } if (execNode.getInputNodes().isEmpty()) { // source node ShuffleStage shuffleStage = new ShuffleStage(); shuffleStage.addNode(execNode); if (nodeToFinalParallelismMap.containsKey(execNode)) { shuffleStage.setParallelism(nodeToFinalParallelismMap.get(execNode), true); } nodeShuffleStageMap.put(execNode, shuffleStage); } else if (execNode instanceof BatchExecExchange && !(((BatchExecExchange) execNode).getDistribution().getType() == RelDistribution.Type.RANGE_DISTRIBUTED)) { // do nothing } else if (execNode instanceof BatchExecSink) { // do nothing } else { Set<ShuffleStage> inputShuffleStages = getInputShuffleStages(execNode); Integer parallelism = nodeToFinalParallelismMap.get(execNode); ShuffleStage inputShuffleStage = mergeInputShuffleStages(inputShuffleStages, parallelism); inputShuffleStage.addNode(execNode); nodeShuffleStageMap.put(execNode, inputShuffleStage); } }
@Override public boolean isExecutable(HiveJoin join) { // Requirements: for SMB, sorted by their keys on both sides and bucketed. // Get key columns JoinPredicateInfo joinPredInfo = join.getJoinPredicateInfo(); List<ImmutableIntList> joinKeysInChildren = new ArrayList<ImmutableIntList>(); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromLeftPartOfJoinKeysInChildSchema())); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromRightPartOfJoinKeysInChildSchema())); for (int i=0; i<join.getInputs().size(); i++) { RelNode input = join.getInputs().get(i); // Is smbJoin possible? We need correct order boolean orderFound = join.getSortedInputs().get(i); if (!orderFound) { return false; } // Is smbJoin possible? We need correct bucketing RelDistribution distribution = RelMetadataQuery.distribution(input); if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false; } if (!distribution.getKeys().containsAll(joinKeysInChildren.get(i))) { return false; } } return true; }
if (distribution.getType() != Type.HASH_DISTRIBUTED) { return false;
OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException { OpAttr inputOpAf = dispatch(exchangeRel.getInput()); if (LOG.isDebugEnabled()) { LOG.debug("Translating operator rel#" + exchangeRel.getId() + ":" + exchangeRel.getRelTypeName() + " with row type: [" + exchangeRel.getRowType() + "]"); } RelDistribution distribution = exchangeRel.getDistribution(); if (distribution.getType() != Type.HASH_DISTRIBUTED) { throw new SemanticException("Only hash distribution supported for LogicalExchange"); } ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getJoinKeys().size()]; for (int index = 0; index < exchangeRel.getJoinKeys().size(); index++) { expressions[index] = convertToExprNode((RexNode) exchangeRel.getJoinKeys().get(index), exchangeRel.getInput(), null, inputOpAf); } exchangeRel.setJoinExpressions(expressions); ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), expressions, -1, -1, Operation.NOT_ACID, strictMode); return inputOpAf.clone(rsOp); }