public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { final RelMetadataQuery mq = input.getCluster().getMetadataQuery(); Double currentMemory = mq.cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; } return true; } return false; }
public List<RelCollation> get() { // CALCITE-88 return RelMdCollation.project(cluster.getMetadataQuery(), convertChild, project.getProjects()); } });
public boolean isBucketedInput() { final RelMetadataQuery mq = this.getInput().getCluster().getMetadataQuery(); return mq.distribution(this).getKeys(). containsAll(groupSet.asList()); }
public MapJoinStreamingRelation getStreamingSide() { RelMetadataQuery mq = left.getCluster().getMetadataQuery(); Double leftInputSize = mq.memory(left); Double rightInputSize = mq.memory(right); if (leftInputSize == null && rightInputSize == null) { return MapJoinStreamingRelation.NONE; } else if (leftInputSize != null && (rightInputSize == null || (leftInputSize < rightInputSize))) { return MapJoinStreamingRelation.RIGHT_RELATION; } else if (rightInputSize != null && (leftInputSize == null || (rightInputSize <= leftInputSize))) { return MapJoinStreamingRelation.LEFT_RELATION; } return MapJoinStreamingRelation.NONE; }
/** Creates an OLAPValuesRel. */ public static OLAPValuesRel create(RelOptCluster cluster, final RelDataType rowType, final ImmutableList<ImmutableList<RexLiteral>> tuples) { final RelMetadataQuery mq = cluster.getMetadataQuery(); final RelTraitSet traitSet = cluster.traitSetOf(OLAPRel.CONVENTION) .replaceIfs(RelCollationTraitDef.INSTANCE, new Supplier<List<RelCollation>>() { public List<RelCollation> get() { return RelMdCollation.values(mq, rowType, tuples); } }).replaceIf(RelDistributionTraitDef.INSTANCE, new Supplier<RelDistribution>() { public RelDistribution get() { return RelMdDistribution.values(rowType, tuples); } }); return new OLAPValuesRel(cluster, rowType, tuples, traitSet); }
public static Pair<RelOptTable, List<Integer>> getColumnOriginSet(RelNode rel, ImmutableBitSet colSet) { RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); RexBuilder rexBuilder = rel.getCluster().getRexBuilder(); Map<RelTableRef, List<Integer>> tabToOriginColumns = new HashMap<>();
RelMetadataQuery mq = rel.getCluster().getMetadataQuery(); double numRows = mq.getRowCount(tScan); List<ColStatistics> colStats = tScan.getColStat(BitSets
public ImmutableBitSet getSortedInputs() throws CalciteSemanticException { ImmutableBitSet.Builder sortedInputsBuilder = ImmutableBitSet.builder(); JoinPredicateInfo joinPredInfo = HiveCalciteUtil.JoinPredicateInfo. constructJoinPredicateInfo(this); List<ImmutableIntList> joinKeysInChildren = new ArrayList<ImmutableIntList>(); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromLeftPartOfJoinKeysInChildSchema())); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromRightPartOfJoinKeysInChildSchema())); final RelMetadataQuery mq = this.left.getCluster().getMetadataQuery(); for (int i=0; i<this.getInputs().size(); i++) { boolean correctOrderFound = RelCollations.contains( mq.collations(this.getInputs().get(i)), joinKeysInChildren.get(i)); if (correctOrderFound) { sortedInputsBuilder.set(i); } } return sortedInputsBuilder.build(); }
joinPredInfo.getProjsFromRightPartOfJoinKeysInChildSchema())); final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); for (int i=0; i<join.getInputs().size(); i++) { RelNode input = join.getInputs().get(i);
public static Integer getSplitCountWithoutRepartition(HiveJoin join) { RelNode largeInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { largeInput = join.getLeft(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { largeInput = join.getRight(); } else { return null; } final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); return mq.splitCount(largeInput); }
@Override public RelOptCost getCost(HiveJoin join) { final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); double leftRCount = mq.getRowCount(join.getLeft()); double rightRCount = mq.getRowCount(join.getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If simple map join, the whole relation goes in memory final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); return mq.cumulativeMemoryWithinPhase(inMemoryInput); }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); // TODO: Split count is not same as no of buckets JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezSMBJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If bucket map join, only a split goes in memory final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double memoryInput = mq.cumulativeMemoryWithinPhase(inMemoryInput); final Integer splitCount = mq.splitCount(inMemoryInput); if (memoryInput == null || splitCount == null) { return null; } return memoryInput / splitCount; }
public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation streamingSide) { Double memory = 0.0; final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.RIGHT_RELATION) { // Left side final Double leftAvgRowSize = mq.getAverageRowSize(join.getLeft()); final Double leftRowCount = mq.getRowCount(join.getLeft()); if (leftAvgRowSize == null || leftRowCount == null) { return null; } memory += leftAvgRowSize * leftRowCount; } if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.LEFT_RELATION) { // Right side final Double rightAvgRowSize = mq.getAverageRowSize(join.getRight()); final Double rightRowCount = mq.getRowCount(join.getRight()); if (rightAvgRowSize == null || rightRowCount == null) { return null; } memory += rightAvgRowSize * rightRowCount; } return memory; }
RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery();
@Override public RelOptCost getAggregateCost(HiveAggregate aggregate) { if (aggregate.isBucketedInput()) { return HiveCost.FACTORY.makeZeroCost(); } else { final RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery(); // 1. Sum of input cardinalities final Double rCount = mq.getRowCount(aggregate.getInput()); if (rCount == null) { return null; } // 2. CPU cost = sorting cost final double cpuCost = algoUtils.computeSortCPUCost(rCount); // 3. IO cost = cost of writing intermediary results to local FS + // cost of reading from local FS for transferring to GBy + // cost of transferring map outputs to GBy operator final Double rAverageSize = mq.getAverageRowSize(aggregate.getInput()); if (rAverageSize == null) { return null; } final double ioCost = algoUtils.computeSortIOCost(new Pair<Double,Double>(rCount,rAverageSize)); // 4. Result return HiveCost.FACTORY.makeCost(rCount, cpuCost, ioCost); } }
public JdbcPrel(RelOptCluster cluster, RelTraitSet traitSet, JdbcIntermediatePrel prel) { super(cluster, traitSet); final RelNode input = prel.getInput(); rows = input.estimateRowCount(cluster.getMetadataQuery()); convention = (DrillJdbcConvention) input.getTraitSet().getTrait(ConventionTraitDef.INSTANCE); // generate sql for tree. final SqlDialect dialect = convention.getPlugin().getDialect(); final JdbcImplementor jdbcImplementor = new JdbcImplementor( dialect, (JavaTypeFactory) getCluster().getTypeFactory()); final JdbcImplementor.Result result = jdbcImplementor.visitChild(0, input.accept(new SubsetRemover())); sql = result.asStatement().toSqlString(dialect).getSql(); rowType = input.getRowType(); }