protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) { super(true); this.mq = mq; this.childRel = childRel; this.childCardinality = mq.getRowCount(childRel); }
public Integer splitCountRepartition(RelNode rel, RelMetadataQuery mq) { // We repartition: new number of splits final Double averageRowSize = mq.getAverageRowSize(rel); final Double rowCount = mq.getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
protected FilterSelectivityEstimator(RelNode childRel) { super(true); this.childRel = childRel; this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel); }
public Integer splitCountRepartition(RelNode rel, RelMetadataQuery mq) { // We repartition: new number of splits final Double averageRowSize = mq.getAverageRowSize(rel); final Double rowCount = mq.getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr, RelMetadataQuery mq) { return new HiveCost(mq.getRowCount(hr), 0, 0); }
@Override public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts)); }
@Override public RelOptCost computeSelfCost(final RelOptPlanner planner, final RelMetadataQuery mq) { return planner.getCostFactory().makeCost(mq.getRowCount(sourceRel), 0, 0).multiplyBy(10); } }
@Override public RelOptCost getCost(HiveJoin join) { RelMetadataQuery mq = RelMetadataQuery.instance(); double leftRCount = mq.getRowCount(join.getLeft()); double rightRCount = mq.getRowCount(join.getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); }
private Double getDistinctRowCount(HiveTableScan htRel, RelMetadataQuery mq, ImmutableBitSet groupKey, RexNode predicate) { List<Integer> projIndxLst = HiveCalciteUtil .translateBitSetToProjIndx(groupKey); List<ColStatistics> colStats = htRel.getColStat(projIndxLst); Double noDistinctRows = 1.0; for (ColStatistics cStat : colStats) { noDistinctRows *= cStat.getCountDistint(); } return Math.min(noDistinctRows, mq.getRowCount(htRel)); }
@Override public RelOptCost getCost(HiveJoin join) { final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); double leftRCount = mq.getRowCount(join.getLeft()); double rightRCount = mq.getRowCount(join.getRight()); return HiveCost.FACTORY.makeCost(leftRCount + rightRCount, 0.0, 0.0); }
@Override public Double getRowCount(Sort rel, RelMetadataQuery mq) { final Double rowCount = mq.getRowCount(rel.getInput()); if (rowCount != null && rel.fetch != null) { final int offset = rel.offset == null ? 0 : RexLiteral.intValue(rel.offset); final int limit = RexLiteral.intValue(rel.fetch); final Double offsetLimit = new Double(offset + limit); // offsetLimit is smaller than rowCount of the input operator // thus, we return the offsetLimit if (offsetLimit < rowCount) { return offsetLimit; } } return rowCount; }
public Double memory(HiveAggregate aggregate, RelMetadataQuery mq) { final Double avgRowSize = mq.getAverageRowSize(aggregate.getInput()); final Double rowCount = mq.getRowCount(aggregate.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; }
public Double memory(HiveAggregate aggregate, RelMetadataQuery mq) { final Double avgRowSize = mq.getAverageRowSize(aggregate.getInput()); final Double rowCount = mq.getRowCount(aggregate.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; }
public Double memory(HiveSortLimit sort, RelMetadataQuery mq) { if (sort.getCollation() != RelCollations.EMPTY) { // It sorts final Double avgRowSize = mq.getAverageRowSize(sort.getInput()); final Double rowCount = mq.getRowCount(sort.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; } // It does not sort, memory footprint is zero return 0.0; }
public Double memory(HiveSortLimit sort, RelMetadataQuery mq) { if (sort.getCollation() != RelCollations.EMPTY) { // It sorts final Double avgRowSize = mq.getAverageRowSize(sort.getInput()); final Double rowCount = mq.getRowCount(sort.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; } // It does not sort, memory footprint is zero return 0.0; }
public Double getSelectivity(Join j, RelMetadataQuery mq, RexNode predicate) { if (j.getJoinType().equals(JoinRelType.INNER)) { return computeInnerJoinSelectivity(j, mq, predicate); } else if (j.getJoinType().equals(JoinRelType.LEFT) || j.getJoinType().equals(JoinRelType.RIGHT)) { double left = mq.getRowCount(j.getLeft()); double right = mq.getRowCount(j.getRight()); double product = left * right; double innerJoinSelectivity = computeInnerJoinSelectivity(j, mq, predicate); if (j.getJoinType().equals(JoinRelType.LEFT)) { return Math.max(innerJoinSelectivity, left/product); } return Math.max(innerJoinSelectivity, right/product); } return 1.0; }
@Override public boolean matches(RelOptRuleCall call) { final HiveSortLimit sortLimit = call.rel(0); // If it is not created by HiveSortJoinReduceRule, we cannot remove it if (!sortLimit.isRuleCreated()) { return false; } // Finally, if we do not reduce the size input enough, we bail out int limit = RexLiteral.intValue(sortLimit.fetch); Double rowCount = call.getMetadataQuery().getRowCount(sortLimit.getInput()); if (rowCount != null && limit <= reductionProportion * rowCount && rowCount - limit >= reductionTuples) { return false; } return true; }
private static double pkSelectivity(Join joinRel, RelMetadataQuery mq, boolean leftChild, RelNode child, double childRowCount) { if ((leftChild && joinRel.getJoinType().generatesNullsOnRight()) || (!leftChild && joinRel.getJoinType().generatesNullsOnLeft())) { return 1.0; } else { HiveTableScan tScan = EstimateUniqueKeys.getTableScan(child, true); if (tScan != null) { double tRowCount = mq.getRowCount(tScan); return childRowCount / tRowCount; } else { return 1.0; } } }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits RelMetadataQuery mq = RelMetadataQuery.instance(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }