private static List<Entry<String, Long>> rankTablesByAccumulatedSize(ParseContext pctx) { Map<String, Long> tableToTotalSize = new HashMap<>(); for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) { TableScanOperator tsOp = e.getValue(); String tableName = tsOp.getConf().getTableMetadata().getDbName() + "." + tsOp.getConf().getTableMetadata().getTableName(); long tableSize = tsOp.getStatistics() != null ? tsOp.getStatistics().getDataSize() : 0L; Long totalSize = tableToTotalSize.get(tableName); if (totalSize != null) { tableToTotalSize.put(tableName, StatsUtils.safeAdd(totalSize, tableSize)); } else { tableToTotalSize.put(tableName, tableSize); } } List<Entry<String, Long>> sortedTables = new ArrayList<>(tableToTotalSize.entrySet()); Collections.sort(sortedTables, Collections.reverseOrder( new Comparator<Map.Entry<String, Long>>() { @Override public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) { return (o1.getValue()).compareTo(o2.getValue()); } })); return sortedTables; }
private float getSelectivitySimpleTree(Operator<? extends OperatorDesc> op) { TableScanOperator tsOp = OperatorUtils .findSingleOperatorUpstream(op, TableScanOperator.class); if (tsOp == null) { // complex tree with multiple parents return getSelectivityComplexTree(op); } else { // simple tree with single parent long inputRow = tsOp.getStatistics().getNumRows(); long outputRow = op.getStatistics().getNumRows(); return (float) outputRow / (float) inputRow; } }
private float getSelectivitySimpleTree(Operator<? extends OperatorDesc> op) { TableScanOperator tsOp = OperatorUtils .findSingleOperatorUpstream(op, TableScanOperator.class); if (tsOp == null) { // complex tree with multiple parents return getSelectivityComplexTree(op); } else { // simple tree with single parent long inputRow = tsOp.getStatistics().getNumRows(); long outputRow = op.getStatistics().getNumRows(); return (float) outputRow / (float) inputRow; } }
ts.getStatistics().getDataSize() < victimTS.getStatistics().getDataSize()) { victimRS = (ReduceSinkOperator) o; victimTS = ts;
OperatorUtils.findOperatorsUpstream(sibling, TableScanOperator.class); for (TableScanOperator source : sources) { if (source.getStatistics() != null) { numberOfBytes += source.getStatistics().getDataSize(); if (LOG.isDebugEnabled()) { LOG.debug("Table source " + source + " has stats: " + source.getStatistics());
if (ts.getStatistics() != null) { long numRows = ts.getStatistics().getNumRows(); if (numRows < pCtx.getConf().getLongVar(ConfVars.TEZ_BIGTABLE_MIN_SIZE_SEMIJOIN_REDUCTION)) { removeSemiJoin = true;
OperatorUtils.findOperatorsUpstream(sibling, TableScanOperator.class); for (TableScanOperator source : sources) { if (source.getStatistics() != null) { numberOfBytes = StatsUtils.safeAdd(numberOfBytes, source.getStatistics().getDataSize()); if (LOG.isDebugEnabled()) { LOG.debug("Table source " + source + " has stats: " + source.getStatistics());
ts.getStatistics().getDataSize() < victimTS.getStatistics().getDataSize()) { victimRS = (ReduceSinkOperator) o; victimTS = ts;
if (ts.getStatistics() != null) { long numRows = ts.getStatistics().getNumRows(); if (numRows < pCtx.getConf().getLongVar(ConfVars.TEZ_BIGTABLE_MIN_SIZE_SEMIJOIN_REDUCTION)) { if (sjInfo.getShouldRemove()) {
currInputStat = root.getStatistics().clone(); } else { currInputStat.addBasicStats(root.getStatistics());
currInputStat.addToDataSize(root.getStatistics().getDataSize());
private float getSelectivitySimpleTree(Operator<? extends OperatorDesc> op) { TableScanOperator tsOp = OperatorUtils .findSingleOperatorUpstream(op, TableScanOperator.class); if (tsOp == null) { // complex tree with multiple parents return getSelectivityComplexTree(op); } else { // simple tree with single parent long inputRow = tsOp.getStatistics().getNumRows(); long outputRow = op.getStatistics().getNumRows(); return (float) outputRow / (float) inputRow; } }