/** * Cost of doing Top-N is proportional to M log N where M is the total number of * input rows and N is the limit for Top-N. This makes Top-N preferable to Sort * since cost of full Sort is proportional to M log M . */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice. return super.computeSelfCost(planner).multiplyBy(0.05); } RelNode child = this.getInput(); double inputRows = relMetadataQuery.getRowCount(child); int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DremioCost.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(limit)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = mq.getRowCount(child); int rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH; double hashCpuCost = DremioCost.HASH_CPU_COST * inputRows * distFields.size(); double svrCpuCost = DremioCost.SVR_CPU_COST * inputRows; double mergeCpuCost = DremioCost.COMPARE_CPU_COST * inputRows * (Math.log(numEndPoints)/Math.log(2)); double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, hashCpuCost + svrCpuCost + mergeCpuCost, 0, networkCost); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double totalInputRowCount = 0; for (int i = 0; i < this.getInputs().size(); i++) { totalInputRowCount += mq.getRowCount(this.getInputs().get(i)); } double cpuCost = totalInputRowCount * DremioCost.BASE_CPU_COST; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double totalInputRowCount = 0; for (int i = 0; i < this.getInputs().size(); i++) { totalInputRowCount += mq.getRowCount(this.getInputs().get(i)); } double cpuCost = totalInputRowCount * DremioCost.BASE_CPU_COST; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double cpuCost = DremioCost.COMPARE_CPU_COST * fetchSize; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(fetchSize, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double cpuCost = DremioCost.COMPARE_CPU_COST * fetchSize; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(fetchSize, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } if (joincategory == JoinCategory.CARTESIAN || joincategory == JoinCategory.INEQUALITY) { return planner.getCostFactory().makeInfiniteCost(); } return computeHashJoinCost(planner, mq); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = mq.getRowCount(child); int numGroupByFields = this.getGroupCount(); int numAggrFields = this.aggCalls.size(); double cpuCost = DremioCost.COMPARE_CPU_COST * numGroupByFields * inputRows; // add cpu cost for computing the aggregate functions cpuCost += DremioCost.FUNC_CPU_COST * numAggrFields * inputRows; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, 0 /* disk i/o cost */, 0 /* network cost */); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double numRows = mq.getRowCount(this); double cpuCost = DremioCost.COMPARE_CPU_COST * numRows; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(numRows, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } // by default, assume cost is proportional to number of rows double rowCount = relMetadataQuery.getRowCount(this); Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(rowCount, rowCount, 0, 0).multiplyBy(0.1); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice. return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = relMetadataQuery.getRowCount(child); // int rowWidth = child.getRowType().getPrecision(); int numSortFields = this.collation.getFieldCollations().size(); double cpuCost = DremioCost.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(inputRows)/Math.log(2)); double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints // TODO: use rowWidth instead of avgFieldWidth * numFields // avgFieldWidth * numFields * inputRows double numFields = this.getRowType().getFieldCount(); long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions() .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).getNumVal(); double memCost = fieldWidth * numFields * inputRows; Factory costFactory = (Factory) planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0, memCost); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } if (joincategory == JoinCategory.CARTESIAN || joincategory == JoinCategory.INEQUALITY) { return ((Factory)planner.getCostFactory()).makeInfiniteCost(); } double leftRowCount = mq.getRowCount(this.getLeft()); double rightRowCount = mq.getRowCount(this.getRight()); // cost of evaluating each leftkey=rightkey join condition double joinConditionCost = DremioCost.COMPARE_CPU_COST * this.getLeftKeys().size(); double cpuCost = joinConditionCost * (leftRowCount + rightRowCount); Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(leftRowCount + rightRowCount, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } // cost is proportional to the number of rows and number of columns being projected double rowCount = this.estimateRowCount(mq); double cpuCost = DremioCost.PROJECT_CPU_COST * rowCount; Factory costFactory = (Factory)planner.getCostFactory(); if (numProjectsPushed > 0) { return costFactory.makeCost(rowCount, cpuCost, 0, 0).multiplyBy(1/numProjectsPushed); } else { return costFactory.makeCost(rowCount, cpuCost, 0, 0); } }
/** * Sends a copy of each batch to one node (same as the data size) */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); final double inputRows = mq.getRowCount(child); final int rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH; final double cpuCost = DremioCost.SVR_CPU_COST * inputRows; final double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth; return new DremioCost(inputRows, cpuCost, 0, networkCost); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } double leftRowCount = mq.getRowCount(this.getLeft()); double rightRowCount = mq.getRowCount(this.getRight()); double nljFactor = PrelUtil.getSettings(getCluster()).getNestedLoopJoinFactor(); // cpu cost of evaluating each leftkey=rightkey join condition double joinConditionCost = DremioCost.COMPARE_CPU_COST * this.getLeftKeys().size(); double cpuCost = joinConditionCost * (leftRowCount * rightRowCount) * nljFactor; Factory costFactory = (Factory) planner.getCostFactory(); return costFactory.makeCost(leftRowCount * rightRowCount, cpuCost, 0, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { // Elasticsearch does not support contains() in project (only in filter). if (hasContains) { return planner.getCostFactory().makeInfiniteCost(); } if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } // cost is proportional to the number of rows and number of columns being projected double rowCount = relMetadataQuery.getRowCount(this); // cpu is proportional to the number of columns and row count. For complex expressions, we also add // additional cost for those columns (multiply by DremioCost.PROJECT_CPU_COST). double cpuCost = (DremioCost.PROJECT_SIMPLE_CPU_COST * rowCount * simpleFieldCount) + (DremioCost.PROJECT_CPU_COST * (nonSimpleFieldCount > 0 ? rowCount : 0) * nonSimpleFieldCount); Factory costFactory = (Factory) planner.getCostFactory(); return costFactory.makeCost(rowCount, cpuCost, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = mq.getRowCount(child); int rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH; double rangePartitionCpuCost = DremioCost.RANGE_PARTITION_CPU_COST * inputRows; double svrCpuCost = DremioCost.SVR_CPU_COST * inputRows; double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth; Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, rangePartitionCpuCost + svrCpuCost, 0, networkCost); }
if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return planner.getCostFactory().makeCost(estimatedRowCount, workCost, workCost);
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) { if ((hasContains && !canHaveContains()) || hasFlatten) { return planner.getCostFactory().makeInfiniteCost(); } if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); double inputRows = relMetadataQuery.getRowCount(child); double cpuCost = estimateCpuCost(relMetadataQuery); Factory costFactory = (Factory)planner.getCostFactory(); return costFactory.makeCost(inputRows, cpuCost, 0, 0); }
/** * In a BroadcastExchange, each sender is sending data to N receivers (for costing * purposes we assume it is also sending to itself). */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) { return super.computeSelfCost(planner).multiplyBy(.1); } RelNode child = this.getInput(); final int numEndPoints = PrelUtil.getSettings(getCluster()).numEndPoints(); final double inputRows = mq.getRowCount(child); final int rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH; final double cpuCost = DremioCost.SVR_CPU_COST * inputRows; final double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth * numEndPoints; return new DremioCost(inputRows, cpuCost, 0, networkCost); }