private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
private void setOutputStats(List<StreamProfile> streams) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } if (jobStats.getOutputBytes() != null) { jobStats.setOutputBytes(jobStats.getOutputBytes() + outputBytes); } else { jobStats.setOutputBytes(outputBytes); } if (jobStats.getOutputRecords() != null) { jobStats.setOutputRecords(jobStats.getOutputRecords() + outputRecords); } else { jobStats.setOutputRecords(outputRecords); } }
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.StreamProfile other) { if (other == com.dremio.exec.proto.UserBitShared.StreamProfile.getDefaultInstance()) return this; if (other.hasRecords()) { setRecords(other.getRecords()); } if (other.hasBatches()) { setBatches(other.getBatches()); } if (other.hasSchemas()) { setSchemas(other.getSchemas()); } if (other.hasSize()) { setSize(other.getSize()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
private void setOutputStatsForQueryResults(List<StreamProfile> streams, boolean isOutputLimited) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } queryOutputBytes += outputBytes; queryOutputRecords += outputRecords; queryOutputLimited = queryOutputLimited || isOutputLimited; }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.StreamProfile message) throws java.io.IOException { if(message.hasRecords()) output.writeInt64(1, message.getRecords(), false); if(message.hasBatches()) output.writeInt64(2, message.getBatches(), false); if(message.hasSchemas()) output.writeInt64(3, message.getSchemas(), false); if(message.hasSize()) output.writeInt64(4, message.getSize(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.StreamProfile message)