private long getRowsProcessed(MinorFragmentProfile profile) { long maxRecords = 0; for (OperatorProfile operatorProfile : profile.getOperatorProfileList()) { long records = 0; for (StreamProfile inputProfile :operatorProfile.getInputProfileList()) { if (inputProfile.hasRecords()) { records += inputProfile.getRecords(); } } maxRecords = Math.max(maxRecords, records); } return maxRecords; }
private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS); for (ImmutablePair<OperatorProfile, Integer> ip : ops) { int minor = ip.getRight(); OperatorProfile op = ip.getLeft(); String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, null); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches, null); builder.appendFormattedInteger(maxRecords, null); builder.appendBytes(op.getPeakLocalMemoryAllocated(), null); } return builder.build(); }
long batches = 0; for (final StreamProfile sp : op.getInputProfileList()) { incomingRecords += sp.getRecords(); batches += sp.getBatches();
assertEquals(1L, scanProfile.get().getInputProfile(0).getRecords());
private void setOutputStats(List<StreamProfile> streams) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } if (jobStats.getOutputBytes() != null) { jobStats.setOutputBytes(jobStats.getOutputBytes() + outputBytes); } else { jobStats.setOutputBytes(outputBytes); } if (jobStats.getOutputRecords() != null) { jobStats.setOutputRecords(jobStats.getOutputRecords() + outputRecords); } else { jobStats.setOutputRecords(outputRecords); } }
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.StreamProfile other) { if (other == com.dremio.exec.proto.UserBitShared.StreamProfile.getDefaultInstance()) return this; if (other.hasRecords()) { setRecords(other.getRecords()); } if (other.hasBatches()) { setBatches(other.getBatches()); } if (other.hasSchemas()) { setSchemas(other.getSchemas()); } if (other.hasSize()) { setSize(other.getSize()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
private void setOutputStatsForQueryResults(List<StreamProfile> streams, boolean isOutputLimited) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } queryOutputBytes += outputBytes; queryOutputRecords += outputRecords; queryOutputLimited = queryOutputLimited || isOutputLimited; }
@Override public Long apply(OperatorProfile operatorProfile) { return operatorProfile.getInputProfile(inputStream).getRecords(); } })) {
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.StreamProfile message) throws java.io.IOException { if(message.hasRecords()) output.writeInt64(1, message.getRecords(), false); if(message.hasBatches()) output.writeInt64(2, message.getBatches(), false); if(message.hasSchemas()) output.writeInt64(3, message.getSchemas(), false); if(message.hasSize()) output.writeInt64(4, message.getSize(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.StreamProfile message)