long batches = 0; for (final StreamProfile sp : op.getInputProfileList()) { incomingRecords += sp.getRecords(); batches += sp.getBatches();
private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
private long getRowsProcessed(MinorFragmentProfile profile) { long maxRecords = 0; for (OperatorProfile operatorProfile : profile.getOperatorProfileList()) { long records = 0; for (StreamProfile inputProfile :operatorProfile.getInputProfileList()) { if (inputProfile.hasRecords()) { records += inputProfile.getRecords(); } } maxRecords = Math.max(maxRecords, records); } return maxRecords; }
return; case 1: builder.addInputProfile(input.mergeObject(com.dremio.exec.proto.UserBitShared.StreamProfile.newBuilder(), com.dremio.exec.proto.SchemaUserBitShared.StreamProfile.MERGE));
assertEquals(1L, scanProfile.get().getInputProfile(0).getRecords());
public com.dremio.exec.proto.UserBitShared.StreamProfile buildPartial() { com.dremio.exec.proto.UserBitShared.StreamProfile result = new com.dremio.exec.proto.UserBitShared.StreamProfile(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) == 0x00000001)) { to_bitField0_ |= 0x00000001; } result.records_ = records_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { to_bitField0_ |= 0x00000002; } result.batches_ = batches_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { to_bitField0_ |= 0x00000004; } result.schemas_ = schemas_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { to_bitField0_ |= 0x00000008; } result.size_ = size_; result.bitField0_ = to_bitField0_; onBuilt(); return result; }
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS); for (ImmutablePair<OperatorProfile, Integer> ip : ops) { int minor = ip.getRight(); OperatorProfile op = ip.getLeft(); String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, null); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches, null); builder.appendFormattedInteger(maxRecords, null); builder.appendBytes(op.getPeakLocalMemoryAllocated(), null); } return builder.build(); }
private void setOutputStats(List<StreamProfile> streams) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } if (jobStats.getOutputBytes() != null) { jobStats.setOutputBytes(jobStats.getOutputBytes() + outputBytes); } else { jobStats.setOutputBytes(outputBytes); } if (jobStats.getOutputRecords() != null) { jobStats.setOutputRecords(jobStats.getOutputRecords() + outputRecords); } else { jobStats.setOutputRecords(outputRecords); } }
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.StreamProfile other) { if (other == com.dremio.exec.proto.UserBitShared.StreamProfile.getDefaultInstance()) return this; if (other.hasRecords()) { setRecords(other.getRecords()); } if (other.hasBatches()) { setBatches(other.getBatches()); } if (other.hasSchemas()) { setSchemas(other.getSchemas()); } if (other.hasSize()) { setSize(other.getSize()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public Builder toBuilder() { return newBuilder(this); }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(com.dremio.exec.proto.UserBitShared.StreamProfile prototype) {
private void setOutputStatsForQueryResults(List<StreamProfile> streams, boolean isOutputLimited) { if (streams == null) { return; } long outputBytes = 0L; long outputRecords = 0L; for (StreamProfile stream : streams) { outputRecords += stream.getRecords(); outputBytes += stream.getSize(); } queryOutputBytes += outputBytes; queryOutputRecords += outputRecords; queryOutputLimited = queryOutputLimited || isOutputLimited; }
public static Builder newBuilder(com.dremio.exec.proto.UserBitShared.StreamProfile prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }
@Override public Long apply(OperatorProfile operatorProfile) { return operatorProfile.getInputProfile(inputStream).getRecords(); } })) {
public com.dremio.exec.proto.UserBitShared.StreamProfile.Builder newMessage() { return com.dremio.exec.proto.UserBitShared.StreamProfile.newBuilder(); } public java.lang.String getFieldName(int number)
public void addStreamProfile(OperatorProfile.Builder builder) { for(int i = 0; i < recordsReceivedByInput.length; i++){ builder.addInputProfile( StreamProfile.newBuilder() .setBatches(batchesReceivedByInput[i]) .setRecords(recordsReceivedByInput[i]) .setSize(sizeInBytesReceivedByInput[i]) ); } }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.StreamProfile message) throws java.io.IOException { if(message.hasRecords()) output.writeInt64(1, message.getRecords(), false); if(message.hasBatches()) output.writeInt64(2, message.getBatches(), false); if(message.hasSchemas()) output.writeInt64(3, message.getSchemas(), false); if(message.hasSize()) output.writeInt64(4, message.getSize(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.StreamProfile message)