@Override public int compare(final ProcessorStatus o1, final ProcessorStatus o2) { if (o1 == null && o2 == null) { return 0; } if (o1 == null) { return 1; } if (o2 == null) { return -1; } return -Long.compare(o1.getProcessingNanos(), o2.getProcessingNanos()); } });
public Map<String, Double> getProcessorMetrics(ProcessorStatus status) { final Map<String, Double> metrics = new HashMap<>(); metrics.put(MetricNames.FLOW_FILES_RECEIVED, new Double(status.getInputCount())); metrics.put(MetricNames.FLOW_FILES_SENT, new Double(status.getOutputCount())); metrics.put(MetricNames.BYTES_READ, new Double(status.getInputBytes())); metrics.put(MetricNames.BYTES_WRITTEN, new Double(status.getOutputBytes())); metrics.put(MetricNames.ACTIVE_THREADS, new Double(status.getActiveThreadCount())); metrics.put(MetricNames.TOTAL_TASK_DURATION, new Double(status.getProcessingNanos())); return metrics; }
private static long calculateTaskMillis(final ProcessGroupStatus status) { long nanos = 0L; for (final ProcessorStatus procStatus : status.getProcessorStatus()) { nanos += procStatus.getProcessingNanos(); } for (final ProcessGroupStatus childStatus : status.getProcessGroupStatus()) { nanos += calculateTaskMillis(childStatus); } return TimeUnit.MILLISECONDS.convert(nanos, TimeUnit.NANOSECONDS); } }
protected long calculateProcessingNanos(final ProcessGroupStatus status) { long nanos = 0L; for (final ProcessorStatus procStats : status.getProcessorStatus()) { nanos += procStats.getProcessingNanos(); } for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { nanos += calculateProcessingNanos(childGroupStatus); } return nanos; }
private long calculateProcessingNanos(final ProcessGroupStatus status) { long nanos = 0L; for (final ProcessorStatus procStats : status.getProcessorStatus()) { nanos += procStats.getProcessingNanos(); } for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { nanos += calculateProcessingNanos(childGroupStatus); } return nanos; }
protected long calculateProcessingNanos(final ProcessGroupStatus status) { long nanos = 0L; for (final ProcessorStatus procStats : status.getProcessorStatus()) { nanos += procStats.getProcessingNanos(); } for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { nanos += calculateProcessingNanos(childGroupStatus); } return nanos; }
/** * Calculate the total processing time of a process group. * * @param status the current process group status. * @return the total amount of nanoseconds spent in each processor in the process group. */ private long calculateProcessingNanos(final ProcessGroupStatus status) { long nanos = 0L; for (final ProcessorStatus procStats : status.getProcessorStatus()) { nanos += procStats.getProcessingNanos(); } for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { nanos += calculateProcessingNanos(childGroupStatus); } return nanos; } }
final String invocations = String.valueOf(processorStatus.getInvocations()); final long nanos = processorStatus.getProcessingNanos(); final String procTime = FormatUtils.formatHoursMinutesSeconds(nanos, TimeUnit.NANOSECONDS); writtenDiff = toDiff(lastStatus.getBytesWritten(), processorStatus.getBytesWritten(), true, false); invocationsDiff = toDiff(lastStatus.getInvocations(), processorStatus.getInvocations()); procTimeDiff = toDiff(lastStatus.getProcessingNanos(), processorStatus.getProcessingNanos(), false, true); } else { inputDiff = toDiff(0L, 0L, processorStatus.getInputCount(), processorStatus.getInputBytes()); writtenDiff = toDiff(0L, processorStatus.getBytesWritten(), true, false); invocationsDiff = toDiff(0L, processorStatus.getInvocations()); procTimeDiff = toDiff(0L, processorStatus.getProcessingNanos(), false, true);
public ProcessorStatusDTO createProcessorStatusDto(final ProcessorStatus procStatus) { final ProcessorStatusDTO dto = new ProcessorStatusDTO(); dto.setId(procStatus.getId()); dto.setGroupId(procStatus.getGroupId()); dto.setName(procStatus.getName()); dto.setStatsLastRefreshed(new Date()); dto.setRunStatus(procStatus.getRunStatus().toString()); final ProcessorStatusSnapshotDTO snapshot = new ProcessorStatusSnapshotDTO(); dto.setAggregateSnapshot(snapshot); snapshot.setId(procStatus.getId()); snapshot.setGroupId(procStatus.getGroupId()); snapshot.setName(procStatus.getName()); snapshot.setFlowFilesOut(procStatus.getOutputCount()); snapshot.setBytesOut(procStatus.getOutputBytes()); snapshot.setFlowFilesIn(procStatus.getInputCount()); snapshot.setBytesIn(procStatus.getInputBytes()); snapshot.setBytesRead(procStatus.getBytesRead()); snapshot.setBytesWritten(procStatus.getBytesWritten()); snapshot.setTaskCount(procStatus.getInvocations()); snapshot.setTasksDurationNanos(procStatus.getProcessingNanos()); snapshot.setTasksDuration(FormatUtils.formatHoursMinutesSeconds(procStatus.getProcessingNanos(), TimeUnit.NANOSECONDS)); // determine the run status snapshot.setRunStatus(procStatus.getRunStatus().toString()); snapshot.setExecutionNode(procStatus.getExecutionNode().toString()); snapshot.setActiveThreadCount(procStatus.getActiveThreadCount()); snapshot.setTerminatedThreadCount(procStatus.getTerminatedThreadCount()); snapshot.setType(procStatus.getType()); StatusMerger.updatePrettyPrintedFields(snapshot); return dto; }
merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes()); merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount()); merged.setProcessingNanos(merged.getProcessingNanos() + statusToMerge.getProcessingNanos()); merged.setFlowFilesRemoved(merged.getFlowFilesRemoved() + statusToMerge.getFlowFilesRemoved());
private void serializeProcessorStatus(final JsonArrayBuilder arrayBuilder, final JsonBuilderFactory factory, final ProcessorStatus status, final DateFormat df, final String hostname, final String applicationName, final String platform, final String parentId, final Date currentDate) { final JsonObjectBuilder builder = factory.createObjectBuilder(); final String componentType = "Processor"; final String componentName = status.getName(); if (componentMatchesFilters(componentType, componentName)) { addCommonFields(builder, df, hostname, applicationName, platform, parentId, currentDate, componentType, componentName); addField(builder, "componentId", status.getId()); addField(builder, "processorType", status.getType()); addField(builder, "averageLineageDurationMS", status.getAverageLineageDuration()); addField(builder, "bytesRead", status.getBytesRead()); addField(builder, "bytesWritten", status.getBytesWritten()); addField(builder, "bytesReceived", status.getBytesReceived()); addField(builder, "bytesSent", status.getBytesSent()); addField(builder, "flowFilesRemoved", status.getFlowFilesRemoved()); addField(builder, "flowFilesReceived", status.getFlowFilesReceived()); addField(builder, "flowFilesSent", status.getFlowFilesSent()); addField(builder, "inputCount", status.getInputCount()); addField(builder, "inputBytes", status.getInputBytes()); addField(builder, "outputCount", status.getOutputCount()); addField(builder, "outputBytes", status.getOutputBytes()); addField(builder, "activeThreadCount", status.getActiveThreadCount()); addField(builder, "invocations", status.getInvocations()); addField(builder, "processingNanos", status.getProcessingNanos()); arrayBuilder.add(builder.build()); } }
processorStats.setFlowfilesSent(inputProcessorStatus.getFlowFilesSent()); processorStats.setInvocations(inputProcessorStatus.getInvocations()); processorStats.setProcessingNanos(inputProcessorStatus.getProcessingNanos());
private void serializeProcessorStatus(final JsonArrayBuilder arrayBuilder, final JsonBuilderFactory factory, final ProcessorStatus status, final DateFormat df, final String hostname, final String applicationName, final String platform, final String parentId, final Date currentDate) { final JsonObjectBuilder builder = factory.createObjectBuilder(); final String componentType = "Processor"; final String componentName = status.getName(); if (componentMatchesFilters(componentType, componentName)) { addCommonFields(builder, df, hostname, applicationName, platform, parentId, currentDate, componentType, componentName); addField(builder, "componentId", status.getId()); addField(builder, "processorType", status.getType()); addField(builder, "averageLineageDurationMS", status.getAverageLineageDuration()); addField(builder, "bytesRead", status.getBytesRead()); addField(builder, "bytesWritten", status.getBytesWritten()); addField(builder, "bytesReceived", status.getBytesReceived()); addField(builder, "bytesSent", status.getBytesSent()); addField(builder, "flowFilesRemoved", status.getFlowFilesRemoved()); addField(builder, "flowFilesReceived", status.getFlowFilesReceived()); addField(builder, "flowFilesSent", status.getFlowFilesSent()); addField(builder, "inputCount", status.getInputCount()); addField(builder, "inputBytes", status.getInputBytes()); addField(builder, "outputCount", status.getOutputCount()); addField(builder, "outputBytes", status.getOutputBytes()); addField(builder, "activeThreadCount", status.getActiveThreadCount()); addField(builder, "invocations", status.getInvocations()); addField(builder, "processingNanos", status.getProcessingNanos()); arrayBuilder.add(builder.build()); } }