private void analyzeProcessGroup(final ProcessGroupStatus processGroupStatus, final NiFiFlow nifiFlow) { processGroupStatus.getConnectionStatus().forEach(c -> nifiFlow.addConnection(c)); processGroupStatus.getProcessorStatus().forEach(p -> nifiFlow.addProcessor(p)); processGroupStatus.getRemoteProcessGroupStatus().forEach(r -> nifiFlow.addRemoteProcessGroup(r)); processGroupStatus.getInputPortStatus().forEach(p -> nifiFlow.addInputPort(p)); processGroupStatus.getOutputPortStatus().forEach(p -> nifiFlow.addOutputPort(p)); // Analyze child ProcessGroups recursively. for (ProcessGroupStatus child : processGroupStatus.getProcessGroupStatus()) { analyzeProcessGroup(child, nifiFlow); } }
public Map<String, Double> getDataFlowMetrics(ProcessGroupStatus status) { final Map<String, Double> metrics = new HashMap<>(); metrics.put(MetricNames.FLOW_FILES_RECEIVED, new Double(status.getFlowFilesReceived())); metrics.put(MetricNames.BYTES_RECEIVED, new Double(status.getBytesReceived())); metrics.put(MetricNames.FLOW_FILES_SENT, new Double(status.getFlowFilesSent())); metrics.put(MetricNames.BYTES_SENT, new Double(status.getBytesSent())); metrics.put(MetricNames.FLOW_FILES_QUEUED, new Double(status.getQueuedCount())); metrics.put(MetricNames.BYTES_QUEUED, new Double(status.getQueuedContentSize())); metrics.put(MetricNames.BYTES_READ, new Double(status.getBytesRead())); metrics.put(MetricNames.BYTES_WRITTEN, new Double(status.getBytesWritten())); metrics.put(MetricNames.ACTIVE_THREADS, new Double(status.getActiveThreadCount())); metrics.put(MetricNames.TOTAL_TASK_DURATION, new Double(calculateProcessingNanos(status))); status.getOutputPortStatus(); return metrics; }
private void populateProcessorStatuses(final ProcessGroupStatus groupStatus, final List<ProcessorStatus> statuses) { statuses.addAll(groupStatus.getProcessorStatus()); for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) { populateProcessorStatuses(childGroupStatus, statuses); } }
private void populateConnectionStatuses(final ProcessGroupStatus groupStatus, final List<ConnectionStatus> statuses) { statuses.addAll(groupStatus.getConnectionStatus()); for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) { populateConnectionStatuses(childGroupStatus, statuses); } }
private void populateInputPortStatuses(final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) { statuses.addAll(groupStatus.getInputPortStatus()); for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) { populateInputPortStatuses(childGroupStatus, statuses); } }
final JsonObjectBuilder builder = factory.createObjectBuilder(); final String componentType = (parentId == null) ? "RootProcessGroup" : "ProcessGroup"; final String componentName = status.getName(); addField(builder, "componentId", status.getId()); addField(builder, "bytesRead", status.getBytesRead()); addField(builder, "bytesWritten", status.getBytesWritten()); addField(builder, "bytesReceived", status.getBytesReceived()); addField(builder, "bytesSent", status.getBytesSent()); addField(builder, "bytesTransferred", status.getBytesTransferred()); addField(builder, "flowFilesReceived", status.getFlowFilesReceived()); addField(builder, "flowFilesSent", status.getFlowFilesSent()); addField(builder, "flowFilesTransferred", status.getFlowFilesTransferred()); addField(builder, "inputContentSize", status.getInputContentSize()); addField(builder, "inputCount", status.getInputCount()); addField(builder, "outputContentSize", status.getOutputContentSize()); addField(builder, "outputCount", status.getOutputCount()); addField(builder, "queuedContentSize", status.getQueuedContentSize()); addField(builder, "activeThreadCount", status.getActiveThreadCount()); addField(builder, "queuedCount", status.getQueuedCount()); for(ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { serializeProcessGroupStatus(arrayBuilder, factory, childGroupStatus, df, hostname, applicationName, platform, status.getId(), currentDate); for(ProcessorStatus processorStatus : status.getProcessorStatus()) { serializeProcessorStatus(arrayBuilder, factory, processorStatus, df, hostname, applicationName, platform, status.getId(), currentDate);
target.setInputCount(target.getInputCount() + toMerge.getInputCount()); target.setInputContentSize(target.getInputContentSize() + toMerge.getInputContentSize()); target.setOutputCount(target.getOutputCount() + toMerge.getOutputCount()); target.setOutputContentSize(target.getOutputContentSize() + toMerge.getOutputContentSize()); target.setQueuedCount(target.getQueuedCount() + toMerge.getQueuedCount()); target.setQueuedContentSize(target.getQueuedContentSize() + toMerge.getQueuedContentSize()); target.setBytesRead(target.getBytesRead() + toMerge.getBytesRead()); target.setBytesWritten(target.getBytesWritten() + toMerge.getBytesWritten()); target.setActiveThreadCount(target.getActiveThreadCount() + toMerge.getActiveThreadCount()); target.setTerminatedThreadCount(target.getTerminatedThreadCount() + toMerge.getTerminatedThreadCount()); target.setFlowFilesTransferred(target.getFlowFilesTransferred() + toMerge.getFlowFilesTransferred()); target.setBytesTransferred(target.getBytesTransferred() + toMerge.getBytesTransferred()); target.setFlowFilesReceived(target.getFlowFilesReceived() + toMerge.getFlowFilesReceived()); target.setBytesReceived(target.getBytesReceived() + toMerge.getBytesReceived()); target.setFlowFilesSent(target.getFlowFilesSent() + toMerge.getFlowFilesSent()); target.setBytesSent(target.getBytesSent() + toMerge.getBytesSent()); if (VersionedFlowState.SYNC_FAILURE.equals(toMerge.getVersionedFlowState())) { target.setVersionedFlowState(VersionedFlowState.SYNC_FAILURE); for (final ConnectionStatus status : target.getConnectionStatus()) { mergedConnectionMap.put(status.getId(), status); for (final ConnectionStatus statusToMerge : toMerge.getConnectionStatus()) { ConnectionStatus merged = mergedConnectionMap.get(statusToMerge.getId()); if (merged == null) { target.setConnectionStatus(mergedConnectionMap.values());
final ProcessGroupStatus status = new ProcessGroupStatus(); status.setId(group.getIdentifier()); status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier()); int activeGroupThreads = 0; int terminatedGroupThreads = 0; status.setProcessorStatus(processorStatusCollection); for (final ProcessorNode procNode : group.getProcessors()) { final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized); status.setProcessGroupStatus(localChildGroupStatusCollection); for (final ProcessGroup childGroup : group.getProcessGroups()) { final ProcessGroupStatus childGroupStatus; activeGroupThreads += childGroupStatus.getActiveThreadCount(); terminatedGroupThreads += childGroupStatus.getTerminatedThreadCount(); bytesRead += childGroupStatus.getBytesRead(); bytesWritten += childGroupStatus.getBytesWritten(); queuedCount += childGroupStatus.getQueuedCount(); queuedContentSize += childGroupStatus.getQueuedContentSize(); flowFilesReceived += childGroupStatus.getFlowFilesReceived(); bytesReceived += childGroupStatus.getBytesReceived(); flowFilesSent += childGroupStatus.getFlowFilesSent(); bytesSent += childGroupStatus.getBytesSent(); flowFilesTransferred += childGroupStatus.getFlowFilesTransferred(); bytesTransferred += childGroupStatus.getBytesTransferred();
public ProcessGroupStatusDTO createConciseProcessGroupStatusDto(final ProcessGroupStatus processGroupStatus) { final ProcessGroupStatusDTO processGroupStatusDto = new ProcessGroupStatusDTO(); processGroupStatusDto.setId(processGroupStatus.getId()); processGroupStatusDto.setName(processGroupStatus.getName()); processGroupStatusDto.setStatsLastRefreshed(new Date()); snapshot.setId(processGroupStatus.getId()); snapshot.setName(processGroupStatus.getName()); if (processGroupStatus.getVersionedFlowState() != null) { snapshot.setVersionedFlowState(processGroupStatus.getVersionedFlowState().name()); snapshot.setFlowFilesQueued(processGroupStatus.getQueuedCount()); snapshot.setBytesQueued(processGroupStatus.getQueuedContentSize()); snapshot.setBytesRead(processGroupStatus.getBytesRead()); snapshot.setBytesWritten(processGroupStatus.getBytesWritten()); snapshot.setFlowFilesIn(processGroupStatus.getInputCount()); snapshot.setBytesIn(processGroupStatus.getInputContentSize()); snapshot.setFlowFilesOut(processGroupStatus.getOutputCount()); snapshot.setBytesOut(processGroupStatus.getOutputContentSize()); snapshot.setFlowFilesTransferred(processGroupStatus.getFlowFilesTransferred()); snapshot.setBytesTransferred(processGroupStatus.getBytesTransferred()); snapshot.setFlowFilesSent(processGroupStatus.getFlowFilesSent()); snapshot.setBytesSent(processGroupStatus.getBytesSent()); snapshot.setFlowFilesReceived(processGroupStatus.getFlowFilesReceived()); snapshot.setBytesReceived(processGroupStatus.getBytesReceived()); snapshot.setActiveThreadCount(processGroupStatus.getActiveThreadCount()); snapshot.setTerminatedThreadCount(processGroupStatus.getTerminatedThreadCount());
private Map<String,Long> getLongMetrics(ProcessGroupStatus status, boolean appendPgId) { final Map<String,Long> metrics = new HashMap<>(); metrics.put(appendPgId(MetricNames.BYTES_RECEIVED, status, appendPgId), status.getBytesReceived()); metrics.put(appendPgId(MetricNames.BYTES_SENT, status, appendPgId), status.getBytesSent()); metrics.put(appendPgId(MetricNames.BYTES_QUEUED, status, appendPgId), status.getQueuedContentSize()); metrics.put(appendPgId(MetricNames.BYTES_READ, status, appendPgId), status.getBytesRead()); metrics.put(appendPgId(MetricNames.BYTES_WRITTEN, status, appendPgId), status.getBytesWritten()); final long durationNanos = calculateProcessingNanos(status); metrics.put(appendPgId(MetricNames.TOTAL_TASK_DURATION_NANOS, status, appendPgId), durationNanos); final long durationSeconds = TimeUnit.SECONDS.convert(durationNanos, TimeUnit.NANOSECONDS); metrics.put(appendPgId(MetricNames.TOTAL_TASK_DURATION_SECONDS, status, appendPgId), durationSeconds); return metrics; }
private Map<String,Integer> getIntegerMetrics(ProcessGroupStatus status, boolean appendPgId) { final Map<String,Integer> metrics = new HashMap<>(); metrics.put(appendPgId(MetricNames.FLOW_FILES_RECEIVED, status, appendPgId), status.getFlowFilesReceived()); metrics.put(appendPgId(MetricNames.FLOW_FILES_SENT, status, appendPgId), status.getFlowFilesSent()); metrics.put(appendPgId(MetricNames.FLOW_FILES_QUEUED, status, appendPgId), status.getQueuedCount()); metrics.put(appendPgId(MetricNames.ACTIVE_THREADS, status, appendPgId), status.getActiveThreadCount()); return metrics; }
private void capture(final ProcessGroupStatus groupStatus, final Date timestamp) { // Capture status for the ProcessGroup final ComponentDetails groupDetails = ComponentDetails.forProcessGroup(groupStatus); final StatusSnapshot groupSnapshot = ComponentMetrics.createSnapshot(groupStatus, timestamp); updateStatusHistory(groupSnapshot, groupDetails, timestamp); // Capture statuses for the Processors for (final ProcessorStatus processorStatus : groupStatus.getProcessorStatus()) { final ComponentDetails componentDetails = ComponentDetails.forProcessor(processorStatus); final StatusSnapshot snapshot = ComponentMetrics.createSnapshot(processorStatus, timestamp); updateStatusHistory(snapshot, componentDetails, timestamp); } // Capture statuses for the Connections for (final ConnectionStatus connectionStatus : groupStatus.getConnectionStatus()) { final ComponentDetails componentDetails = ComponentDetails.forConnection(connectionStatus); final StatusSnapshot snapshot = ComponentMetrics.createSnapshot(connectionStatus, timestamp); updateStatusHistory(snapshot, componentDetails, timestamp); } // Capture statuses for the RPG's for (final RemoteProcessGroupStatus rpgStatus : groupStatus.getRemoteProcessGroupStatus()) { final ComponentDetails componentDetails = ComponentDetails.forRemoteProcessGroup(rpgStatus); final StatusSnapshot snapshot = ComponentMetrics.createSnapshot(rpgStatus, timestamp); updateStatusHistory(snapshot, componentDetails, timestamp); } // Capture statuses for the child groups for (final ProcessGroupStatus childStatus : groupStatus.getProcessGroupStatus()) { capture(childStatus, timestamp); } }
/** * Recursively enumertes the ProcessGroupStatus looking for ConnectionStatus that are "busy" based on the user defined parameters. * * @param pg * @return */ private List<ConnectionStatus> searchForBusyConnections(ProcessGroupStatus pg) { List<ConnectionStatus> bcs = new ArrayList<>(); getLogger().info("Examining ProcessGroup: " + pg.getName()); //Gets all of the Connections in this group Collection<ConnectionStatus> connectionStatuses = pg.getConnectionStatus(); Iterator<ConnectionStatus> iterator = connectionStatuses.iterator(); while (iterator.hasNext()) { ConnectionStatus cs = iterator.next(); if (cs.getQueuedCount() > 10) { bcs.add(cs); } } //Recursively call of the ProcessGroups nested inside of this ProcessGroup Iterator<ProcessGroupStatus> pgs = pg.getProcessGroupStatus().iterator(); while (pgs.hasNext()) { bcs.addAll(searchForBusyConnections(pgs.next())); } return bcs; }
private void populateOutputPortStatuses(final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) { statuses.addAll(groupStatus.getOutputPortStatus()); for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) { populateOutputPortStatuses(childGroupStatus, statuses); } }
/** * Gets the status for the specified processor. * * @param processorId processor id * @return the status for the specified processor */ public ProcessorStatus getProcessorStatus(final String processorId) { final ProcessGroup root = getRootGroup(); final ProcessorNode processor = root.findProcessor(processorId); // ensure the processor was found if (processor == null) { throw new ResourceNotFoundException(String.format("Unable to locate processor with id '%s'.", processorId)); } // calculate the process group status final String groupId = processor.getProcessGroup().getIdentifier(); final ProcessGroupStatus processGroupStatus = flowController.getEventAccess().getGroupStatus(groupId, NiFiUserUtils.getNiFiUser(), 1); if (processGroupStatus == null) { throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); } final ProcessorStatus status = processGroupStatus.getProcessorStatus().stream().filter(processorStatus -> processorId.equals(processorStatus.getId())).findFirst().orElse(null); if (status == null) { throw new ResourceNotFoundException(String.format("Unable to locate processor with id '%s'.", processorId)); } return status; }
/** * Gets the status for the specified connection. * * @param connectionId connection id * @return the status for the specified connection */ public ConnectionStatus getConnectionStatus(final String connectionId) { final ProcessGroup root = getRootGroup(); final Connection connection = root.findConnection(connectionId); // ensure the connection was found if (connection == null) { throw new ResourceNotFoundException(String.format("Unable to locate connection with id '%s'.", connectionId)); } // calculate the process group status final String groupId = connection.getProcessGroup().getIdentifier(); final ProcessGroupStatus processGroupStatus = flowController.getEventAccess().getGroupStatus(groupId, NiFiUserUtils.getNiFiUser(), 1); if (processGroupStatus == null) { throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); } final ConnectionStatus status = processGroupStatus.getConnectionStatus().stream().filter(connectionStatus -> connectionId.equals(connectionStatus.getId())).findFirst().orElse(null); if (status == null) { throw new ResourceNotFoundException(String.format("Unable to locate connection with id '%s'.", connectionId)); } return status; }
@Override public Long value() { final ProcessGroupStatus status = latestStatus.get(); if (status == null) { return 0L; } final Long value = status.getBytesRead(); return (value == null) ? 0L : value; } });
@Override public Long value() { final ProcessGroupStatus status = latestStatus.get(); if (status == null) { return 0L; } final Long value = status.getBytesWritten(); return (value == null) ? 0L : value; } });
@Override public Integer value() { final ProcessGroupStatus status = latestStatus.get(); if (status == null) { return 0; } final Integer value = status.getFlowFilesReceived(); return (value == null) ? 0 : value; } });
@Override public Integer value() { final ProcessGroupStatus status = latestStatus.get(); if (status == null) { return 0; } final Integer value = status.getActiveThreadCount(); return (value == null) ? 0 : value; } });