private void serializeConnectionStatus(final JsonArrayBuilder arrayBuilder, final JsonBuilderFactory factory, final ConnectionStatus status, final DateFormat df, final String hostname, final String applicationName, final String platform, final String parentId, final Date currentDate) { final JsonObjectBuilder builder = factory.createObjectBuilder(); final String componentType = "Connection"; final String componentName = status.getName(); if (componentMatchesFilters(componentType, componentName)) { addCommonFields(builder, df, hostname, applicationName, platform, parentId, currentDate, componentType, componentName); addField(builder, "componentId", status.getId()); addField(builder, "sourceId", status.getSourceId()); addField(builder, "sourceName", status.getSourceName()); addField(builder, "destinationId", status.getDestinationId()); addField(builder, "destinationName", status.getDestinationName()); addField(builder, "maxQueuedBytes", status.getMaxQueuedBytes()); addField(builder, "maxQueuedCount", status.getMaxQueuedCount()); addField(builder, "queuedBytes", status.getQueuedBytes()); addField(builder, "queuedCount", status.getQueuedCount()); addField(builder, "inputBytes", status.getInputBytes()); addField(builder, "inputCount", status.getInputCount()); addField(builder, "outputBytes", status.getOutputBytes()); addField(builder, "outputCount", status.getOutputCount()); addField(builder, "backPressureBytesThreshold", status.getBackPressureBytesThreshold()); addField(builder, "backPressureObjectThreshold", status.getBackPressureObjectThreshold()); addField(builder, "isBackPressureEnabled", Boolean.toString((status.getBackPressureObjectThreshold() > 0 && status.getBackPressureObjectThreshold() <= status.getQueuedCount()) || (status.getBackPressureBytesThreshold() > 0 && status.getBackPressureBytesThreshold() <= status.getMaxQueuedBytes()))); arrayBuilder.add(builder.build()); } }
public static ComponentDetails forConnection(final ConnectionStatus status) { return forConnection(status.getId(), status.getGroupId(), status.getName(), status.getSourceName(), status.getDestinationName()); }
public Map<String,String> getConnectionStatusTags(ConnectionStatus status) { final Map<String, String> connectionTags = new HashMap<>(); connectionTags.put(MetricNames.CONNECTION_ID, status.getId()); connectionTags.put(MetricNames.CONNECTION_NAME, status.getName()); connectionTags.put(MetricNames.CONNECTION_GROUP_ID, status.getGroupId()); connectionTags.put(MetricNames.CONNECTION_DESTINATION_ID, status.getDestinationId()); connectionTags.put(MetricNames.CONNECTTION_DESTINATION_NAME, status.getDestinationName()); connectionTags.put(MetricNames.CONNECTION_SOURCE_ID, status.getSourceId()); connectionTags.put(MetricNames.CONNECTION_SOURCE_NAME, status.getSourceName()); return connectionTags; }
public Map<String, Double> getConnectionStatusMetrics(ConnectionStatus status) { final Map<String, Double> metrics = new HashMap<>(); metrics.put(MetricNames.INPUT_COUNT, new Double(status.getInputCount())); metrics.put(MetricNames.INPUT_BYTES, new Double(status.getInputBytes())); metrics.put(MetricNames.QUEUED_COUNT, new Double(status.getQueuedCount())); metrics.put(MetricNames.QUEUED_BYTES, new Double(status.getQueuedBytes())); metrics.put(MetricNames.OUTPUT_COUNT, new Double(status.getOutputCount())); metrics.put(MetricNames.OUTPUT_BYTES, new Double(status.getOutputBytes())); return metrics; }
final String input = connectionStatus.getInputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getInputBytes()); final String output = connectionStatus.getOutputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getOutputBytes()); final String queued = connectionStatus.getQueuedCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getQueuedBytes()); final ConnectionStatus lastStatus = lastConnectionStatus.get(connectionStatus.getId()); if (showDeltas && lastStatus != null) { inputDiff = toDiff(lastStatus.getInputCount(), lastStatus.getInputBytes(), connectionStatus.getInputCount(), connectionStatus.getInputBytes()); outputDiff = toDiff(lastStatus.getOutputCount(), lastStatus.getOutputBytes(), connectionStatus.getOutputCount(), connectionStatus.getOutputBytes()); queuedDiff = toDiff(lastStatus.getQueuedCount(), lastStatus.getQueuedBytes(), connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes()); } else { inputDiff = toDiff(0L, 0L, connectionStatus.getInputCount(), connectionStatus.getInputBytes()); outputDiff = toDiff(0L, 0L, connectionStatus.getOutputCount(), connectionStatus.getOutputBytes()); queuedDiff = toDiff(0L, 0L, connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes()); connectionStatus.getId(), connectionStatus.getSourceName(), connectionStatus.getName(), connectionStatus.getDestinationName(), input + inputDiff, output + outputDiff, } else { builder.append(String.format(connectionLineFormat, connectionStatus.getId(), connectionStatus.getSourceName(), connectionStatus.getName(), connectionStatus.getDestinationName(), input, output,
mergedConnectionMap.put(status.getId(), status); ConnectionStatus merged = mergedConnectionMap.get(statusToMerge.getId()); if (merged == null) { mergedConnectionMap.put(statusToMerge.getId(), statusToMerge.clone()); continue; merged.setQueuedCount(merged.getQueuedCount() + statusToMerge.getQueuedCount()); merged.setQueuedBytes(merged.getQueuedBytes() + statusToMerge.getQueuedBytes()); merged.setInputCount(merged.getInputCount() + statusToMerge.getInputCount()); merged.setInputBytes(merged.getInputBytes() + statusToMerge.getInputBytes()); merged.setOutputCount(merged.getOutputCount() + statusToMerge.getOutputCount()); merged.setOutputBytes(merged.getOutputBytes() + statusToMerge.getOutputBytes());
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { if (!ProvenanceEventType.SEND.equals(event.getEventType()) && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) { return null; } final boolean isRemoteInputPort = event.getComponentType().equals("Remote Input Port"); final String type = isRemoteInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; final S2SPort s2SPort = analyzeS2SPort(event, context.getClusterResolver()); // Find connections that connects to/from the remote port. final String componentId = event.getComponentId(); final List<ConnectionStatus> connections = isRemoteInputPort ? context.findConnectionTo(componentId) : context.findConnectionFrom(componentId); if (connections == null || connections.isEmpty()) { logger.warn("Connection was not found: {}", new Object[]{event}); return null; } // The name of remote port can be retrieved from any connection, use the first one. final ConnectionStatus connection = connections.get(0); final Referenceable ref = new Referenceable(type); ref.set(ATTR_NAME, isRemoteInputPort ? connection.getDestinationName() : connection.getSourceName()); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.clusterName, s2SPort.targetPortId)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
private static <E> Map<String, E> transformStatusCollection(Collection<E> statusCollection) { Map<String, E> statusMap = new HashMap<>(); for (E status : statusCollection) { if (status instanceof ProcessorStatus) { statusMap.put(((ProcessorStatus) status).getId(), status); if (((ProcessorStatus) status).getName() != null) { statusMap.put(((ProcessorStatus) status).getName(), status); } } else if (status instanceof ConnectionStatus) { statusMap.put(((ConnectionStatus) status).getId(), status); if (((ConnectionStatus) status).getName() != null) { statusMap.put(((ConnectionStatus) status).getName(), status); } } else if (status instanceof RemoteProcessGroupStatus) { statusMap.put(((RemoteProcessGroupStatus) status).getId(), status); if (((RemoteProcessGroupStatus) status).getName() != null) { statusMap.put(((RemoteProcessGroupStatus) status).getName(), status); } } } return statusMap; }
/** * Gets the status for the specified connection. * * @param connectionId connection id * @return the status for the specified connection */ public ConnectionStatus getConnectionStatus(final String connectionId) { final ProcessGroup root = getRootGroup(); final Connection connection = root.findConnection(connectionId); // ensure the connection was found if (connection == null) { throw new ResourceNotFoundException(String.format("Unable to locate connection with id '%s'.", connectionId)); } // calculate the process group status final String groupId = connection.getProcessGroup().getIdentifier(); final ProcessGroupStatus processGroupStatus = flowController.getEventAccess().getGroupStatus(groupId, NiFiUserUtils.getNiFiUser(), 1); if (processGroupStatus == null) { throw new ResourceNotFoundException(String.format("Unable to locate group with id '%s'.", groupId)); } final ConnectionStatus status = processGroupStatus.getConnectionStatus().stream().filter(connectionStatus -> connectionId.equals(connectionStatus.getId())).findFirst().orElse(null); if (status == null) { throw new ResourceNotFoundException(String.format("Unable to locate connection with id '%s'.", connectionId)); } return status; }
@Override public int compare(final ConnectionStatus o1, final ConnectionStatus o2) { if (o1 == null && o2 == null) { return 0; } if (o1 == null) { return 1; } if (o2 == null) { return -1; } return -Long.compare(o1.getQueuedBytes(), o2.getQueuedBytes()); } });
getLogger().info("\tBusy Connection: " + cs.getName());
/** * Recursively enumertes the ProcessGroupStatus looking for ConnectionStatus that are "busy" based on the user defined parameters. * * @param pg * @return */ private List<ConnectionStatus> searchForBusyConnections(ProcessGroupStatus pg) { List<ConnectionStatus> bcs = new ArrayList<>(); getLogger().info("Examining ProcessGroup: " + pg.getName()); //Gets all of the Connections in this group Collection<ConnectionStatus> connectionStatuses = pg.getConnectionStatus(); Iterator<ConnectionStatus> iterator = connectionStatuses.iterator(); while (iterator.hasNext()) { ConnectionStatus cs = iterator.next(); if (cs.getQueuedCount() > 10) { bcs.add(cs); } } //Recursively call of the ProcessGroups nested inside of this ProcessGroup Iterator<ProcessGroupStatus> pgs = pg.getProcessGroupStatus().iterator(); while (pgs.hasNext()) { bcs.addAll(searchForBusyConnections(pgs.next())); } return bcs; }
static ConnectionStatusBean parseConnectionStatusRequest(ConnectionStatus inputConnectionStatus, String statusTypes, Logger logger) { ConnectionStatusBean connectionStatusBean = new ConnectionStatusBean(); connectionStatusBean.setId(inputConnectionStatus.getId()); connectionStatusBean.setName(inputConnectionStatus.getName()); String[] statusSplits = statusTypes.split(","); for (String statusType : statusSplits) { switch (statusType.toLowerCase().trim()) { case "health": ConnectionHealth connectionHealth = new ConnectionHealth(); connectionHealth.setQueuedBytes(inputConnectionStatus.getQueuedBytes()); connectionHealth.setQueuedCount(inputConnectionStatus.getQueuedCount()); connectionStatusBean.setConnectionHealth(connectionHealth); break; case "stats": ConnectionStats connectionStats = new ConnectionStats(); connectionStats.setInputBytes(inputConnectionStatus.getInputBytes()); connectionStats.setInputCount(inputConnectionStatus.getInputCount()); connectionStats.setOutputCount(inputConnectionStatus.getOutputCount()); connectionStats.setOutputBytes(inputConnectionStatus.getOutputBytes()); connectionStatusBean.setConnectionStats(connectionStats); break; } } return connectionStatusBean; }
componentNameMap.put(connectionStatus.getId(), connectionStatus.getName()); componentToParentGroupMap.put(connectionStatus.getId(), thisProcessGroupNode); componentNameMap.computeIfAbsent(connectionStatus.getSourceId(), k -> connectionStatus.getSourceName()); componentNameMap.computeIfAbsent(connectionStatus.getDestinationId(), k -> connectionStatus.getDestinationName()); sourceToConnectionParentGroupMap.put(connectionStatus.getSourceId(), connectionStatus.getGroupId()); destinationToConnectionParentGroupMap.put(connectionStatus.getDestinationId(), connectionStatus.getGroupId());
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { if (!ProvenanceEventType.SEND.equals(event.getEventType()) && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) { return null; } final boolean isInputPort = event.getComponentType().equals("Input Port"); final String type = isInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; final String rootPortId = event.getComponentId(); final S2SPort s2SPort = analyzeS2SPort(event, context.getClusterResolver()); // Find connections connecting to/from the remote port. final List<ConnectionStatus> connections = isInputPort ? context.findConnectionFrom(rootPortId) : context.findConnectionTo(rootPortId); if (connections == null || connections.isEmpty()) { logger.warn("Connection was not found: {}", new Object[]{event}); return null; } // The name of the port can be retrieved from any connection, use the first one. final ConnectionStatus connection = connections.get(0); final Referenceable ref = new Referenceable(type); ref.set(ATTR_NAME, isInputPort ? connection.getSourceName() : connection.getDestinationName()); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.clusterName, rootPortId)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
final PermissionsDTO permissions = createPermissionsDto(connNode); final ConnectionStatusDTO status = getComponentStatus( () -> groupStatus.getConnectionStatus().stream().filter(connectionStatus -> connNode.getIdentifier().equals(connectionStatus.getId())).findFirst().orElse(null), connectionStatus -> createConnectionStatusDto(connectionStatus) );
public ConnectionStatusDTO createConnectionStatusDto(final ConnectionStatus connectionStatus) { final ConnectionStatusDTO connectionStatusDto = new ConnectionStatusDTO(); connectionStatusDto.setGroupId(connectionStatus.getGroupId()); connectionStatusDto.setId(connectionStatus.getId()); connectionStatusDto.setName(connectionStatus.getName()); connectionStatusDto.setSourceId(connectionStatus.getSourceId()); connectionStatusDto.setSourceName(connectionStatus.getSourceName()); connectionStatusDto.setDestinationId(connectionStatus.getDestinationId()); connectionStatusDto.setDestinationName(connectionStatus.getDestinationName()); connectionStatusDto.setStatsLastRefreshed(new Date()); snapshot.setId(connectionStatus.getId()); snapshot.setGroupId(connectionStatus.getGroupId()); snapshot.setName(connectionStatus.getName()); snapshot.setSourceName(connectionStatus.getSourceName()); snapshot.setDestinationName(connectionStatus.getDestinationName()); snapshot.setFlowFilesQueued(connectionStatus.getQueuedCount()); snapshot.setBytesQueued(connectionStatus.getQueuedBytes()); snapshot.setFlowFilesIn(connectionStatus.getInputCount()); snapshot.setBytesIn(connectionStatus.getInputBytes()); snapshot.setFlowFilesOut(connectionStatus.getOutputCount()); snapshot.setBytesOut(connectionStatus.getOutputBytes()); if (connectionStatus.getBackPressureObjectThreshold() > 0) { snapshot.setPercentUseCount(Math.min(100, StatusMerger.getUtilization(connectionStatus.getQueuedCount(), connectionStatus.getBackPressureObjectThreshold()))); if (connectionStatus.getBackPressureBytesThreshold() > 0) {
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { if (!ProvenanceEventType.SEND.equals(event.getEventType()) && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) { return null; } final boolean isRemoteInputPort = event.getComponentType().equals("Remote Input Port"); final String type = isRemoteInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; final S2SPort s2SPort = analyzeS2SPort(event, context.getClusterResolver()); // Find connections that connects to/from the remote port. final String componentId = event.getComponentId(); final List<ConnectionStatus> connections = isRemoteInputPort ? context.findConnectionTo(componentId) : context.findConnectionFrom(componentId); if (connections == null || connections.isEmpty()) { logger.warn("Connection was not found: {}", new Object[]{event}); return null; } // The name of remote port can be retrieved from any connection, use the first one. final ConnectionStatus connection = connections.get(0); final Referenceable ref = new Referenceable(type); ref.set(ATTR_NAME, isRemoteInputPort ? connection.getDestinationName() : connection.getSourceName()); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.clusterName, s2SPort.targetPortId)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
final PermissionsDTO accessPolicy = createPermissionsDto(connection); final ConnectionStatusDTO status = getComponentStatus( () -> groupStatus.getConnectionStatus().stream().filter(connectionStatus -> connection.getIdentifier().equals(connectionStatus.getId())).findFirst().orElse(null), connectionStatus -> createConnectionStatusDto(connectionStatus) );
private void serializeConnectionStatus(final JsonArrayBuilder arrayBuilder, final JsonBuilderFactory factory, final ConnectionStatus status, final DateFormat df, final String hostname, final String applicationName, final String platform, final String parentId, final Date currentDate) { final JsonObjectBuilder builder = factory.createObjectBuilder(); final String componentType = "Connection"; final String componentName = status.getName(); if (componentMatchesFilters(componentType, componentName)) { addCommonFields(builder, df, hostname, applicationName, platform, parentId, currentDate, componentType, componentName); addField(builder, "componentId", status.getId()); addField(builder, "sourceId", status.getSourceId()); addField(builder, "sourceName", status.getSourceName()); addField(builder, "destinationId", status.getDestinationId()); addField(builder, "destinationName", status.getDestinationName()); addField(builder, "maxQueuedBytes", status.getMaxQueuedBytes()); addField(builder, "maxQueuedCount", status.getMaxQueuedCount()); addField(builder, "queuedBytes", status.getQueuedBytes()); addField(builder, "queuedCount", status.getQueuedCount()); addField(builder, "inputBytes", status.getInputBytes()); addField(builder, "inputCount", status.getInputCount()); addField(builder, "outputBytes", status.getOutputBytes()); addField(builder, "outputCount", status.getOutputCount()); addField(builder, "backPressureBytesThreshold", status.getBackPressureBytesThreshold()); addField(builder, "backPressureObjectThreshold", status.getBackPressureObjectThreshold()); addField(builder, "isBackPressureEnabled", Boolean.toString((status.getBackPressureObjectThreshold() > 0 && status.getBackPressureObjectThreshold() <= status.getQueuedCount()) || (status.getBackPressureBytesThreshold() > 0 && status.getBackPressureBytesThreshold() <= status.getMaxQueuedBytes()))); arrayBuilder.add(builder.build()); } }