/** * Extracts configured settings from the specified connection only if they have also been specified in the connectionDTO. * * @param connection connection * @param connectionDTO dto * @return properties */ private Map<String, String> extractConfiguredPropertyValues(Connection connection, ConnectionDTO connectionDTO) { Map<String, String> values = new HashMap<>(); if (connectionDTO.getName() != null) { values.put(NAME, connection.getName()); } if (connectionDTO.getFlowFileExpiration() != null) { values.put(FLOW_FILE_EXPIRATION, String.valueOf(connection.getFlowFileQueue().getFlowFileExpiration())); } if (connectionDTO.getBackPressureObjectThreshold() != null) { values.put(BACK_PRESSURE_OBJECT_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureObjectThreshold())); } if (connectionDTO.getBackPressureDataSizeThreshold() != null) { values.put(BACK_PRESSURE_DATA_SIZE_THRESHOLD, String.valueOf(connection.getFlowFileQueue().getBackPressureDataSizeThreshold())); } if (connectionDTO.getPrioritizers() != null) { List<String> prioritizers = new ArrayList<>(); for (FlowFilePrioritizer prioritizer : connection.getFlowFileQueue().getPriorities()) { prioritizers.add(prioritizer.getClass().getCanonicalName()); } values.put(PRIORITIZERS, StringUtils.join(prioritizers, ", ")); } return values; }
@Override public String toString() { return "SwappablePriorityQueue[queueId=" + flowFileQueue.getIdentifier() + "]"; } }
@Override public Set<Relationship> getAvailableRelationships() { for (final Connection connection : connectable.getConnections()) { if (connection.getFlowFileQueue().isFull()) { return Collections.emptySet(); } } final Collection<Relationship> relationships = connectable.getRelationships(); if (relationships instanceof Set) { return (Set<Relationship>) relationships; } return new HashSet<>(connectable.getRelationships()); }
queue.setBackPressureDataSizeThreshold(proposed.getBackPressureDataSizeThreshold()); queue.setBackPressureObjectThreshold(proposed.getBackPressureObjectThreshold()); queue.setFlowFileExpiration(proposed.getFlowFileExpiration()); .collect(Collectors.toList()); queue.setPriorities(prioritizers); queue.setLoadBalanceStrategy(LoadBalanceStrategy.DO_NOT_LOAD_BALANCE, proposed.getPartitioningAttribute()); } else { final LoadBalanceStrategy loadBalanceStrategy = LoadBalanceStrategy.valueOf(loadBalanceStrategyName); final String partitioningAttribute = proposed.getPartitioningAttribute(); queue.setLoadBalanceStrategy(loadBalanceStrategy, partitioningAttribute); queue.setLoadBalanceCompression(LoadBalanceCompression.DO_NOT_COMPRESS); } else { queue.setLoadBalanceCompression(LoadBalanceCompression.valueOf(compressionName));
public VersionedConnection mapConnection(final Connection connection) { final FlowFileQueue queue = connection.getFlowFileQueue(); final VersionedConnection versionedConnection = new InstantiatedVersionedConnection(connection.getIdentifier(), connection.getProcessGroup().getIdentifier()); versionedConnection.setIdentifier(getId(connection.getVersionedComponentId(), connection.getIdentifier())); versionedConnection.setGroupIdentifier(getGroupId(connection.getProcessGroup().getIdentifier())); versionedConnection.setName(connection.getName()); versionedConnection.setBackPressureDataSizeThreshold(queue.getBackPressureDataSizeThreshold()); versionedConnection.setBackPressureObjectThreshold(queue.getBackPressureObjectThreshold()); versionedConnection.setFlowFileExpiration(queue.getFlowFileExpiration()); versionedConnection.setLabelIndex(connection.getLabelIndex()); versionedConnection.setPrioritizers(queue.getPriorities().stream().map(p -> p.getClass().getName()).collect(Collectors.toList())); versionedConnection.setSelectedRelationships(connection.getRelationships().stream().map(Relationship::getName).collect(Collectors.toSet())); versionedConnection.setzIndex(connection.getZIndex()); final FlowFileQueue flowFileQueue = connection.getFlowFileQueue(); versionedConnection.setLoadBalanceStrategy(flowFileQueue.getLoadBalanceStrategy().name()); versionedConnection.setPartitioningAttribute(flowFileQueue.getPartitioningAttribute()); versionedConnection.setLoadBalanceCompression(flowFileQueue.getLoadBalanceCompression().name()); versionedConnection.setBends(connection.getBendPoints().stream() .map(this::mapPosition) .collect(Collectors.toList())); versionedConnection.setSource(mapConnectable(connection.getSource())); versionedConnection.setDestination(mapConnectable(connection.getDestination())); return versionedConnection; }
connStatus.setDestinationId(conn.getDestination().getIdentifier()); connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier()); connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold()); connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold()); final QueueSize queueSize = conn.getFlowFileQueue().size(); final int connectionQueuedCount = queueSize.getObjectCount(); final long connectionQueuedBytes = queueSize.getByteCount();
@Override public FlowFileQueue createFlowFileQueue(final LoadBalanceStrategy loadBalanceStrategy, final String partitioningAttribute, final ConnectionEventListener eventListener) { final FlowFileQueue flowFileQueue; if (clusterCoordinator == null) { flowFileQueue = new StandardFlowFileQueue(id, eventListener, flowFileRepository, provenanceRepository, resourceClaimManager, processScheduler, swapManager, eventReporter, nifiProperties.getQueueSwapThreshold(), nifiProperties.getDefaultBackPressureObjectThreshold(), nifiProperties.getDefaultBackPressureDataSizeThreshold()); } else { flowFileQueue = new SocketLoadBalancedFlowFileQueue(id, eventListener, processScheduler, flowFileRepository, provenanceRepository, contentRepository, resourceClaimManager, clusterCoordinator, loadBalanceClientRegistry, swapManager, nifiProperties.getQueueSwapThreshold(), eventReporter); flowFileQueue.setBackPressureObjectThreshold(nifiProperties.getDefaultBackPressureObjectThreshold()); flowFileQueue.setBackPressureDataSizeThreshold(nifiProperties.getDefaultBackPressureDataSizeThreshold()); } return flowFileQueue; } };
final Map<String, FlowFileQueue> queueMap = new HashMap<>(); for (final FlowFileQueue queue : queueProvider.getAllQueues()) { queueMap.put(queue.getIdentifier(), queue); numFlowFilesMissingQueue++; } else { queue.put(flowFile);
private int getExpiration() { return flowFileQueue.getFlowFileExpiration(TimeUnit.MILLISECONDS); }
final LoadBalanceCompression compression = connection.getFlowFileQueue().getLoadBalanceCompression(); logger.debug("Receiving FlowFiles from Peer {} for Connection {}; Compression = {}", peerDescription, connectionId, compression);
connection.getFlowFileQueue().setFlowFileExpiration(connectionDTO.getFlowFileExpiration()); connection.getFlowFileQueue().setBackPressureObjectThreshold(connectionDTO.getBackPressureObjectThreshold()); connection.getFlowFileQueue().setBackPressureDataSizeThreshold(connectionDTO.getBackPressureDataSizeThreshold()); connection.getFlowFileQueue().setPriorities(newPrioritizers); if (isNotNull(loadBalanceStrategyName)) { final LoadBalanceStrategy loadBalanceStrategy = LoadBalanceStrategy.valueOf(loadBalanceStrategyName); connection.getFlowFileQueue().setLoadBalanceStrategy(loadBalanceStrategy, loadBalancePartitionAttribute); connection.getFlowFileQueue().setLoadBalanceCompression(LoadBalanceCompression.valueOf(loadBalanceCompressionName));
dto.setBackPressureObjectThreshold(flowFileQueue.getBackPressureObjectThreshold()); dto.setBackPressureDataSizeThreshold(flowFileQueue.getBackPressureDataSizeThreshold()); dto.setFlowFileExpiration(flowFileQueue.getFlowFileExpiration()); dto.setPrioritizers(new ArrayList<String>()); for (final FlowFilePrioritizer comparator : flowFileQueue.getPriorities()) { dto.getPrioritizers().add(comparator.getClass().getCanonicalName()); final LoadBalanceStrategy loadBalanceStrategy = flowFileQueue.getLoadBalanceStrategy(); dto.setLoadBalancePartitionAttribute(flowFileQueue.getPartitioningAttribute()); dto.setLoadBalanceStrategy(loadBalanceStrategy.name()); dto.setLoadBalanceCompression(flowFileQueue.getLoadBalanceCompression().name()); } else if (flowFileQueue.isActivelyLoadBalancing()) { dto.setLoadBalanceStatus(ConnectionDTO.LOAD_BALANCE_ACTIVE); } else {
private void expireFlowFiles(final Connectable connectable) { // determine if the incoming connections for this Connectable have Expiration configured. boolean expirationConfigured = false; for (final Connection incomingConn : connectable.getIncomingConnections()) { if (FormatUtils.getTimeDuration(incomingConn.getFlowFileQueue().getFlowFileExpiration(), TimeUnit.MILLISECONDS) > 0) { expirationConfigured = true; break; } } // If expiration is not configured... don't bother running through the FlowFileQueue if (!expirationConfigured) { return; } final StandardProcessSession session = createSession(connectable); session.expireFlowFiles(); session.commit(); }
connection.getFlowFileQueue().setPriorities(newPrioritizers); connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold()); connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold()); connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration()); connection.getFlowFileQueue().setLoadBalanceStrategy(LoadBalanceStrategy.valueOf(dto.getLoadBalanceStrategy()), dto.getLoadBalancePartitionAttribute()); connection.getFlowFileQueue().setLoadBalanceCompression(LoadBalanceCompression.valueOf(dto.getLoadBalanceCompression()));
addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold()); addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold()); addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration()); for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) { final String className = comparator.getClass().getCanonicalName(); addTextElement(element, "queuePrioritizerClass", className); addTextElement(element, "loadBalanceStrategy", connection.getFlowFileQueue().getLoadBalanceStrategy().name()); addTextElement(element, "partitioningAttribute", connection.getFlowFileQueue().getPartitioningAttribute()); addTextElement(element, "loadBalanceCompression", connection.getFlowFileQueue().getLoadBalanceCompression().name());
for (final FlowFilePrioritizer comparator : queue.getPriorities()) { addIfAppropriate(searchStr, comparator.getClass().getName(), "Prioritizer", matches); final int expirationMillis = connection.getFlowFileQueue().getFlowFileExpiration(TimeUnit.MILLISECONDS); if (expirationMillis > 0) { matches.add("FlowFile expiration: " + connection.getFlowFileQueue().getFlowFileExpiration()); final String backPressureDataSize = connection.getFlowFileQueue().getBackPressureDataSizeThreshold(); final Double backPressureBytes = DataUnit.parseDataSize(backPressureDataSize, DataUnit.B); if (backPressureBytes > 0) { final long backPressureCount = connection.getFlowFileQueue().getBackPressureObjectThreshold(); if (backPressureCount > 0) { matches.add("Back pressure count: " + backPressureCount);
private String getQueueIdentifier() { return flowFileQueue.getIdentifier(); }
public Set<Relationship> getAvailableRelationships() { final Set<Relationship> set = new HashSet<>(); for (final Relationship relationship : getConnectable().getRelationships()) { final Collection<Connection> connections = getConnections(relationship); if (connections.isEmpty()) { set.add(relationship); } else { boolean available = true; for (final Connection connection : connections) { if (connection.getFlowFileQueue().isFull()) { available = false; } } if (available) { set.add(relationship); } } } return set; } }
record.markForDelete(); expiredRecords.add(record); expiredReporter.expire(flowFile, "Expiration Threshold = " + connection.getFlowFileQueue().getFlowFileExpiration()); decrementClaimCount(flowFile.getContentClaim());
connection.getFlowFileQueue().setPriorities(newPrioritizers); connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold()); connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold()); connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration());