private NodeIdentifier addNodeIdentities(final NodeIdentifier nodeId, final Set<String> nodeIdentities) { return new NodeIdentifier(nodeId.getId(), nodeId.getApiAddress(), nodeId.getApiPort(), nodeId.getSocketAddress(), nodeId.getSocketPort(), nodeId.getLoadBalanceAddress(), nodeId.getLoadBalancePort(), nodeId.getSiteToSiteAddress(), nodeId.getSiteToSitePort(), nodeId.getSiteToSiteHttpApiPort(), nodeId.isSiteToSiteSecure(), nodeIdentities); }
@Override public void mergeStatus(PortStatusDTO clientStatus, boolean clientStatusReadablePermission, PortStatusDTO status, boolean statusReadablePermission, NodeIdentifier statusNodeIdentifier) { StatusMerger.merge(clientStatus, clientStatusReadablePermission, status, statusReadablePermission, statusNodeIdentifier.getId(), statusNodeIdentifier.getApiAddress(), statusNodeIdentifier.getApiPort()); } }
private URI createURI(final URI exampleUri, final NodeIdentifier nodeId) { return createURI(exampleUri.getScheme(), nodeId.getApiAddress(), nodeId.getApiPort(), exampleUri.getPath(), exampleUri.getQuery()); }
@Override public ClusterNodeInformation getNodeInformation() { final List<NodeInformation> nodeInfoCollection; try { nodeInfoCollection = clusterCoordinator.getClusterWorkload().entrySet().stream().map(entry -> { final NodeIdentifier nodeId = entry.getKey(); final NodeInformation nodeInfo = new NodeInformation(nodeId.getSiteToSiteAddress(), nodeId.getSiteToSitePort(), nodeId.getSiteToSiteHttpApiPort(), nodeId.getApiPort(), nodeId.isSiteToSiteSecure(), entry.getValue().getFlowFileCount()); return nodeInfo; }).collect(Collectors.toList()); } catch (IOException e) { throw new RuntimeException("Failed to retrieve cluster workload due to " + e, e); } final ClusterNodeInformation nodeInfo = new ClusterNodeInformation(); nodeInfo.setNodeInformation(nodeInfoCollection); return nodeInfo; }
final Map<NodeIdentifier, NodeWorkload> clusterWorkload = clusterCoordinator.getClusterWorkload(); clusterWorkload.forEach((nodeId, workload) -> { final String siteToSiteHostname = nodeId.getSiteToSiteAddress() == null ? nodeId.getApiAddress() : nodeId.getSiteToSiteAddress(); final int siteToSitePort = nodeId.getSiteToSiteHttpApiPort() == null ? nodeId.getApiPort() : nodeId.getSiteToSiteHttpApiPort(); PeerDescription target = new PeerDescription(siteToSiteHostname, siteToSitePort, nodeId.isSiteToSiteSecure());
@Override public String getSwapPartitionName() { return nodeIdentifier.getId(); }
sortedNodeIdentifiers.sort(Comparator.comparing(nodeId -> nodeId.getApiAddress() + ":" + nodeId.getApiPort())); for (int i = 0; i < sortedNodeIdentifiers.size(); i++) { final NodeIdentifier nodeId = sortedNodeIdentifiers.get(i); if (nodeId.equals(clusterCoordinator.getLocalNodeIdentifier())) { updatedQueuePartitions[i] = localPartition; localPartitionIncluded = true;
private SocketChannel createChannel() throws IOException { final SocketChannel socketChannel = SocketChannel.open(); try { socketChannel.configureBlocking(true); final Socket socket = socketChannel.socket(); socket.setSoTimeout(timeoutMillis); socket.connect(new InetSocketAddress(nodeIdentifier.getLoadBalanceAddress(), nodeIdentifier.getLoadBalancePort())); socket.setSoTimeout(timeoutMillis); return socketChannel; } catch (final Exception e) { try { socketChannel.close(); } catch (final Exception closeException) { e.addSuppressed(closeException); } throw e; } }
private void storeState() { final ObjectMapper mapper = new ObjectMapper(); final JsonFactory jsonFactory = new JsonFactory(); jsonFactory.setCodec(mapper); try { final Map<String, String> stateMap = new HashMap<>(); final NodeIdentifier localNodeId = getLocalNodeIdentifier(); for (final NodeIdentifier nodeId : getNodeIdentifiers()) { final boolean isLocalId = nodeId.equals(localNodeId); final NodeIdentifierDescriptor descriptor = NodeIdentifierDescriptor.fromNodeIdentifier(nodeId, isLocalId); try (final StringWriter writer = new StringWriter()) { final JsonGenerator jsonGenerator = jsonFactory.createGenerator(writer); jsonGenerator.writeObject(descriptor); final String serializedDescriptor = writer.toString(); stateMap.put(nodeId.getId(), serializedDescriptor); } } stateManager.setState(stateMap, Scope.LOCAL); logger.debug("Stored the following state as the Cluster Topology: {}", stateMap); } catch (final Exception e) { logger.warn("Failed to store cluster topology to local State Manager. Upon restart of NiFi, the cluster topology may not be accurate until joining the cluster.", e); } }
response.setFailure(failure, new NodeIdentifier());
private ProvenanceEventRecord createSendEvent(final FlowFileRecord flowFile, final NodeIdentifier nodeIdentifier) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.SEND) .setDetails("Re-distributed for Load-balanced connection") .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()) .setSourceSystemFlowFileIdentifier(flowFile.getAttribute(CoreAttributes.UUID.key())) .setTransitUri("nifi://" + nodeIdentifier.getApiAddress() + "/loadbalance/" + flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord sendEvent = builder.build(); return sendEvent; }
@Override public void mergeStatus(ProcessGroupStatusDTO clientStatus, boolean clientStatusReadablePermission, ProcessGroupStatusDTO status, boolean statusReadablePermission, NodeIdentifier statusNodeIdentifier) { StatusMerger.merge(clientStatus, clientStatusReadablePermission, status, statusReadablePermission, statusNodeIdentifier.getId(), statusNodeIdentifier.getApiAddress(), statusNodeIdentifier.getApiPort()); }
@Override public String toString() { final StringBuilder sb = new StringBuilder(); sb.append("NodeResponse[nodeUri=").append(nodeId.getApiAddress()).append(":").append(nodeId.getApiPort()).append(",") .append("method=").append(httpMethod) .append(",URI=").append(requestUri) .append(",ResponseCode=").append(getStatus()) .append(",Duration=").append(TimeUnit.MILLISECONDS.convert(requestDurationNanos, TimeUnit.NANOSECONDS)).append(" ms]"); return sb.toString(); } }
@Override public NodeIdentifier getNodeIdentifier(final String uuid) { for (final NodeIdentifier nodeId : nodeStatuses.keySet()) { if (nodeId.getId().equals(uuid)) { return nodeId; } } return null; }
this.nodeId = new NodeIdentifier(nodeUuid, nodeApiAddress.getHostName(), nodeApiAddress.getPort(), nodeSocketAddress.getHostName(), nodeSocketAddress.getPort(),
private NodeIdentifier resolveNodeId(final NodeIdentifier proposedIdentifier) { final NodeConnectionStatus proposedConnectionStatus = new NodeConnectionStatus(proposedIdentifier, DisconnectionCode.NOT_YET_CONNECTED); final NodeConnectionStatus existingStatus = nodeStatuses.putIfAbsent(proposedIdentifier, proposedConnectionStatus); NodeIdentifier resolvedNodeId = proposedIdentifier; if (existingStatus == null) { // there is no node with that ID resolvedNodeId = proposedIdentifier; logger.debug("No existing node with ID {}; resolved node ID is as-proposed", proposedIdentifier.getFullDescription()); onNodeAdded(resolvedNodeId, true); } else if (existingStatus.getNodeIdentifier().logicallyEquals(proposedIdentifier)) { // there is a node with that ID but it's the same node. resolvedNodeId = proposedIdentifier; logger.debug("A node already exists with ID {} and is logically equivalent; resolved node ID is as-proposed: {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription()); } else { // there is a node with that ID and it's a different node resolvedNodeId = new NodeIdentifier(UUID.randomUUID().toString(), proposedIdentifier.getApiAddress(), proposedIdentifier.getApiPort(), proposedIdentifier.getSocketAddress(), proposedIdentifier.getSocketPort(), proposedIdentifier.getLoadBalanceAddress(), proposedIdentifier.getLoadBalancePort(), proposedIdentifier.getSiteToSiteAddress(), proposedIdentifier.getSiteToSitePort(), proposedIdentifier.getSiteToSiteHttpApiPort(), proposedIdentifier.isSiteToSiteSecure()); logger.debug("A node already exists with ID {}. Proposed Node Identifier was {}; existing Node Identifier is {}; Resolved Node Identifier is {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription(), getNodeIdentifier(proposedIdentifier.getId()).getFullDescription(), resolvedNodeId.getFullDescription()); } return resolvedNodeId; }
@Override public void mergeStatus(ConnectionStatusDTO clientStatus, boolean clientStatusReadablePermission, ConnectionStatusDTO status, boolean statusReadablePermission, NodeIdentifier statusNodeIdentifier) { StatusMerger.merge(clientStatus, clientStatusReadablePermission, status, statusReadablePermission, statusNodeIdentifier.getId(), statusNodeIdentifier.getApiAddress(), statusNodeIdentifier.getApiPort()); } }
/** * Normalizes the validation errors. * * @param errorMap validation errors for each node * @param totalNodes total number of nodes * @return the normalized validation errors */ public static Set<String> normalizedMergedErrors(final Map<String, Set<NodeIdentifier>> errorMap, int totalNodes) { final Set<String> normalizedErrors = new HashSet<>(); for (final Map.Entry<String, Set<NodeIdentifier>> validationEntry : errorMap.entrySet()) { final String msg = validationEntry.getKey(); final Set<NodeIdentifier> nodeIds = validationEntry.getValue(); if (nodeIds.size() == totalNodes) { normalizedErrors.add(msg); } else { nodeIds.forEach(id -> normalizedErrors.add(id.getApiAddress() + ":" + id.getApiPort() + " -- " + msg)); } } return normalizedErrors; }
@Override public String getClusterNodeIdentifier() { final NodeIdentifier nodeId = flowController.getNodeId(); return nodeId == null ? null : nodeId.getId(); } }
@Override public NodeIdentifier unmarshal(final AdaptedNodeIdentifier aNi) { if (aNi == null) { return null; } else { return new NodeIdentifier(aNi.getId(), aNi.getApiAddress(), aNi.getApiPort(), aNi.getSocketAddress(), aNi.getSocketPort(), aNi.getLoadBalanceAddress(), aNi.getLoadBalancePort(), aNi.getSiteToSiteAddress(), aNi.getSiteToSitePort(),aNi.getSiteToSiteHttpApiPort(), aNi.isSiteToSiteSecure()); } }