@Override public void setLocalNodeIdentifier(final NodeIdentifier nodeId) { if (nodeId == null || nodeId.equals(this.nodeId)) { return; } this.nodeId = nodeId; nodeStatuses.computeIfAbsent(nodeId, id -> new NodeConnectionStatus(id, DisconnectionCode.NOT_YET_CONNECTED)); eventListeners.forEach(listener -> listener.onLocalNodeIdentifierSet(nodeId)); }
@Override public void finishNodeOffload(final NodeIdentifier nodeId) { final NodeConnectionState state = getConnectionState(nodeId); if (state == null) { logger.warn("Attempted to finish node offload for {} but node is not known.", nodeId); return; } if (state != NodeConnectionState.OFFLOADING) { logger.warn("Attempted to finish node offload for {} but node is not in the offloading state, it is currently {}.", nodeId, state); return; } logger.info("{} is now offloaded", nodeId); updateNodeStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.OFFLOADED)); }
@Override public void shutdown() { if (closed) { return; } closed = true; final NodeIdentifier localId = getLocalNodeIdentifier(); if (localId != null) { final NodeConnectionStatus shutdownStatus = new NodeConnectionStatus(localId, DisconnectionCode.NODE_SHUTDOWN); updateNodeStatus(shutdownStatus, false); logger.info("Successfully notified other nodes that I am shutting down"); } }
@Override public void disconnectionRequestedByNode(final NodeIdentifier nodeId, final DisconnectionCode disconnectionCode, final String explanation) { logger.info("{} requested disconnection from cluster due to {}", nodeId, explanation == null ? disconnectionCode : explanation); updateNodeStatus(new NodeConnectionStatus(nodeId, disconnectionCode, explanation)); final Severity severity; switch (disconnectionCode) { case STARTUP_FAILURE: case MISMATCHED_FLOWS: case UNKNOWN: severity = Severity.ERROR; break; case LACK_OF_HEARTBEAT: severity = Severity.WARNING; break; default: severity = Severity.INFO; break; } reportEvent(nodeId, severity, "Node disconnected from cluster due to " + explanation); }
private List<NodeConnectionStatus> getUpdatedStatuses(final List<NodeConnectionStatus> nodeStatusList) { // Map node's statuses by NodeIdentifier for quick & easy lookup final Map<NodeIdentifier, NodeConnectionStatus> nodeStatusMap = nodeStatusList.stream() .collect(Collectors.toMap(status -> status.getNodeIdentifier(), Function.identity())); // Check if our connection status is the same for each Node Identifier and if not, add our version of the status // to a List of updated statuses. final List<NodeConnectionStatus> currentStatuses = clusterCoordinator.getConnectionStatuses(); final List<NodeConnectionStatus> updatedStatuses = new ArrayList<>(); for (final NodeConnectionStatus currentStatus : currentStatuses) { final NodeConnectionStatus nodeStatus = nodeStatusMap.get(currentStatus.getNodeIdentifier()); if (!currentStatus.equals(nodeStatus)) { updatedStatuses.add(currentStatus); } } // If the node has any statuses that we do not have, add a REMOVED status to the update list final Set<NodeIdentifier> nodeIds = currentStatuses.stream().map(status -> status.getNodeIdentifier()).collect(Collectors.toSet()); for (final NodeConnectionStatus nodeStatus : nodeStatusList) { if (!nodeIds.contains(nodeStatus.getNodeIdentifier())) { updatedStatuses.add(new NodeConnectionStatus(nodeStatus.getNodeIdentifier(), NodeConnectionState.REMOVED, null)); } } logger.debug("\n\nCalculated diff between current cluster status and node cluster status as follows:\nNode: {}\nSelf: {}\nDifference: {}\n\n", nodeStatusList, currentStatuses, updatedStatuses); return updatedStatuses; }
@Override public void finishNodeConnection(final NodeIdentifier nodeId) { final NodeConnectionState state = getConnectionState(nodeId); if (state == null) { logger.debug("Attempted to finish node connection for {} but node is not known. Requesting that node connect", nodeId); requestNodeConnect(nodeId, null); return; } if (state == NodeConnectionState.CONNECTED) { // already connected. Nothing to do. return; } if (state == NodeConnectionState.DISCONNECTED || state == NodeConnectionState.DISCONNECTING) { logger.debug("Attempted to finish node connection for {} but node state was {}. Requesting that node connect", nodeId, state); requestNodeConnect(nodeId, null); return; } logger.info("{} is now connected", nodeId); updateNodeStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTED)); }
private void disconnect(final String explanation) { writeLock.lock(); try { logger.info("Disconnecting node due to " + explanation); // mark node as not connected controller.setConnectionStatus(new NodeConnectionStatus(nodeId, DisconnectionCode.UNKNOWN, explanation)); // turn off primary flag controller.setPrimary(false); // stop heartbeating controller.stopHeartbeating(); // set node to not clustered controller.setClustered(false, null); clusterCoordinator.setConnected(false); logger.info("Node disconnected due to " + explanation); } finally { writeLock.unlock(); } }
final NodeIdentifier nodeId = nodeIdDesc.toNodeIdentifier(); connectionStatusMap.put(nodeId, new NodeConnectionStatus(nodeId, DisconnectionCode.NOT_YET_CONNECTED)); if (nodeIdDesc.isLocalNodeIdentifier()) { if (localNodeId == null) {
for (final NodeIdentifier nodeId : failedNodeIds) { final NodeConnectionStatus reconnectionStatus = new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTING); updateNodeStatus(reconnectionStatus); requestNodeConnect(nodeId, null);
@Override public void removeNode(final NodeIdentifier nodeId, final String userDn) { reportEvent(nodeId, Severity.INFO, "User " + userDn + " requested that node be removed from cluster"); notifyOthersOfNodeStatusChange(new NodeConnectionStatus(nodeId, NodeConnectionState.REMOVED)); removeNode(nodeId); storeState(); }
@Override public void requestNodeDisconnect(final NodeIdentifier nodeId, final DisconnectionCode disconnectionCode, final String explanation) { final Set<NodeIdentifier> connectedNodeIds = getNodeIdentifiers(NodeConnectionState.CONNECTED); if (connectedNodeIds.size() == 1 && connectedNodeIds.contains(nodeId)) { throw new IllegalNodeDisconnectionException("Cannot disconnect node " + nodeId + " because it is the only node currently connected"); } logger.info("Requesting that {} disconnect due to {}", nodeId, explanation == null ? disconnectionCode : explanation); updateNodeStatus(new NodeConnectionStatus(nodeId, disconnectionCode, explanation)); // There is no need to tell the node that it's disconnected if it is due to being // shutdown, as we will not be able to connect to the node anyway. if (disconnectionCode == DisconnectionCode.NODE_SHUTDOWN) { return; } final DisconnectMessage request = new DisconnectMessage(); request.setNodeId(nodeId); request.setExplanation(explanation); addNodeEvent(nodeId, "Disconnection requested due to " + explanation); disconnectAsynchronously(request, 10, 5); }
@Override public void requestNodeOffload(final NodeIdentifier nodeId, final OffloadCode offloadCode, final String explanation) { final Set<NodeIdentifier> offloadNodeIds = getNodeIdentifiers(NodeConnectionState.OFFLOADING, NodeConnectionState.OFFLOADED); if (offloadNodeIds.contains(nodeId)) { logger.debug("Attempted to offload node but the node is already offloading or offloaded"); // no need to do anything here, the node is currently offloading or already offloaded return; } final Set<NodeIdentifier> disconnectedNodeIds = getNodeIdentifiers(NodeConnectionState.DISCONNECTED); if (!disconnectedNodeIds.contains(nodeId)) { throw new IllegalNodeOffloadException("Cannot offload node " + nodeId + " because it is not currently disconnected"); } logger.info("Requesting that {} is offloaded due to {}", nodeId, explanation == null ? offloadCode : explanation); updateNodeStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.OFFLOADING, offloadCode, explanation)); final OffloadMessage request = new OffloadMessage(); request.setNodeId(nodeId); request.setExplanation(explanation); addNodeEvent(nodeId, "Offload requested due to " + explanation); offloadAsynchronously(request, 10, 5); }
private ConnectionResponseMessage createConnectionResponse(final ConnectionRequest request, final NodeIdentifier resolvedNodeIdentifier, final DataFlow clusterDataFlow) { if (clusterDataFlow == null) { final ConnectionResponseMessage responseMessage = new ConnectionResponseMessage(); responseMessage.setConnectionResponse(new ConnectionResponse(5, "The cluster dataflow is not yet available")); return responseMessage; } // Set node's status to 'CONNECTING' NodeConnectionStatus status = getConnectionStatus(resolvedNodeIdentifier); if (status == null) { addNodeEvent(resolvedNodeIdentifier, "Connection requested from new node. Setting status to connecting."); } else { addNodeEvent(resolvedNodeIdentifier, "Connection requested from existing node. Setting status to connecting."); } status = new NodeConnectionStatus(resolvedNodeIdentifier, NodeConnectionState.CONNECTING, null, null, null, System.currentTimeMillis()); updateNodeStatus(status); final ConnectionResponse response = new ConnectionResponse(resolvedNodeIdentifier, clusterDataFlow, instanceId, getConnectionStatuses(), revisionManager.getAllRevisions().stream().map(rev -> ComponentRevision.fromRevision(rev)).collect(Collectors.toList())); final ConnectionResponseMessage responseMessage = new ConnectionResponseMessage(); responseMessage.setConnectionResponse(response); return responseMessage; }
@Override public void requestNodeConnect(final NodeIdentifier nodeId, final String userDn) { if (requireElection && !flowElection.isElectionComplete() && flowElection.isVoteCounted(nodeId)) { // If we receive a heartbeat from a node that we already know, we don't want to request that it reconnect // to the cluster because no flow has yet been elected. However, if the node has not yet voted, we want to send // a reconnect request because we want this node to cast its vote for the flow, and this happens on connection logger.debug("Received heartbeat for {} and node is not connected. Will not request node connect to cluster, " + "though, because the Flow Election is still in progress", nodeId); return; } if (userDn == null) { reportEvent(nodeId, Severity.INFO, "Requesting that node connect to cluster"); } else { reportEvent(nodeId, Severity.INFO, "Requesting that node connect to cluster on behalf of " + userDn); } updateNodeStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTING, null, null, null, System.currentTimeMillis())); // create the request final ReconnectionRequestMessage request = new ReconnectionRequestMessage(); request.setNodeId(nodeId); request.setInstanceId(instanceId); // If we still are requiring that an election take place, we do not want to include our local dataflow, because we don't // yet know what the cluster's dataflow looks like. However, if we don't require election, then we've connected to the // cluster, which means that our flow is correct. final boolean includeDataFlow = !requireElection; requestReconnectionAsynchronously(request, 10, 5, includeDataFlow); }
controller.setConnectionStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.CONNECTED));
clusterCoordinator.setConnected(false); controller.setConnectionStatus(new NodeConnectionStatus(nodeId, DisconnectionCode.NOT_YET_CONNECTED));
@Override public NodeConnectionStatus unmarshal(final AdaptedNodeConnectionStatus adapted) throws Exception { return new NodeConnectionStatus(adapted.getUpdateId(), adapted.getNodeId(), adapted.getState(), adapted.getOffloadCode(), adapted.getDisconnectCode(), adapted.getReason(), adapted.getConnectionRequestTime()); }
controller.setConnectionStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.OFFLOADING, OffloadCode.OFFLOADED, explanation)); controller.setConnectionStatus(new NodeConnectionStatus(nodeId, NodeConnectionState.OFFLOADED, OffloadCode.OFFLOADED, explanation)); clusterCoordinator.finishNodeOffload(getNodeId());
private NodeIdentifier resolveNodeId(final NodeIdentifier proposedIdentifier) { final NodeConnectionStatus proposedConnectionStatus = new NodeConnectionStatus(proposedIdentifier, DisconnectionCode.NOT_YET_CONNECTED); final NodeConnectionStatus existingStatus = nodeStatuses.putIfAbsent(proposedIdentifier, proposedConnectionStatus); NodeIdentifier resolvedNodeId = proposedIdentifier; if (existingStatus == null) { // there is no node with that ID resolvedNodeId = proposedIdentifier; logger.debug("No existing node with ID {}; resolved node ID is as-proposed", proposedIdentifier.getFullDescription()); onNodeAdded(resolvedNodeId, true); } else if (existingStatus.getNodeIdentifier().logicallyEquals(proposedIdentifier)) { // there is a node with that ID but it's the same node. resolvedNodeId = proposedIdentifier; logger.debug("A node already exists with ID {} and is logically equivalent; resolved node ID is as-proposed: {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription()); } else { // there is a node with that ID and it's a different node resolvedNodeId = new NodeIdentifier(UUID.randomUUID().toString(), proposedIdentifier.getApiAddress(), proposedIdentifier.getApiPort(), proposedIdentifier.getSocketAddress(), proposedIdentifier.getSocketPort(), proposedIdentifier.getLoadBalanceAddress(), proposedIdentifier.getLoadBalancePort(), proposedIdentifier.getSiteToSiteAddress(), proposedIdentifier.getSiteToSitePort(), proposedIdentifier.getSiteToSiteHttpApiPort(), proposedIdentifier.isSiteToSiteSecure()); logger.debug("A node already exists with ID {}. Proposed Node Identifier was {}; existing Node Identifier is {}; Resolved Node Identifier is {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription(), getNodeIdentifier(proposedIdentifier.getId()).getFullDescription(), resolvedNodeId.getFullDescription()); } return resolvedNodeId; }
this.connectionStatus = new NodeConnectionStatus(nodeId, DisconnectionCode.NOT_YET_CONNECTED); heartbeatBeanRef.set(new HeartbeatBean(rootGroup, false));