totalFlowFileCount += nodeInfo.getFlowFileCount(); final int flowFileCount = nodeInfo.getFlowFileCount(); PeerStatus status = destinations.get(index); if (status == null) { status = new PeerStatus(nodeInfo.getPeerDescription(), nodeInfo.getFlowFileCount(), nodeInfo.isQueryForPeers()); destinations.set(index, status); break;
private void persistPeerStatuses(final Set<PeerStatus> statuses) { if (persistenceFile == null) { return; } try (final OutputStream fos = new FileOutputStream(persistenceFile); final OutputStream out = new BufferedOutputStream(fos)) { for (final PeerStatus status : statuses) { final PeerDescription description = status.getPeerDescription(); final String line = description.getHostname() + ":" + description.getPort() + ":" + description.isSecure() + ":" + status.isQueryForPeers() + "\n"; out.write(line.getBytes(StandardCharsets.UTF_8)); } } catch (final IOException e) { error(logger, eventReporter, "Failed to persist list of Peers due to {}; if restarted and peer's NCM is down," + " may be unable to transfer data until communications with NCM are restored", e.toString()); logger.error("", e); } }
private Set<PeerStatus> getPeerStatuses() { final PeerStatusCache cache = this.peerStatusCache; if (cache == null || cache.getStatuses() == null || cache.getStatuses().isEmpty()) { return null; } if (cache.getTimestamp() + PEER_CACHE_MILLIS < systemTime.currentTimeMillis()) { final Set<PeerStatus> equalizedSet = new HashSet<>(cache.getStatuses().size()); for (final PeerStatus status : cache.getStatuses()) { final PeerStatus equalizedStatus = new PeerStatus(status.getPeerDescription(), 1, status.isQueryForPeers()); equalizedSet.add(equalizedStatus); } return equalizedSet; } return cache.getStatuses(); }
public boolean isPenalized(final PeerStatus peerStatus) { final Long expirationEnd = peerTimeoutExpirations.get(peerStatus.getPeerDescription()); return (expirationEnd != null && expirationEnd > systemTime.currentTimeMillis()); }
private static Set<PeerStatus> recoverPersistedPeerStatuses(final File file) throws IOException { if (!file.exists()) { return null; } final Set<PeerStatus> statuses = new HashSet<>(); try (final InputStream fis = new FileInputStream(file); final BufferedReader reader = new BufferedReader(new InputStreamReader(fis))) { String line; while ((line = reader.readLine()) != null) { final String[] splits = line.split(Pattern.quote(":")); if (splits.length != 3 && splits.length != 4) { continue; } final String hostname = splits[0]; final int port = Integer.parseInt(splits[1]); final boolean secure = Boolean.parseBoolean(splits[2]); final boolean supportQueryForPeer = splits.length == 4 && Boolean.parseBoolean(splits[3]); statuses.add(new PeerStatus(new PeerDescription(hostname, port, secure), 1, supportQueryForPeer)); } } return statuses; }
lastFetched.stream().map(peer -> peer.getPeerDescription()) .forEach(desc -> peersToRequestClusterInfoFrom.add(desc)); final Set<PeerStatus> statuses = peerStatusProvider.fetchRemotePeerStatuses(peerDescription); lastFetchedQueryablePeers = statuses.stream() .filter(p -> p.isQueryForPeers()) .collect(Collectors.toSet());
@Override public boolean equals(final Object obj) { if (obj == null) { return false; } if (!(obj instanceof PeerStatus)) { return false; } final PeerStatus other = (PeerStatus) obj; return description.equals(other.getPeerDescription()); } }
@Override public Set<PeerStatus> getPeerStatuses(final Peer peer) throws IOException { if (!handshakeComplete) { throw new IllegalStateException("Handshake has not been performed"); } logger.debug("{} Get Peer Statuses from {}", this, peer); final CommunicationsSession commsSession = peer.getCommunicationsSession(); final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream()); final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream()); final boolean queryPeersForOtherPeers = getVersionNegotiator().getVersion() >= 6; RequestType.REQUEST_PEER_LIST.writeRequestType(dos); dos.flush(); final int numPeers = dis.readInt(); final Set<PeerStatus> peers = new HashSet<>(numPeers); for (int i = 0; i < numPeers; i++) { final String hostname = dis.readUTF(); final int port = dis.readInt(); final boolean secure = dis.readBoolean(); final int flowFileCount = dis.readInt(); peers.add(new PeerStatus(new PeerDescription(hostname, port, secure), flowFileCount, queryPeersForOtherPeers)); } logger.debug("{} Received {} Peer Statuses from {}", this, peers.size(), peer); return peers; }
private CommunicationsSession establishSiteToSiteConnection(final PeerStatus peerStatus) throws IOException { final PeerDescription description = peerStatus.getPeerDescription(); return establishSiteToSiteConnection(description.getHostname(), description.getPort()); }
@Override public Set<PeerStatus> fetchRemotePeerStatuses(PeerDescription peerDescription) throws IOException { // Each node should has the same URL structure and network reach-ability with the proxy configuration. try (final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter())) { final String scheme = peerDescription.isSecure() ? "https" : "http"; apiClient.setBaseUrl(scheme, peerDescription.getHostname(), peerDescription.getPort()); final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS); apiClient.setConnectTimeoutMillis(timeoutMillis); apiClient.setReadTimeoutMillis(timeoutMillis); apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS)); apiClient.setLocalAddress(config.getLocalAddress()); final Collection<PeerDTO> peers = apiClient.getPeers(); if(peers == null || peers.size() == 0){ throw new IOException("Couldn't get any peer to communicate with. " + apiClient.getBaseUrl() + " returned zero peers."); } // Convert the PeerDTO's to PeerStatus objects. Use 'true' for the query-peer-for-peers flag because Site-to-Site over HTTP // was added in NiFi 1.0.0, which means that peer-to-peer queries are always allowed. return peers.stream().map(p -> new PeerStatus(new PeerDescription(p.getHostname(), p.getPort(), p.isSecure()), p.getFlowFileCount(), true)) .collect(Collectors.toSet()); } }
final PeerDescription peerDescription = peerStatus.getPeerDescription(); BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerDescription); if (connectionQueue == null) { commsSession = establishSiteToSiteConnection(peerStatus); } catch (final IOException ioe) { peerSelector.penalize(peerStatus.getPeerDescription(), penalizationMillis); throw ioe;
final String nodeApiUrl = resolveNodeApiUrl(peerStatus.getPeerDescription()); final StringBuilder clusterUrls = new StringBuilder(); config.getUrls().forEach(url -> { final Peer peer = new Peer(peerStatus.getPeerDescription(), commSession, nodeApiUrl, clusterUrls.toString());