@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
private synchronized void updateAllNodes(AllNodes allNodes) { if (includeCoordinator) { currentCount = allNodes.getActiveNodes().size(); } else { currentCount = Sets.difference(allNodes.getActiveNodes(), allNodes.getActiveCoordinators()).size(); } if (currentCount >= executionMinCount) { ImmutableList<SettableFuture<?>> listeners = ImmutableList.copyOf(futures); futures.clear(); executor.submit(() -> listeners.forEach(listener -> listener.set(null))); } } }
@Override public AllNodes getAllNodes() { return new AllNodes(ImmutableSet.<Node>builder().add(localNode).addAll(remoteNodes.values()).build(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(localNode)); }
private boolean allNodesGloballyVisible() { for (TestingPrestoServer server : servers) { AllNodes allNodes = server.refreshNodes(); if (!allNodes.getInactiveNodes().isEmpty() || (allNodes.getActiveNodes().size() != servers.size())) { return false; } } return true; }
public void refreshNodes() { AllNodes allNodes; do { try { MILLISECONDS.sleep(10); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } allNodes = server.refreshNodes(); } while (allNodes.getActiveNodes().isEmpty()); }
@Test(timeOut = 60000) public void testNodeChangeListener() throws Exception { DiscoveryNodeManager manager = new DiscoveryNodeManager(selector, nodeInfo, new NoOpFailureDetector(), expectedVersion, testHttpClient, internalCommunicationConfig); try { manager.startPollingNodeStates(); BlockingQueue<AllNodes> notifications = new ArrayBlockingQueue<>(100); manager.addNodeChangeListener(notifications::add); AllNodes allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); selector.announceNodes(ImmutableSet.of(currentNode), ImmutableSet.of(coordinator)); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), ImmutableSet.of(currentNode, coordinator)); assertEquals(allNodes.getActiveCoordinators(), ImmutableSet.of(coordinator)); selector.announceNodes(activeNodes, inactiveNodes); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); } finally { manager.stop(); } }
SetView<Node> missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); for (Node missingNode : missingNodes) { log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHostAndPort()); AllNodes allNodes = new AllNodes(activeNodesBuilder.build(), inactiveNodesBuilder.build(), shuttingDownNodesBuilder.build(), coordinatorsBuilder.build()); if (!allNodes.equals(this.allNodes)) {
private void pollWorkers() { AllNodes allNodes = getAllNodes(); Set<Node> aliveNodes = ImmutableSet.<Node>builder() .addAll(allNodes.getActiveNodes()) .addAll(allNodes.getShuttingDownNodes()) .build(); ImmutableSet<String> aliveNodeIds = aliveNodes.stream() .map(Node::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore // Make a copy to materialize the set difference Set<String> deadNodes = difference(nodeStates.keySet(), aliveNodeIds).immutableCopy(); nodeStates.keySet().removeAll(deadNodes); // Add new nodes for (Node node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } // Schedule refresh nodeStates.values().forEach(RemoteNodeState::asyncRefresh); // update indexes refreshNodesInternal(); }
@Managed public int getInactiveNodeCount() { return getAllNodes().getInactiveNodes().size(); }
@Managed public int getShuttingDownNodeCount() { return getAllNodes().getShuttingDownNodes().size(); }
@Managed public int getActiveNodeCount() { return getAllNodes().getActiveNodes().size(); }
@Test public void testGetAllNodes() { DiscoveryNodeManager manager = new DiscoveryNodeManager(selector, nodeInfo, new NoOpFailureDetector(), expectedVersion, testHttpClient, internalCommunicationConfig); try { AllNodes allNodes = manager.getAllNodes(); Set<Node> activeNodes = allNodes.getActiveNodes(); assertEqualsIgnoreOrder(activeNodes, this.activeNodes); for (Node actual : activeNodes) { for (Node expected : this.activeNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(activeNodes, manager.getNodes(ACTIVE)); Set<Node> inactiveNodes = allNodes.getInactiveNodes(); assertEqualsIgnoreOrder(inactiveNodes, this.inactiveNodes); for (Node actual : inactiveNodes) { for (Node expected : this.inactiveNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(inactiveNodes, manager.getNodes(INACTIVE)); } finally { manager.stop(); } }
@Test(timeOut = 60000) public void testNodeChangeListener() throws Exception { DiscoveryNodeManager manager = new DiscoveryNodeManager(selector, nodeInfo, new NoOpFailureDetector(), expectedVersion, testHttpClient, internalCommunicationConfig); try { manager.startPollingNodeStates(); BlockingQueue<AllNodes> notifications = new ArrayBlockingQueue<>(100); manager.addNodeChangeListener(notifications::add); AllNodes allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); selector.announceNodes(ImmutableSet.of(currentNode), ImmutableSet.of(coordinator)); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), ImmutableSet.of(currentNode, coordinator)); assertEquals(allNodes.getActiveCoordinators(), ImmutableSet.of(coordinator)); selector.announceNodes(activeNodes, inactiveNodes); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); } finally { manager.stop(); } }
SetView<Node> missingNodes = difference(allNodes.getActiveNodes(), Sets.union(activeNodesBuilder.build(), shuttingDownNodesBuilder.build())); for (Node missingNode : missingNodes) { log.info("Previously active node is missing: %s (last seen at %s)", missingNode.getNodeIdentifier(), missingNode.getHostAndPort()); AllNodes allNodes = new AllNodes(activeNodesBuilder.build(), inactiveNodesBuilder.build(), shuttingDownNodesBuilder.build(), coordinatorsBuilder.build()); if (!allNodes.equals(this.allNodes)) {
private void pollWorkers() { AllNodes allNodes = getAllNodes(); Set<Node> aliveNodes = ImmutableSet.<Node>builder() .addAll(allNodes.getActiveNodes()) .addAll(allNodes.getShuttingDownNodes()) .build(); ImmutableSet<String> aliveNodeIds = aliveNodes.stream() .map(Node::getNodeIdentifier) .collect(toImmutableSet()); // Remove nodes that don't exist anymore // Make a copy to materialize the set difference Set<String> deadNodes = difference(nodeStates.keySet(), aliveNodeIds).immutableCopy(); nodeStates.keySet().removeAll(deadNodes); // Add new nodes for (Node node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } // Schedule refresh nodeStates.values().forEach(RemoteNodeState::asyncRefresh); // update indexes refreshNodesInternal(); }
@Managed public int getInactiveNodeCount() { return getAllNodes().getInactiveNodes().size(); }
@Managed public int getShuttingDownNodeCount() { return getAllNodes().getShuttingDownNodes().size(); }
@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
@Inject public TaskCountEstimator(NodeSchedulerConfig nodeSchedulerConfig, InternalNodeManager nodeManager) { requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(nodeManager, "nodeManager is null"); this.numberOfNodes = () -> { Set<Node> activeNodes = nodeManager.getAllNodes().getActiveNodes(); if (nodeSchedulerConfig.isIncludeCoordinator()) { return activeNodes.size(); } return toIntExact(activeNodes.stream() .filter(node -> !node.isCoordinator()) .count()); }; }
@Test public void testGetAllNodes() { DiscoveryNodeManager manager = new DiscoveryNodeManager(selector, nodeInfo, new NoOpFailureDetector(), expectedVersion, testHttpClient, internalCommunicationConfig); try { AllNodes allNodes = manager.getAllNodes(); Set<Node> activeNodes = allNodes.getActiveNodes(); assertEqualsIgnoreOrder(activeNodes, this.activeNodes); for (Node actual : activeNodes) { for (Node expected : this.activeNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(activeNodes, manager.getNodes(ACTIVE)); Set<Node> inactiveNodes = allNodes.getInactiveNodes(); assertEqualsIgnoreOrder(inactiveNodes, this.inactiveNodes); for (Node actual : inactiveNodes) { for (Node expected : this.inactiveNodes) { assertNotSame(actual, expected); } } assertEqualsIgnoreOrder(inactiveNodes, manager.getNodes(INACTIVE)); } finally { manager.stop(); } }