public void refreshNodes() { AllNodes allNodes; do { try { MILLISECONDS.sleep(10); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } allNodes = server.refreshNodes(); } while (allNodes.getActiveNodes().isEmpty()); }
private boolean allNodesGloballyVisible() { for (TestingPrestoServer server : servers) { AllNodes allNodes = server.refreshNodes(); if (!allNodes.getInactiveNodes().isEmpty() || (allNodes.getActiveNodes().size() != servers.size())) { return false; } } return true; }
private synchronized void updateAllNodes(AllNodes allNodes) { if (includeCoordinator) { currentCount = allNodes.getActiveNodes().size(); } else { currentCount = Sets.difference(allNodes.getActiveNodes(), allNodes.getActiveCoordinators()).size(); } if (currentCount >= executionMinCount) { ImmutableList<SettableFuture<?>> listeners = ImmutableList.copyOf(futures); futures.clear(); executor.submit(() -> listeners.forEach(listener -> listener.set(null))); } } }
private synchronized void updateAllNodes(AllNodes allNodes) { if (includeCoordinator) { currentCount = allNodes.getActiveNodes().size(); } else { currentCount = Sets.difference(allNodes.getActiveNodes(), allNodes.getActiveCoordinators()).size(); } if (currentCount >= executionMinCount) { ImmutableList<SettableFuture<?>> listeners = ImmutableList.copyOf(futures); futures.clear(); executor.submit(() -> listeners.forEach(listener -> listener.set(null))); } } }
@Managed public int getActiveNodeCount() { return getAllNodes().getActiveNodes().size(); }
@Managed public int getActiveNodeCount() { return getAllNodes().getActiveNodes().size(); }
@Inject public TaskCountEstimator(NodeSchedulerConfig nodeSchedulerConfig, InternalNodeManager nodeManager) { requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(nodeManager, "nodeManager is null"); this.numberOfNodes = () -> { Set<Node> activeNodes = nodeManager.getAllNodes().getActiveNodes(); if (nodeSchedulerConfig.isIncludeCoordinator()) { return activeNodes.size(); } return toIntExact(activeNodes.stream() .filter(node -> !node.isCoordinator()) .count()); }; }
@Inject public TaskCountEstimator(NodeSchedulerConfig nodeSchedulerConfig, InternalNodeManager nodeManager) { requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(nodeManager, "nodeManager is null"); this.numberOfNodes = () -> { Set<Node> activeNodes = nodeManager.getAllNodes().getActiveNodes(); if (nodeSchedulerConfig.isIncludeCoordinator()) { return activeNodes.size(); } return toIntExact(activeNodes.stream() .filter(node -> !node.isCoordinator()) .count()); }; }
@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
@Override public Set<Node> getNodes(NodeState state) { switch (state) { case ACTIVE: return getAllNodes().getActiveNodes(); case INACTIVE: return getAllNodes().getInactiveNodes(); case SHUTTING_DOWN: return getAllNodes().getShuttingDownNodes(); default: throw new IllegalArgumentException("Unknown node state " + state); } }
static void waitForNodeRefresh(TestingPrestoServer server) throws InterruptedException { long start = System.nanoTime(); while (server.refreshNodes().getActiveNodes().size() < 1) { assertLessThan(nanosSince(start), new Duration(10, SECONDS)); MILLISECONDS.sleep(10); } }
static void waitForNodeRefresh(TestingPrestoServer server) throws InterruptedException { long start = System.nanoTime(); while (server.refreshNodes().getActiveNodes().size() < 1) { assertLessThan(nanosSince(start), new Duration(10, SECONDS)); MILLISECONDS.sleep(10); } }
@Test(timeOut = 60000) public void testNodeChangeListener() throws Exception { DiscoveryNodeManager manager = new DiscoveryNodeManager(selector, nodeInfo, new NoOpFailureDetector(), expectedVersion, testHttpClient, internalCommunicationConfig); try { manager.startPollingNodeStates(); BlockingQueue<AllNodes> notifications = new ArrayBlockingQueue<>(100); manager.addNodeChangeListener(notifications::add); AllNodes allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); selector.announceNodes(ImmutableSet.of(currentNode), ImmutableSet.of(coordinator)); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), ImmutableSet.of(currentNode, coordinator)); assertEquals(allNodes.getActiveCoordinators(), ImmutableSet.of(coordinator)); selector.announceNodes(activeNodes, inactiveNodes); allNodes = notifications.take(); assertEquals(allNodes.getActiveNodes(), activeNodes); assertEquals(allNodes.getInactiveNodes(), inactiveNodes); } finally { manager.stop(); } }
@Test public void testInitializationTimeout() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setCoordinatorProperties(ImmutableMap.<String, String>builder() .put("query-manager.initialization-required-workers", "5") .put("query-manager.initialization-timeout", "1ns") .build()) .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); } }
private void waitForNodes(int numberOfNodes) throws InterruptedException { DistributedQueryRunner queryRunner = (DistributedQueryRunner) getQueryRunner(); long start = System.nanoTime(); while (queryRunner.getCoordinator().refreshNodes().getActiveNodes().size() < numberOfNodes) { assertLessThan(nanosSince(start), new Duration(10, SECONDS)); MILLISECONDS.sleep(10); } } }
@Test public void testSufficientInitialWorkerNodes() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() .setSingleCoordinatorProperty("query-manager.initialization-required-workers", "4") .setNodeCount(4) .build()) { queryRunner.execute("SELECT 1"); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 4); // Query should still be allowed to run if active workers drop down below the minimum required nodes queryRunner.getServers().get(0).close(); assertEquals(queryRunner.getCoordinator().refreshNodes().getActiveNodes().size(), 3); queryRunner.execute("SELECT 1"); } }
@Override public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) { Builder table = InMemoryRecordSet.builder(NODES_TABLE); AllNodes allNodes = nodeManager.getAllNodes(); addRows(table, allNodes.getActiveNodes(), ACTIVE); addRows(table, allNodes.getInactiveNodes(), INACTIVE); addRows(table, allNodes.getShuttingDownNodes(), SHUTTING_DOWN); return table.build().cursor(); }
@Override public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) { Builder table = InMemoryRecordSet.builder(NODES_TABLE); AllNodes allNodes = nodeManager.getAllNodes(); addRows(table, allNodes.getActiveNodes(), ACTIVE); addRows(table, allNodes.getInactiveNodes(), INACTIVE); addRows(table, allNodes.getShuttingDownNodes(), SHUTTING_DOWN); return table.build().cursor(); }