@BeforeMethod public void setup() { testHttpClient = new TestingHttpClient(input -> new TestingResponse(OK, ArrayListMultimap.create(), ACTIVE.name().getBytes())); expectedVersion = new NodeVersion("1"); coordinator = new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion, false); activeNodes = ImmutableList.of( new PrestoNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion, false), new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion, false), new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion, false), coordinator); inactiveNodes = ImmutableList.of( new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN, false), new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2"), false)); List<ServiceDescriptor> descriptors = new ArrayList<>(); for (PrestoNode node : Iterables.concat(activeNodes, inactiveNodes)) { descriptors.add(serviceDescriptor("presto") .setNodeId(node.getNodeIdentifier()) .addProperty("http", node.getHttpUri().toString()) .addProperty("node_version", node.getNodeVersion().toString()) .addProperty("coordinator", String.valueOf(node.equals(coordinator))) .build()); } selector = new StaticServiceSelector(descriptors); }
@PostConstruct public void startPollingNodeStates() { // poll worker states only on the coordinators if (currentNode.isCoordinator()) { nodeStateUpdateExecutor.scheduleWithFixedDelay(() -> { try { pollWorkers(); } catch (Exception e) { log.error(e, "Error polling state of nodes"); } }, 5, 5, TimeUnit.SECONDS); } pollWorkers(); }
private NodeState getNodeState(PrestoNode node) { if (expectedNodeVersion.equals(node.getNodeVersion())) { if (isNodeShuttingDown(node.getNodeIdentifier())) { return SHUTTING_DOWN; } else { return ACTIVE; } } else { return INACTIVE; } }
private static PrestoNode findCurrentNode(List<ServiceDescriptor> allServices, String currentNodeId, NodeVersion expectedNodeVersion, boolean httpsRequired) { for (ServiceDescriptor service : allServices) { URI uri = getHttpUri(service, httpsRequired); NodeVersion nodeVersion = getNodeVersion(service); if (uri != null && nodeVersion != null) { PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, isCoordinator(service)); if (node.getNodeIdentifier().equals(currentNodeId)) { checkState( node.getNodeVersion().equals(expectedNodeVersion), "INVARIANT: current node version (%s) should be equal to %s", node.getNodeVersion(), expectedNodeVersion); return node; } } } throw new IllegalStateException("INVARIANT: current node not returned from service selector"); }
public InMemoryNodeManager(URI localUri) { localNode = new PrestoNode("local", localUri, NodeVersion.UNKNOWN, false); }
private static String getNodeVersion(Node node) { if (node instanceof PrestoNode) { return ((PrestoNode) node).getNodeVersion().toString(); } return ""; }
public TestingNodeManager(String environment) { this(environment, new PrestoNode("local", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true), ImmutableSet.of()); }
NodeVersion nodeVersion = getNodeVersion(service); if (uri != null && nodeVersion != null) { PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion); NodeState nodeState = getNodeState(node); if (node.getNodeIdentifier().equals(nodeInfo.getNodeId())) { currentNode = node; checkState(currentNode.getNodeVersion().equals(expectedNodeVersion), "INVARIANT: current node version should be equal to expected node version");
private NodeState getNodeState(PrestoNode node) { if (expectedNodeVersion.equals(node.getNodeVersion())) { if (isNodeShuttingDown(node.getNodeIdentifier())) { return SHUTTING_DOWN; } else { return ACTIVE; } } else { return INACTIVE; } }
private static String getNodeVersion(Node node) { if (node instanceof PrestoNode) { return ((PrestoNode) node).getNodeVersion().toString(); } return ""; }
@BeforeMethod public void setup() { testHttpClient = new TestingHttpClient(input -> new TestingResponse(OK, ArrayListMultimap.create(), ACTIVE.name().getBytes())); expectedVersion = new NodeVersion("1"); coordinator = new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.2.8"), expectedVersion); activeNodes = ImmutableList.of( new PrestoNode(nodeInfo.getNodeId(), URI.create("http://192.0.1.1"), expectedVersion), new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.1:8080"), expectedVersion), new PrestoNode(UUID.randomUUID().toString(), URI.create("http://192.0.2.3"), expectedVersion), coordinator); inactiveNodes = ImmutableList.of( new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.3.9"), NodeVersion.UNKNOWN), new PrestoNode(UUID.randomUUID().toString(), URI.create("https://192.0.4.9"), new NodeVersion("2")) ); List<ServiceDescriptor> descriptors = new ArrayList<>(); for (PrestoNode node : Iterables.concat(activeNodes, inactiveNodes)) { descriptors.add(serviceDescriptor("presto") .setNodeId(node.getNodeIdentifier()) .addProperty("http", node.getHttpUri().toString()) .addProperty("node_version", node.getNodeVersion().toString()) .addProperty("coordinator", String.valueOf(node.equals(coordinator))) .build()); } selector = new StaticServiceSelector(descriptors); }
private static Node createTestingNode(String hostname) { return new PrestoNode(hostname, URI.create(format("http://%s:8080", hostname)), NodeVersion.UNKNOWN, false); } }
public TestSourcePartitionedScheduler() { nodeManager.addNode(CONNECTOR_ID, new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); }
private static List<Node> generateRandomNodes(int count) { return IntStream.range(0, count) .mapToObj(i -> new PrestoNode("other " + i, URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)) .collect(toImmutableList()); } }
boolean coordinator = isCoordinator(service); if (uri != null && nodeVersion != null) { PrestoNode node = new PrestoNode(service.getNodeId(), uri, nodeVersion, coordinator); NodeState nodeState = getNodeState(node);
private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) { return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {}), true); }
@BeforeMethod public void setUp() { finalizerService = new FinalizerService(); nodeTaskMap = new NodeTaskMap(finalizerService); nodeManager = new InMemoryNodeManager(); ImmutableList.Builder<Node> nodeBuilder = ImmutableList.builder(); nodeBuilder.add(new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); ImmutableList<Node> nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) .setIncludeCoordinator(false) .setMaxPendingSplitsPerTask(10); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); // contents of taskMap indicate the node-task map for the current stage taskMap = new HashMap<>(); nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); remoteTaskExecutor = newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")); remoteTaskScheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s")); finalizerService.start(); }
Node newNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, newNode);
new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); Node additionalNode = new PrestoNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false); nodeManager.addNode(CONNECTOR_ID, additionalNode);
PrestoNode node = new PrestoNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false); long bytes = nodeEntry.getValue(); if (bytes == 0) {