@Test public void testScheduleSlowSplitSource() { QueuedSplitSource queuedSplitSource = new QueuedSplitSource(TestingSplit::createRemoteSplit); StageExecutionPlan plan = createPlan(queuedSplitSource); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); // schedule with no splits - will block ScheduleResult scheduleResult = scheduler.schedule(); assertFalse(scheduleResult.isFinished()); assertFalse(scheduleResult.getBlocked().isDone()); assertEquals(scheduleResult.getNewTasks().size(), 0); assertEquals(stage.getAllTasks().size(), 0); queuedSplitSource.addSplits(1); assertTrue(scheduleResult.getBlocked().isDone()); }
@Test public void testScheduleNoSplits() { StageExecutionPlan plan = createPlan(createFixedSplitSource(0, TestingSplit::createRemoteSplit)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); ScheduleResult scheduleResult = scheduler.schedule(); assertEquals(scheduleResult.getNewTasks().size(), 1); assertEffectivelyFinished(scheduleResult, scheduler); stage.abort(); }
@BeforeMethod public void setUp() { finalizerService = new FinalizerService(); nodeTaskMap = new NodeTaskMap(finalizerService); nodeManager = new InMemoryNodeManager(); ImmutableList.Builder<Node> nodeBuilder = ImmutableList.builder(); nodeBuilder.add(new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false)); nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); ImmutableList<Node> nodes = nodeBuilder.build(); nodeManager.addNode(CONNECTOR_ID, nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) .setIncludeCoordinator(false) .setMaxPendingSplitsPerTask(10); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); // contents of taskMap indicate the node-task map for the current stage taskMap = new HashMap<>(); nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); remoteTaskExecutor = newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")); remoteTaskScheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s")); finalizerService.start(); }
@Test public void testScheduleSplitsBatched() { StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 7); for (int i = 0; i <= (60 / 7); i++) { ScheduleResult scheduleResult = scheduler.schedule(); // finishes when last split is fetched if (i == (60 / 7)) { assertEffectivelyFinished(scheduleResult, scheduler); } else { assertFalse(scheduleResult.isFinished()); } // never blocks assertTrue(scheduleResult.getBlocked().isDone()); // first three splits create new tasks assertEquals(scheduleResult.getNewTasks().size(), i == 0 ? 3 : 0); assertEquals(stage.getAllTasks().size(), 3); assertPartitionedSplitCount(stage, min((i + 1) * 7, 60)); } for (RemoteTask remoteTask : stage.getAllTasks()) { assertEquals(remoteTask.getPartitionedSplitCount(), 20); } stage.abort(); }
@Test public void testScheduleSplitsOneAtATime() { StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); for (int i = 0; i < 60; i++) { ScheduleResult scheduleResult = scheduler.schedule(); // only finishes when last split is fetched if (i == 59) { assertEffectivelyFinished(scheduleResult, scheduler); } else { assertFalse(scheduleResult.isFinished()); } // never blocks assertTrue(scheduleResult.getBlocked().isDone()); // first three splits create new tasks assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0); assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3); assertPartitionedSplitCount(stage, min(i + 1, 60)); } for (RemoteTask remoteTask : stage.getAllTasks()) { assertEquals(remoteTask.getPartitionedSplitCount(), 20); } stage.abort(); }
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
@Test public void testNoNodes() { try { NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap); StageExecutionPlan plan = createPlan(createFixedSplitSource(20, TestingSplit::createRemoteSplit)); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); StageScheduler scheduler = newSourcePartitionedSchedulerAsStageScheduler( stage, Iterables.getOnlyElement(plan.getSplitSources().keySet()), Iterables.getOnlyElement(plan.getSplitSources().values()), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(CONNECTOR_ID), stage::getAllTasks), 2); scheduler.schedule(); fail("expected PrestoException"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), NO_NODES_AVAILABLE.toErrorCode()); } }
@Test public void testBlockCausesFullSchedule() NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager();
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
nodeManager, nodeSchedulerConfig, new NodeTaskMap(finalizerService)); this.pageSinkManager = new PageSinkManager(); CatalogManager catalogManager = new CatalogManager();
@BeforeClass public void setUp() { TaskCountEstimator taskCountEstimator = new TaskCountEstimator(() -> NUMBER_OF_NODES); costCalculatorUsingExchanges = new CostCalculatorUsingExchanges(taskCountEstimator); costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, taskCountEstimator); session = testSessionBuilder().setCatalog("tpch").build(); CatalogManager catalogManager = new CatalogManager(); catalogManager.registerCatalog(createBogusTestingCatalog("tpch")); transactionManager = createTestTransactionManager(catalogManager); metadata = createTestMetadataManager(transactionManager, new FeaturesConfig()); finalizerService = new FinalizerService(); finalizerService.start(); nodeScheduler = new NodeScheduler( new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); nodePartitioningManager = new NodePartitioningManager(nodeScheduler); planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig()); }
new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
@Test public void testScheduleNoSplits() throws Exception { StageExecutionPlan plan = createPlan(createFixedSplitSource(0, TestingSplit::createRemoteSplit)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); SourcePartitionedScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); ScheduleResult scheduleResult = scheduler.schedule(); assertTrue(scheduleResult.isFinished()); assertTrue(scheduleResult.getBlocked().isDone()); assertTrue(scheduleResult.getNewTasks().isEmpty()); stage.abort(); }
@Test public void testScheduleSlowSplitSource() throws Exception { QueuedSplitSource queuedSplitSource = new QueuedSplitSource(TestingSplit::createRemoteSplit); StageExecutionPlan plan = createPlan(queuedSplitSource); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); SourcePartitionedScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); // schedule with no splits - will block ScheduleResult scheduleResult = scheduler.schedule(); assertFalse(scheduleResult.isFinished()); assertFalse(scheduleResult.getBlocked().isDone()); assertEquals(scheduleResult.getNewTasks().size(), 0); assertEquals(stage.getAllTasks().size(), 0); queuedSplitSource.addSplits(1); assertTrue(scheduleResult.getBlocked().isDone()); }
@Test public void testScheduleSplitsOneAtATime() throws Exception { StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit)); NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); SourcePartitionedScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1); for (int i = 0; i < 60; i++) { ScheduleResult scheduleResult = scheduler.schedule(); // only finishes when last split is fetched assertEquals(scheduleResult.isFinished(), i == 59); // never blocks assertTrue(scheduleResult.getBlocked().isDone()); // first three splits create new tasks assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0); assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3); assertPartitionedSplitCount(stage, min(i + 1, 60)); } for (RemoteTask remoteTask : stage.getAllTasks()) { assertEquals(remoteTask.getPartitionedSplitCount(), 20); } stage.abort(); }
@BeforeMethod public void setUp() throws Exception { finalizerService = new FinalizerService(); nodeTaskMap = new NodeTaskMap(finalizerService); nodeManager = new InMemoryNodeManager(); ImmutableList.Builder<Node> nodeBuilder = ImmutableList.builder(); nodeBuilder.add(new PrestoNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN)); nodeBuilder.add(new PrestoNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN)); nodeBuilder.add(new PrestoNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN)); ImmutableList<Node> nodes = nodeBuilder.build(); nodeManager.addNode("foo", nodes); NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() .setMaxSplitsPerNode(20) .setIncludeCoordinator(false) .setMaxPendingSplitsPerNodePerTask(10); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); // contents of taskMap indicate the node-task map for the current stage taskMap = new HashMap<>(); nodeSelector = nodeScheduler.createNodeSelector("foo"); remoteTaskExecutor = Executors.newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s")); finalizerService.start(); }
throws Exception NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
@Test public void testNoNodes() throws Exception { try { NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap); StageExecutionPlan plan = createPlan(createFixedSplitSource(20, TestingSplit::createRemoteSplit)); SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap); SourcePartitionedScheduler scheduler = new SourcePartitionedScheduler( stage, plan.getDataSource().get(), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector("test"), stage::getAllTasks), 2); scheduler.schedule(); fail("expected PrestoException"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), NO_NODES_AVAILABLE.toErrorCode()); } }