@Test public void testGetQueueNames() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id", "val")); queues.take("tenant2", "consumer.0"); queues.take("tenant3", "consumer.1"); queues.removeAll("tenant4"); queues.getBeingConsumed("tenant5"); queues.getQueued("tenant6"); ImmutableSet<String> expected = ImmutableSet.of("tenant1", "tenant2", "tenant3", "tenant4", "tenant5", "tenant6"); ImmutableSet<String> actual = ImmutableSet.copyOf(queues.getQueueNames()); Assert.assertEquals(expected, actual); }
@Test public void testGetSize() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1", "val")); queues.add("tenant1", new Element("id2", "val")); queues.add("tenant1", new Element("id3", "val")); queues.add("tenant1", new Element("id4", "val")); queues.add("tenant2", new Element("id1", "val")); queues.add("tenant2", new Element("id2", "val")); queues.add("tenant3", new Element("id1", "val")); Assert.assertEquals(4, queues.size("tenant1")); Assert.assertEquals(2, queues.size("tenant2")); Assert.assertEquals(1, queues.size("tenant3")); // size includes elements being consumed queues.take("tenant1", "consumer"); Assert.assertEquals(4, queues.size("tenant1")); Assert.assertEquals(2, queues.size("tenant2")); Assert.assertEquals(1, queues.size("tenant3")); // size does not include elements that are finished being consumed queues.recordProgress("consumer", "tenant1", "id1", TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "result"); Assert.assertEquals(3, queues.size("tenant1")); Assert.assertEquals(2, queues.size("tenant2")); Assert.assertEquals(1, queues.size("tenant3")); }
@Test public void testInstanceInitializedWithExistingData() throws Exception { QueueGroup instance1 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); instance1.startAndWait(); instance1.add("tenant1", new Element("val1")); instance1.add("tenant2", new Element("val2")); QueueGroup instance2 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); instance2.startAndWait(); waitForQueueNames(Sets.newHashSet("tenant1", "tenant2"), instance2); instance1.stop(); instance2.stop(); }
@Override public QueueMetrics load(String tenantId) { int numQueued = Iterators.size(taskQueues.getQueued(tenantId)); int numInProgress = Iterators.size(taskQueues.getBeingConsumed(tenantId)); return new QueueMetrics(numQueued, numInProgress); } });
Assert.assertEquals(Node.Status.IN_PROGRESS, node4.getActions().get(0).getStatus()); Assert.assertTrue(jobQueues.removeAll(queueName)); Assert.assertEquals(0, Iterators.size(provisionerQueues.getQueued(queueName))); Assert.assertEquals(0, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); provisionerQueues.add(queueName, new Element(task1.getTaskId(), "")); provisionerQueues.add(queueName, new Element(task2.getTaskId(), "")); provisionerQueues.add(queueName, new Element(task3.getTaskId(), "")); provisionerQueues.add(queueName, new Element(task4.getTaskId(), "")); Assert.assertEquals(4, Iterators.size(provisionerQueues.getQueued(queueName))); provisionerQueues.takeIterator("consumer1").next(); provisionerQueues.takeIterator("consumer2").next(); Assert.assertEquals(2, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); Assert.assertEquals(2, Iterators.size(provisionerQueues.getQueued(queueName))); provisionerQueues.takeIterator("consumer3").next(); Assert.assertEquals(3, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); Assert.assertEquals(1, Iterators.size(provisionerQueues.getQueued(queueName))); Assert.assertEquals(1, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); Assert.assertEquals(1, Iterators.size(provisionerQueues.getQueued(queueName))); Assert.assertEquals(2, jobQueues.size(queueName));
@Test public void testOneQueueGetQueuedAndConsumed() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; queues.add(queueName, new Element("id1", "val")); queues.add(queueName, new Element("id2", "val")); // check being consumed is correct Assert.assertEquals(ImmutableSet.<String>of(), getIds(queues.getBeingConsumed(queueName))); // check queued is correct. Assert.assertEquals(ImmutableSet.of("id1", "id2"), getIds(queues.getQueued(queueName))); // take one element queues.take(queueName, "consumer1"); // check being consumed is correct Assert.assertEquals(ImmutableSet.of("id1"), getIds(queues.getBeingConsumed(queueName))); // check queued is correct. Assert.assertEquals(ImmutableSet.of("id2"), getIds(queues.getQueued(queueName))); // take next element queues.take(queueName, "consumer2"); // check being consumed is correct Assert.assertEquals(ImmutableSet.of("id1", "id2"), getIds(queues.getBeingConsumed(queueName))); // check queued is correct. Assert.assertEquals(ImmutableSet.<String>of(), getIds(queues.getQueued(queueName))); // finish first element queues.recordProgress("consumer1", queueName, "id1", TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "result"); // check being consumed is correct Assert.assertEquals(ImmutableSet.of("id2"), getIds(queues.getBeingConsumed(queueName))); // check queued is correct. Assert.assertEquals(ImmutableSet.<String>of(), getIds(queues.getQueued(queueName))); }
@Test public void testRemoveAllForOneQueue() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1", "val")); queues.add("tenant2", new Element("id2", "val")); queues.add("tenant3", new Element("id3", "val")); queues.add("tenant4", new Element("id4", "val")); Assert.assertTrue(queues.removeAll("tenant3")); Assert.assertNull(queues.take("tenant3", "consumer")); Assert.assertNotNull(queues.take("tenant1", "consumer")); Assert.assertNotNull(queues.take("tenant2", "consumer")); Assert.assertNotNull(queues.take("tenant4", "consumer")); }
@Test public void testChangesSeenAcrossInstances() throws Exception { QueueGroup instance1 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); QueueGroup instance2 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); instance1.startAndWait(); instance2.startAndWait(); // add a queue for tenant3 with 2 elements String tenant = "tenantX"; Set<String> expectedQueueNames = Sets.newHashSet(tenant); instance1.add(tenant, new Element("id3-1", "val1")); instance1.add(tenant, new Element("id3-2", "val2")); // check both instances see tenant3 Assert.assertEquals(expectedQueueNames, instance1.getQueueNames()); waitForQueueNames(expectedQueueNames, instance2); // make sure each instance gets an accurate picture of the queue Iterator<GroupElement> queuesIter1 = instance1.takeIterator("consumer1"); Iterator<GroupElement> queuesIter2 = instance1.takeIterator("consumer2"); GroupElement gelement = queuesIter1.next(); Assert.assertEquals(tenant, gelement.getQueueName()); Assert.assertEquals("id3-1", gelement.getElement().getId()); Assert.assertEquals("val1", gelement.getElement().getValue()); gelement = queuesIter2.next(); Assert.assertEquals(tenant, gelement.getQueueName()); Assert.assertEquals("id3-2", gelement.getElement().getId()); Assert.assertEquals("val2", gelement.getElement().getValue()); Assert.assertFalse(queuesIter1.hasNext()); Assert.assertFalse(queuesIter2.hasNext()); instance1.stop(); instance2.stop(); }
@Test public void testRemoveAll() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1", "val")); queues.add("tenant2", new Element("id2", "val")); queues.add("tenant3", new Element("id3", "val")); queues.add("tenant4", new Element("id4", "val")); Assert.assertTrue(queues.removeAll()); Assert.assertFalse(queues.takeIterator("consumer").hasNext()); Assert.assertNull(queues.take("tenant1", "consumer")); Assert.assertNull(queues.take("tenant2", "consumer")); Assert.assertNull(queues.take("tenant3", "consumer")); Assert.assertNull(queues.take("tenant4", "consumer")); }
@Override public void run() { try { barrier.await(); } catch (Exception e) { Throwables.propagate(e); } for (int j = 0; j < addsPerThread; j++) { queues.add(queueName, new Element(String.valueOf(producerNum * addsPerThread + j))); } latch.countDown(); } });
@Test public void testMultiQueueTakeWithQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1-1", "val")); queues.add("tenant1", new Element("id1-2", "val")); queues.add("tenant1", new Element("id1-3", "val")); queues.add("tenant2", new Element("id2-1", "val")); queues.add("tenant2", new Element("id2-2", "val")); queues.add("tenant2", new Element("id2-3", "val")); queues.add("tenant3", new Element("id3-1", "val")); queues.add("tenant4", new Element("id4-1", "val")); Element taken = queues.take("tenant1", "consumer"); Assert.assertEquals("id1-1", taken.getId()); taken = queues.take("tenant1", "consumer"); Assert.assertEquals("id1-2", taken.getId()); taken = queues.take("tenant2", "consumer"); Assert.assertEquals("id2-1", taken.getId()); taken = queues.take("tenant2", "consumer"); Assert.assertEquals("id2-2", taken.getId()); taken = queues.take("tenant2", "consumer"); Assert.assertEquals("id2-3", taken.getId()); taken = queues.take("tenant3", "consumer"); Assert.assertEquals("id3-1", taken.getId());
@Test public void testMultiQueueTakeIterator() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1", "val")); queues.add("tenant1", new Element("id2", "val")); queues.add("tenant1", new Element("id3", "val")); queues.add("tenant2", new Element("id1", "val")); queues.add("tenant2", new Element("id2", "val")); queues.add("tenant2", new Element("id3", "val")); queues.add("tenant2", new Element("id4", "val")); queues.add("tenant2", new Element("id5", "val")); Iterator<GroupElement> iter = queues.takeIterator("consumer");
@Override public void run() { try { Iterator<GroupElement> jobIter = jobQueues.takeIterator(consumerId); while (jobIter.hasNext()) { GroupElement gElement = jobIter.next(); LOG.debug("Advancing to next stage {} for job {}", job.getCurrentStageNumber(), job.getJobId()); job.advanceStage(); jobQueues.add(queueName, new Element(jobIdStr)); } else { taskService.completeJob(job, cluster); jobQueues.recordProgress(consumerId, queueName, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "");
Element task = taskQueues.take(tenantId, consumerId); if (task == null) { break; taskQueues.recordProgress(consumerId, tenantId, clusterTask.getTaskId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Skipped due to job failure."); taskService.dropTask(clusterTask); jobQueues.add(tenantId, new Element(clusterTask.getJobId())); clusterTask = null; } else { taskQueues.recordProgress(consumerId, tenantId, task.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Skipped due to empty task JSON.");
@Test public void testQueuedTaskMissingFromStoreIsRemovedFromQueue() { ClusterCleanup clusterCleanup = new ClusterCleanup(clusterStore, clusterService, nodeService, taskService, jobQueues, provisionerQueues, -10, 1, 1); String queueName = account.getTenantId(); ClusterTask task = new ClusterTask(ProvisionerAction.CREATE, TaskId.fromString("3-1-1"), "node1", "service", ClusterAction.CLUSTER_CREATE, "test", account); task.setStatus(ClusterTask.Status.IN_PROGRESS); Cluster cluster = Entities.ClusterExample.createCluster(); TaskConfig taskConfig = TaskConfig.from(cluster, Entities.ClusterExample.NODE1, Entities.ServiceExample.NAMENODE, cluster.getConfig(), ProvisionerAction.START, null); SchedulableTask schedulableTask = new SchedulableTask(task, taskConfig); // add a task to the queue without storing it.x provisionerQueues.add(queueName, new Element(task.getTaskId(), gson.toJson(schedulableTask))); provisionerQueues.takeIterator("0").next(); clusterCleanup.run(); Assert.assertEquals(0, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); }
CallbackScheduler callbackScheduler = injector.getInstance(CallbackScheduler.class); clusterQueues.add(tenantId, new Element(cluster.getId(), ClusterAction.CLUSTER_CREATE.name())); clusterScheduler.run(); waitForCallback(callbackScheduler); Assert.assertEquals(1, jobQueues.size(tenantId)); String consumerId = "testJobScheduler"; Element jobQueueElement = jobQueues.take(tenantId, consumerId); String jobId = jobQueueElement.getValue(); jobQueues.add(tenantId, new Element(jobId)); JobScheduler jobScheduler = injector.getInstance(JobScheduler.class); jobScheduler.run(); jobQueues.add(tenantId, new Element(jobId)); jobScheduler.run(); job = clusterStore.getClusterJob(JobId.fromString(jobId));
@Test public void testOnlyCorrectClustersAreCleaned() throws Exception { long now = System.currentTimeMillis(); for (int i = 0; i < 20; i++) { createCluster(String.valueOf(i), now - 1000, now - 100, Cluster.Status.ACTIVE); } String queueName = account.getTenantId(); ClusterCleanup clusterCleanup = new ClusterCleanup(clusterStore, clusterService, nodeService, taskService, jobQueues, provisionerQueues, -10, 3, 7); Assert.assertEquals(0, Iterators.size(clusterQueues.getQueued(queueName))); clusterCleanup.run(); // clusters 3, 10, and 17 should have been scheduled for deletion Assert.assertEquals(3, Iterators.size(clusterQueues.getQueued(queueName))); Element e1 = clusterQueues.take(queueName, "consumer1"); Element e2 = clusterQueues.take(queueName, "consumer1"); Element e3 = clusterQueues.take(queueName, "consumer1"); Assert.assertEquals(ImmutableSet.of("3", "10", "17"), ImmutableSet.of(e1.getId(), e2.getId(), e3.getId())); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e1.getValue()); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e2.getValue()); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e3.getValue()); }
@Test(timeout = 20000) public void testFalseOnStartStopsJob() throws Exception { String tenantId = "q"; ClusterScheduler clusterScheduler = injector.getInstance(ClusterScheduler.class); clusterQueues.add(tenantId, new Element(cluster.getId(), ClusterAction.CLUSTER_CREATE.name())); clusterScheduler.run(); CallbackScheduler callbackScheduler = injector.getInstance(CallbackScheduler.class); // should be no job in the queue until the start callback runs Assert.assertEquals(0, jobQueues.size(tenantId)); // tell mock callback to return false for onStart callback mockClusterCallback.setReturnOnStart(false); // wait for start callback to finish waitForCallback(callbackScheduler); Assert.assertEquals(CallbackData.Type.START, mockClusterCallback.getReceivedCallbacks().get(0).getType()); // wait for fail callback to finish if (mockClusterCallback.getReceivedCallbacks().size() < 2) { waitForCallback(callbackScheduler); } Assert.assertEquals(CallbackData.Type.FAILURE, mockClusterCallback.getReceivedCallbacks().get(1).getType()); // there also should not be any jobs in the queue Assert.assertEquals(0, jobQueues.size(tenantId)); }
LOG.debug("Task fail time = {}", taskFailTime); Iterator<QueuedElement> beingConsumed = provisionerQueues.getBeingConsumed(queueName); LOG.warn("provisioner queue contains task {} which is not in the cluster store, removing it from the queue.", taskId); provisionerQueues.remove(queueName, taskId); continue; if (provisionerQueues.remove(queueName, task.getTaskId())) { LOG.debug("Timing out task {} whose queue time is {}", task.getTaskId(), queuedElement.getStatusTime()); jobQueues.add(queueName, new Element(task.getJobId()));