@Override public void run() { try { Iterator<GroupElement> solveIter = solverQueues.takeIterator(id); while (solveIter.hasNext()) { final GroupElement gElement = solveIter.next(); final Element solveElement = gElement.getElement(); final ListenableFuture<String> future = executorService.submit(new SolverRunner(gElement)); future.addListener(new Runnable() { @Override public void run() { try { solverQueues.recordProgress(id, gElement.getQueueName(), solveElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, future.get()); } catch (Exception e) { LOG.error("Unable to record progress for cluster {}", solveElement.getId()); } } }, executorService); } } catch (Exception e) { LOG.error("Got exception:", e); } }
@Override public void run() { try { Iterator<GroupElement> callbackIter = callbackQueues.takeIterator(id); while (callbackIter.hasNext()) { final GroupElement gElement = callbackIter.next(); final Element element = gElement.getElement(); final ListenableFuture future = executorService.submit(new CallbackRunner(gElement)); future.addListener(new Runnable() { @Override public void run() { try { callbackQueues.recordProgress(id, gElement.getQueueName(), element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Executed"); } catch (Exception e) { LOG.error("Exception processing callback", e); } } }, executorService); } } catch (Exception e) { LOG.error("Got exception:", e); } }
@Override public void run() { try { barrier.await(); } catch (Exception e) { Throwables.propagate(e); } while (numTaken.get() < expectedNumElements) { Iterator<GroupElement> iter = queues.takeIterator(consumerId); while (iter.hasNext()) { GroupElement taken = iter.next(); numTaken.getAndIncrement(); elements.put(Integer.valueOf(taken.getElement().getValue()), Integer.valueOf(taken.getQueueName())); } try { TimeUnit.MILLISECONDS.sleep(10); } catch (InterruptedException e) { Throwables.propagate(e); } } latch.countDown(); } });
@Override public void run() { try { Iterator<GroupElement> jobIter = jobQueues.takeIterator(consumerId); while (jobIter.hasNext()) { GroupElement gElement = jobIter.next();
@Override public void run() { try { Iterator<GroupElement> clusterIter = clusterQueues.takeIterator(id); while (clusterIter.hasNext()) { GroupElement gElement = clusterIter.next();
@Test public void testOneQueueAddTakeWithoutQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; String consumerId = "worker.0"; queues.add(queueName, new Element("id", "val")); GroupElement taken = queues.takeIterator(consumerId).next(); Assert.assertEquals(queueName, taken.getQueueName()); Assert.assertEquals("id", taken.getElement().getId()); Assert.assertEquals("val", taken.getElement().getValue()); }
@Test public void testQueuedTaskMissingFromStoreIsRemovedFromQueue() { ClusterCleanup clusterCleanup = new ClusterCleanup(clusterStore, clusterService, nodeService, taskService, jobQueues, provisionerQueues, -10, 1, 1); String queueName = account.getTenantId(); ClusterTask task = new ClusterTask(ProvisionerAction.CREATE, TaskId.fromString("3-1-1"), "node1", "service", ClusterAction.CLUSTER_CREATE, "test", account); task.setStatus(ClusterTask.Status.IN_PROGRESS); Cluster cluster = Entities.ClusterExample.createCluster(); TaskConfig taskConfig = TaskConfig.from(cluster, Entities.ClusterExample.NODE1, Entities.ServiceExample.NAMENODE, cluster.getConfig(), ProvisionerAction.START, null); SchedulableTask schedulableTask = new SchedulableTask(task, taskConfig); // add a task to the queue without storing it.x provisionerQueues.add(queueName, new Element(task.getTaskId(), gson.toJson(schedulableTask))); provisionerQueues.takeIterator("0").next(); clusterCleanup.run(); Assert.assertEquals(0, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); }
@Test public void testChangesSeenAcrossInstances() throws Exception { QueueGroup instance1 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); QueueGroup instance2 = new ZKQueueGroup(zkClient, QueueType.PROVISIONER); instance1.startAndWait(); instance2.startAndWait(); // add a queue for tenant3 with 2 elements String tenant = "tenantX"; Set<String> expectedQueueNames = Sets.newHashSet(tenant); instance1.add(tenant, new Element("id3-1", "val1")); instance1.add(tenant, new Element("id3-2", "val2")); // check both instances see tenant3 Assert.assertEquals(expectedQueueNames, instance1.getQueueNames()); waitForQueueNames(expectedQueueNames, instance2); // make sure each instance gets an accurate picture of the queue Iterator<GroupElement> queuesIter1 = instance1.takeIterator("consumer1"); Iterator<GroupElement> queuesIter2 = instance1.takeIterator("consumer2"); GroupElement gelement = queuesIter1.next(); Assert.assertEquals(tenant, gelement.getQueueName()); Assert.assertEquals("id3-1", gelement.getElement().getId()); Assert.assertEquals("val1", gelement.getElement().getValue()); gelement = queuesIter2.next(); Assert.assertEquals(tenant, gelement.getQueueName()); Assert.assertEquals("id3-2", gelement.getElement().getId()); Assert.assertEquals("val2", gelement.getElement().getValue()); Assert.assertFalse(queuesIter1.hasNext()); Assert.assertFalse(queuesIter2.hasNext()); instance1.stop(); instance2.stop(); }
Assert.assertEquals(provisionerResult, actualNode.getProvisionerResults()); Assert.assertFalse(provisionerQueues.takeIterator("worker1").hasNext());
queues.add("tenant2", new Element("id5", "val")); Iterator<GroupElement> iter = queues.takeIterator("consumer");
@Test public void testRemoveAll() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1", "val")); queues.add("tenant2", new Element("id2", "val")); queues.add("tenant3", new Element("id3", "val")); queues.add("tenant4", new Element("id4", "val")); Assert.assertTrue(queues.removeAll()); Assert.assertFalse(queues.takeIterator("consumer").hasNext()); Assert.assertNull(queues.take("tenant1", "consumer")); Assert.assertNull(queues.take("tenant2", "consumer")); Assert.assertNull(queues.take("tenant3", "consumer")); Assert.assertNull(queues.take("tenant4", "consumer")); }
provisionerQueues.takeIterator("consumer1").next(); provisionerQueues.takeIterator("consumer2").next(); Assert.assertEquals(2, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); Assert.assertEquals(2, Iterators.size(provisionerQueues.getQueued(queueName))); provisionerQueues.takeIterator("consumer3").next(); Assert.assertEquals(3, Iterators.size(provisionerQueues.getBeingConsumed(queueName))); Assert.assertEquals(1, Iterators.size(provisionerQueues.getQueued(queueName)));