@Test public void testOneQueueAddTakeWithQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; String consumerId = "worker.0"; queues.add(queueName, new Element("id", "val")); Element taken = queues.take(queueName, consumerId); Assert.assertEquals("id", taken.getId()); Assert.assertEquals("val", taken.getValue()); }
@Override public ListenableFuture<String> add(Element element) { Preconditions.checkArgument(element != null, "element to add must not be null"); SettableFuture<String> result = addConsumingResultToWaitFor(element.getId()); if (!elementsTracking.addToQueue(element)) { result.setException(new RuntimeException("failed to add element to a queue " + element.toString())); stopWaitingForConsumingResult(element.getId()); } return result; }
@Override public void run() { while (true) { Element element = balanceQueue.take(id); if (element == null) { return; } try { tenantProvisionerService.rebalanceTenantWorkers(element.getValue()); balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "balanced"); } catch (CapacityException e) { LOG.error("Not enough capacity trying to balance workers for tenant {}", element.getValue(), e); // a failed status puts the element back in the queue, we don't want to consume this element again. // when another provisioner comes online, workers will get rebalanced once again. balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "not enough capacity"); } catch (IOException e) { LOG.error("Exception trying to balance workers for tenant {}", element.getValue(), e); balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "failed"); } } } }
Map<String, ListenableFuture<String>> results = Maps.newHashMap(); for (int i = 0; i < 60; i++) { Element element = new Element(producerName + "_" + i, String.valueOf(i)); results.put(element.getId(), queue.add(element)); System.out.println("produced: " + element.getId()); producedCount++; producedSum += i;
/** * Write a provisioner, and queue jobs to rebalance workers for all tenants in the system. * * @param provisioner Provisioner to write * @throws IOException */ public void writeProvisioner(Provisioner provisioner) throws IOException { tenantLock.lock(); try { provisionerStore.writeProvisioner(provisioner); // rebalance tenants every time a provisioner registers itself for (Tenant tenant : tenantStore.getAllTenants()) { balanceQueue.add(new Element(tenant.getId())); } } finally { tenantLock.unlock(); } }
private void stopAndReschedule(Entry entry) throws Exception { entry.consumerId = NO_CONSUMER_ASSIGNED; entry.lastProgressReportTs = 0; queueElements.put(entry.element.getId(), entry); }
@Override public void run() { try { barrier.await(); } catch (Exception e) { Throwables.propagate(e); } while (numTaken.get() < expectedNumElements) { Iterator<GroupElement> iter = queues.takeIterator(consumerId); while (iter.hasNext()) { GroupElement taken = iter.next(); numTaken.getAndIncrement(); elements.put(Integer.valueOf(taken.getElement().getValue()), Integer.valueOf(taken.getQueueName())); } try { TimeUnit.MILLISECONDS.sleep(10); } catch (InterruptedException e) { Throwables.propagate(e); } } latch.countDown(); } });
CallbackScheduler callbackScheduler = injector.getInstance(CallbackScheduler.class); clusterQueues.add(tenantId, new Element(cluster.getId(), ClusterAction.CLUSTER_CREATE.name())); clusterScheduler.run(); waitForCallback(callbackScheduler); String consumerId = "testJobScheduler"; Element jobQueueElement = jobQueues.take(tenantId, consumerId); String jobId = jobQueueElement.getValue(); jobQueues.add(tenantId, new Element(jobId)); JobScheduler jobScheduler = injector.getInstance(JobScheduler.class); jobScheduler.run(); jobQueues.add(tenantId, new Element(jobId)); jobScheduler.run(); job = clusterStore.getClusterJob(JobId.fromString(jobId));
private void printQueueElements(ElementsTrackingQueue queue) { System.out.println("Queued: "); int count = Iterators.size(queue.getQueued()); System.out.println("# of queued: " + count); Iterator<QueuedElement> beingConsumed = queue.getBeingConsumed(); System.out.println("Being Consumed:"); while (beingConsumed.hasNext()) { QueuedElement entry = beingConsumed.next(); System.out.println(entry.getConsumerId() + ": " + entry.getElement().toString()); } System.out.println("# of being consumed: " + Iterators.size(beingConsumed)); } }
@Override public void run() { try { barrier.await(); } catch (Exception e) { Throwables.propagate(e); } for (int j = 0; j < addsPerThread; j++) { queues.add(queueName, new Element(String.valueOf(producerNum * addsPerThread + j))); } latch.countDown(); } });
LOG.trace("Task {} with queue time {} has not timed out yet", queuedElement.getElement().getId(), queuedElement.getStatusTime()); continue; String taskId = queuedElement.getElement().getId(); ClusterTask task = clusterStore.getClusterTask(TaskId.fromString(taskId)); jobQueues.add(queueName, new Element(task.getJobId()));
cycle++; Element element = queue.take(consumerName); System.out.println("took: " + (element == null ? "null" : element.getId())); if (element == null) { System.out.println("failed to consume, retry: " + retries++); queue.recordProgress(consumerName, element.getId(), TrackingQueue.ConsumingStatus.FAILED, null); System.out.println("failed consuming: " + element.getId()); continue; queue.recordProgress(consumerName, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "result-" + element.getId()); System.out.println("posesses: " + state.name() + ", elem: " + element.getId()); System.out.println("consumed: " + element.getId()); consumedCount++; consumedSum += Integer.valueOf(element.getValue());
@Override public synchronized boolean remove(String elementId) { PrioritizedElement toRemove = null; for (PrioritizedElement elem : notStarted) { if (elementId.equals(elem.element.getId())) { toRemove = elem; break; } } if (toRemove != null) { notStarted.remove(toRemove); } inProgress.remove(elementId); return true; }
@Override public void run() { CallbackData callbackData = gson.fromJson(gElement.getElement().getValue(), CallbackData.class); CallbackContext callbackContext = new CallbackContext(clusterStoreService, userStore, callbackData.getCluster().getAccount()); switch (callbackData.getType()) { case START: onStart(callbackData, callbackContext); break; case SUCCESS: clusterCallback.onSuccess(callbackData, callbackContext); break; case FAILURE: clusterCallback.onFailure(callbackData, callbackContext); break; default: LOG.error("Unknown callback type {}", callbackData.getType()); } }
@Test public void testOneQueueAddTakeWithoutQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; String consumerId = "worker.0"; queues.add(queueName, new Element("id", "val")); GroupElement taken = queues.takeIterator(consumerId).next(); Assert.assertEquals(queueName, taken.getQueueName()); Assert.assertEquals("id", taken.getElement().getId()); Assert.assertEquals("val", taken.getElement().getValue()); }
/** * Delete and reassign workers for provisioners that have not sent a heartbeat since the given timestamp in * milliseconds. * * @param timeoutTs Timestamp in milliseconds to use as a cut off for deleting provisioners. * @throws IOException */ public void timeoutProvisioners(long timeoutTs) throws IOException { tenantLock.lock(); try { Set<String> affectedTenants = Sets.newHashSet(); for (Provisioner provisioner : provisionerStore.getTimedOutProvisioners(timeoutTs)) { String provisionerId = provisioner.getId(); LOG.error("provisioner {} has not sent a heartbeat in over {} seconds, deleting it...", provisionerId, provisionerTimeoutSecs); provisionerStore.deleteProvisioner(provisioner.getId()); affectedTenants.addAll(provisioner.getAssignedTenants()); } for (String affectedTenant : affectedTenants) { balanceQueue.add(new Element(affectedTenant)); } } finally { tenantLock.unlock(); } }
@Test public void testMultiQueueTakeWithQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); queues.add("tenant1", new Element("id1-1", "val")); queues.add("tenant1", new Element("id1-2", "val")); queues.add("tenant1", new Element("id1-3", "val")); queues.add("tenant2", new Element("id2-1", "val")); queues.add("tenant2", new Element("id2-2", "val")); queues.add("tenant2", new Element("id2-3", "val")); queues.add("tenant3", new Element("id3-1", "val")); queues.add("tenant4", new Element("id4-1", "val")); Assert.assertEquals("id1-1", taken.getId()); Assert.assertEquals("id1-2", taken.getId()); Assert.assertEquals("id2-1", taken.getId()); Assert.assertEquals("id2-2", taken.getId()); Assert.assertEquals("id2-3", taken.getId()); Assert.assertEquals("id3-1", taken.getId()); Assert.assertEquals("id4-1", taken.getId()); Assert.assertEquals("id1-3", taken.getId());
GroupElement gElement = clusterIter.next(); Element clusterElement = gElement.getElement(); Cluster cluster = clusterStore.getCluster(clusterElement.getId()); ClusterJob job = clusterStore.getClusterJob(JobId.fromString(cluster.getLatestJobId())); ClusterAction clusterAction = ClusterAction.valueOf(clusterElement.getValue()); LOG.debug("Got cluster {} with action {}", cluster.getName(), clusterAction); try { if (actionOrder == null) { LOG.error("Cluster action {} does not have any provisioner actions defined", clusterAction); clusterQueues.recordProgress(id, gElement.getQueueName(), clusterElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "No actions defined"); continue; if (clusterNodes == null || clusterNodes.isEmpty()) { LOG.error("Cluster {} has no nodes defined", cluster.getId()); clusterQueues.recordProgress(id, gElement.getQueueName(), clusterElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "No nodes defined"); continue; clusterQueues.recordProgress(id, gElement.getQueueName(), clusterElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Scheduled"); } catch (Throwable e) { clusterQueues.recordProgress(id, gElement.getQueueName(), clusterElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Exception during scheduling");
@Override public synchronized boolean toHighestPriority(String elementId) { // we adjust priority only in those not being consumed. If consuming fails for the element it will be promoted to // highest priority anyways PrioritizedElement toChange = null; for (PrioritizedElement elem : notStarted) { if (elementId.equals(elem.element.getId())) { toChange = elem; break; } } if (toChange != null) { notStarted.remove(toChange); toChange.priority = HIGHEST_PRIORITY; notStarted.add(toChange); } return true; }
SolverRequest solverRequest = gson.fromJson(solveElement.getValue(), SolverRequest.class); try { solverJob.setJobStatus(ClusterJob.Status.RUNNING);