private void stopAndReschedule(Entry entry) throws Exception { entry.consumerId = NO_CONSUMER_ASSIGNED; entry.lastProgressReportTs = 0; queueElements.put(entry.element.getId(), entry); }
@Override public synchronized boolean remove(String elementId) { PrioritizedElement toRemove = null; for (PrioritizedElement elem : notStarted) { if (elementId.equals(elem.element.getId())) { toRemove = elem; break; } } if (toRemove != null) { notStarted.remove(toRemove); } inProgress.remove(elementId); return true; }
@Override public synchronized boolean toHighestPriority(String elementId) { // we adjust priority only in those not being consumed. If consuming fails for the element it will be promoted to // highest priority anyways PrioritizedElement toChange = null; for (PrioritizedElement elem : notStarted) { if (elementId.equals(elem.element.getId())) { toChange = elem; break; } } if (toChange != null) { notStarted.remove(toChange); toChange.priority = HIGHEST_PRIORITY; notStarted.add(toChange); } return true; }
@Override public void run() { try { solverQueues.recordProgress(id, gElement.getQueueName(), solveElement.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, future.get()); } catch (Exception e) { LOG.error("Unable to record progress for cluster {}", solveElement.getId()); } } }, executorService);
@Override public synchronized Element startConsuming(String consumerId) { PrioritizedElement element = notStarted.poll(); if (element == null) { return null; } inProgress.put(element.element.getId(), new ElementBeingConsumed(element.element, consumerId, element.priority)); return element.element; }
@Override public boolean toHighestPriority(String elementId) { try { globalLock.get().lock(); try { Entry entry = queueElements.get(elementId); if (NO_CONSUMER_ASSIGNED.equals(entry.consumerId)) { entry.priority = HIGHEST_PRIORITY; queueElements.put(entry.element.getId(), entry); } } finally { globalLock.get().unlock(); } } catch (Exception e) { LOG.error("error during promoting element to highest priority", e); // should be OK not do anything: we'll walk thru next time :) } return true; }
private synchronized void stopAndReschedule(ElementBeingConsumed element) { inProgress.remove(element.element.getId()); notStarted.add(new PrioritizedElement(element.element, element.priority)); }
@Override public boolean recordProgress(String elementId, String consumerId) { try { globalLock.get().lock(); try { Entry entry = queueElements.get(elementId); if (entry == null || !consumerId.equals(entry.consumerId)) { return false; } entry.lastProgressReportTs = System.currentTimeMillis(); queueElements.put(entry.element.getId(), entry); } finally { globalLock.get().unlock(); } } catch (Exception e) { LOG.error("error during checking state", e); // should be OK not do anything: we are just reporting a progress... } // TODO: what if we failed? return true; }
@Override public ListenableFuture<String> add(Element element) { Preconditions.checkArgument(element != null, "element to add must not be null"); SettableFuture<String> result = addConsumingResultToWaitFor(element.getId()); if (!elementsTracking.addToQueue(element)) { result.setException(new RuntimeException("failed to add element to a queue " + element.toString())); stopWaitingForConsumingResult(element.getId()); } return result; }
@Override public void run() { try { callbackQueues.recordProgress(id, gElement.getQueueName(), element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "Executed"); } catch (Exception e) { LOG.error("Exception processing callback", e); } } }, executorService);
@Override public boolean addToQueue(Element element) { try { // we actually may need no lock here: we just adding new element (it is assumed that test adds unique // elems into queue) globalLock.get().lock(); try { Entry entry = new Entry(element, getCurrentHighestPriority()); queueElements.put(entry.element.getId(), entry); return true; } finally { globalLock.get().unlock(); } } catch (Exception e) { LOG.error("error during adding to queue", e); return false; } }
@Override public Element startConsuming(String consumerId) { try { globalLock.get().lock(); try { Entry entry = getNotStartedWithHighestPriority(); if (entry == null) { return null; } entry.consumerId = consumerId; entry.lastProgressReportTs = System.currentTimeMillis(); entry.priority = getCurrentHighestPriority(); queueElements.put(entry.element.getId(), entry); return entry.element; } finally { globalLock.get().unlock(); } } catch (Exception e) { LOG.error("error during start consuming", e); return null; } }
private SolverRunner(GroupElement gElement) { this.solveElement = gElement.getElement(); this.queueName = gElement.getQueueName(); this.clusterId = solveElement.getId(); this.solverJob = null; this.plannerJob = null; }
private Set<String> getIds(Iterator<QueuedElement> iter) { Set<String> out = Sets.newHashSet(); while (iter.hasNext()) { out.add(iter.next().getElement().getId()); } return out; } }
@Override public void run() { while (true) { Element element = balanceQueue.take(id); if (element == null) { return; } try { tenantProvisionerService.rebalanceTenantWorkers(element.getValue()); balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "balanced"); } catch (CapacityException e) { LOG.error("Not enough capacity trying to balance workers for tenant {}", element.getValue(), e); // a failed status puts the element back in the queue, we don't want to consume this element again. // when another provisioner comes online, workers will get rebalanced once again. balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "not enough capacity"); } catch (IOException e) { LOG.error("Exception trying to balance workers for tenant {}", element.getValue(), e); balanceQueue.recordProgress(id, element.getId(), TrackingQueue.ConsumingStatus.FINISHED_SUCCESSFULLY, "failed"); } } } }
@Test public void testOnlyCorrectClustersAreCleaned() throws Exception { long now = System.currentTimeMillis(); for (int i = 0; i < 20; i++) { createCluster(String.valueOf(i), now - 1000, now - 100, Cluster.Status.ACTIVE); } String queueName = account.getTenantId(); ClusterCleanup clusterCleanup = new ClusterCleanup(clusterStore, clusterService, nodeService, taskService, jobQueues, provisionerQueues, -10, 3, 7); Assert.assertEquals(0, Iterators.size(clusterQueues.getQueued(queueName))); clusterCleanup.run(); // clusters 3, 10, and 17 should have been scheduled for deletion Assert.assertEquals(3, Iterators.size(clusterQueues.getQueued(queueName))); Element e1 = clusterQueues.take(queueName, "consumer1"); Element e2 = clusterQueues.take(queueName, "consumer1"); Element e3 = clusterQueues.take(queueName, "consumer1"); Assert.assertEquals(ImmutableSet.of("3", "10", "17"), ImmutableSet.of(e1.getId(), e2.getId(), e3.getId())); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e1.getValue()); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e2.getValue()); Assert.assertEquals(ClusterAction.CLUSTER_DELETE.name(), e3.getValue()); }
Assert.assertEquals("id1-1", taken.getId()); Assert.assertEquals("id1-2", taken.getId()); Assert.assertEquals("id2-1", taken.getId()); Assert.assertEquals("id2-2", taken.getId()); Assert.assertEquals("id2-3", taken.getId()); Assert.assertEquals("id3-1", taken.getId()); Assert.assertEquals("id4-1", taken.getId()); Assert.assertEquals("id1-3", taken.getId());
@Test public void testOneQueueAddTakeWithQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; String consumerId = "worker.0"; queues.add(queueName, new Element("id", "val")); Element taken = queues.take(queueName, consumerId); Assert.assertEquals("id", taken.getId()); Assert.assertEquals("val", taken.getValue()); }
@Test public void testOneQueueAddTakeWithoutQueueName() { QueueGroup queues = getQueueGroup(QueueType.PROVISIONER); String queueName = "tenant1"; String consumerId = "worker.0"; queues.add(queueName, new Element("id", "val")); GroupElement taken = queues.takeIterator(consumerId).next(); Assert.assertEquals(queueName, taken.getQueueName()); Assert.assertEquals("id", taken.getElement().getId()); Assert.assertEquals("val", taken.getElement().getValue()); }
@Test public void testAddCluster() throws Exception { String clusterName = "my-cluster"; String tenantId = USER1_ACCOUNT.getTenantId(); ClusterCreateRequest clusterCreateRequest = ClusterCreateRequest.builder() .setName(clusterName) .setClusterTemplateName(reactorTemplate.getName()) .setNumMachines(5) .build(); HttpResponse response = doPostExternalAPI("/clusters", gson.toJson(clusterCreateRequest), USER1_HEADERS); assertResponseStatus(response, HttpResponseStatus.OK); String clusterId = getIdFromResponse(response); // check there was an element added to the cluster queue for creating this cluster Element element = solverQueues.take(tenantId, "0"); Assert.assertEquals(clusterId, element.getId()); ClusterCreateRequest expected = ClusterCreateRequest.builder() .setName(clusterName) .setClusterTemplateName(reactorTemplate.getName()) .setNumMachines(5) .setInitialLeaseDuration(-1L) .build(); SolverRequest expectedSolverRequest = new SolverRequest(SolverRequest.Type.CREATE_CLUSTER, gson.toJson(expected)); Assert.assertEquals(expectedSolverRequest, gson.fromJson(element.getValue(), SolverRequest.class)); }