@Override public void expect(ClusterState state, SchedulerDriver mockDriver) { Mockito.verify(mockDriver, Mockito.times(totalTimes)).reviveOffers(); }
public synchronized void requestJenkinsSlave(Mesos.SlaveRequest request, Mesos.SlaveResult result) { Metrics.metricRegistry().meter("mesos.scheduler.slave.requests").mark(); LOGGER.fine("Enqueuing jenkins slave request"); requests.add(new Request(request, result)); if (driver != null) { // Ask mesos to send all offers, even the those we declined earlier. // See comment in resourceOffers() for further details. Timer.Context ctx = Metrics.metricRegistry().timer("mesos.scheduler.revives").time(); driver.reviveOffers(); ctx.stop(); } }
/** * Pings the manager to perform a revive call to Mesos if one was previously requested. This must be invoked * periodically to trigger revives. This structure allows us to enforce a rate limit on revive calls. */ void reviveIfRequested() { if (!reviveRequested) { return; } if (!reviveTokenBucket.tryAcquire()) { LOGGER.info("Revive attempt has been throttled"); Metrics.incrementReviveThrottles(); return; } LOGGER.info("Reviving offers"); Driver.getInstance().reviveOffers(); reviveRequested = false; // NOTE: We intentionally do not clear isSuppressed here. Instead, we wait until we've actually received new // offers. This is a 'just in case' measure to avoid a zombie state if the revive call is dropped. In practice, // there isn't a confirmed case of this ever happening, but it doesn't hurt to be conservative here. Metrics.incrementRevives(); } }
driver.reviveOffers(); offersSuppressed = false;
private void suppressOrRevive() { boolean hasOperations = !planManager.getPlan().isComplete() || recoveryScheduler.hasOperations(); LOGGER.debug(hasOperations ? "Scheduler has operations to perform." : "Scheduler has no operations to perform."); if (hasOperations) { // Revive offers only if they were previously suppressed. if (cassandraState.isSuppressed()) { LOGGER.info("Reviving offers."); driver.reviveOffers(); cassandraState.setSuppressed(false); } } else { LOGGER.info("Suppressing offers."); driver.suppressOffers(); cassandraState.setSuppressed(true); } }
@Test public void dontReviveWhenNotRequested() { ReviveManager manager = getReviveManager(); manager.reviveIfRequested(); verify(driver, times(0)).reviveOffers(); }
@Test public void dontReviveWhenThrottled() { ReviveManager manager = getReviveManager(); manager.requestRevive(); manager.reviveIfRequested(); manager.requestRevive(); manager.reviveIfRequested(); verify(driver, times(1)).reviveOffers(); }
@Test public void suppressRevive() { ReviveManager manager = new ReviveManager( TokenBucket.newBuilder().acquireInterval(Duration.ZERO).build(), mockSchedulerConfig); // Suppress: manager.suppressIfActive(); verify(driver, times(1)).suppressOffers(); // Revive: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(1)).reviveOffers(); // Revive again, because previous revive apparently didn't go through: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(2)).reviveOffers(); // Finally get an offer, un-suppress must've worked. manager.notifyOffersReceived(); // Now that we aren't suppressed, revive is not triggered by just needing offers: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(2)).reviveOffers(); // .. but still revives if specifically requested (due to new work): manager.requestRevive(); manager.reviveIfRequested(); verify(driver, times(3)).reviveOffers(); }
@Test public void dontReviveAcrossManagers() { // Both managers should share the same underlying token bucket: TokenBucket tokenBucket = TokenBucket.newBuilder().acquireInterval(Duration.ofDays(1)).build(); ReviveManager a = new ReviveManager(tokenBucket, mockSchedulerConfig); ReviveManager b = new ReviveManager(tokenBucket, mockSchedulerConfig); a.requestRevive(); b.requestRevive(); a.reviveIfRequested(); // pass b.reviveIfRequested(); // throttled verify(driver, times(1)).reviveOffers(); }