public List<SingularitySlave> getSlavesToTrackUsageFor() { List<SingularitySlave> slaves = slaveManager.getObjects(); List<SingularitySlave> slavesToTrack = new ArrayList<>(slaves.size()); for (SingularitySlave slave : slaves) { if (slave.getCurrentState().getState().isInactive() || slave.getCurrentState().getState() == MachineState.DECOMMISSIONED) { continue; } slavesToTrack.add(slave); } return slavesToTrack; }
@JsonIgnore public SingularitySlave withResources(MesosResourcesObject resources) { return new SingularitySlave(getId(), getFirstSeenAt(), getCurrentState(), host, rackId, attributes, Optional.of(resources)); }
@Override public int getNumActive() { if (leaderCache.active()) { return Math.toIntExact(leaderCache.getSlaves().stream().filter(x -> x.getCurrentState().getState().equals(MachineState.ACTIVE)).count()); } return super.getNumActive(); }
void slaveLost(AgentID slaveIdObj) { final String slaveId = slaveIdObj.getValue(); Optional<SingularitySlave> slave = slaveManager.getObject(slaveId); if (slave.isPresent()) { MachineState previousState = slave.get().getCurrentState().getState(); slaveManager.changeState(slave.get(), MachineState.DEAD, Optional.absent(), Optional.absent()); if (configuration.getDisasterDetection().isEnabled()) { updateDisasterCounter(previousState); } checkRackAfterSlaveLoss(slave.get()); } else { LOG.warn("Lost a slave {}, but didn't know about it", slaveId); } }
private void checkDeadSlaves() { final long start = System.currentTimeMillis(); final List<SingularitySlave> deadSlaves = slaveManager.getObjectsFiltered(MachineState.DEAD); if (deadSlaves.isEmpty()) { LOG.trace("No dead slaves"); return; } int deleted = 0; final long maxDuration = TimeUnit.HOURS.toMillis(configuration.getDeleteDeadSlavesAfterHours()); for (SingularitySlave deadSlave : slaveManager.getObjectsFiltered(MachineState.DEAD)) { final long duration = System.currentTimeMillis() - deadSlave.getCurrentState().getTimestamp(); if (duration > maxDuration) { SingularityDeleteResult result = slaveManager.deleteObject(deadSlave.getId()); deleted++; LOG.info("Removing dead slave {} ({}) after {} (max {})", deadSlave.getId(), result, JavaUtils.durationFromMillis(duration), JavaUtils.durationFromMillis(maxDuration)); } } LOG.debug("Checked {} dead slaves, deleted {} in {}", deadSlaves.size(), deleted, JavaUtils.duration(start)); }
void checkStateAfterFinishedTask(SingularityTaskId taskId, String slaveId, SingularityLeaderCache leaderCache) { Optional<SingularitySlave> slave = slaveManager.getSlave(slaveId); if (!slave.isPresent()) { final String message = String.format("Couldn't find slave with id %s for task %s", slaveId, taskId); LOG.warn(message); exceptionNotifier.notify(message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString())); return; } if (slave.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) { if (!hasTaskLeftOnSlave(taskId, slaveId, leaderCache)) { slaveManager.changeState(slave.get(), MachineState.DECOMMISSIONED, slave.get().getCurrentState().getMessage(), slave.get().getCurrentState().getUser()); } } Optional<SingularityRack> rack = rackManager.getObject(slave.get().getRackId()); if (!rack.isPresent()) { final String message = String.format("Couldn't find rack with id %s for task %s", slave.get().getRackId(), taskId); LOG.warn(message); exceptionNotifier.notify(message, ImmutableMap.of("rackId", slave.get().getRackId(), "taskId", taskId.toString())); return; } if (rack.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) { if (!hasTaskLeftOnRack(taskId, leaderCache)) { rackManager.changeState(rack.get(), MachineState.DECOMMISSIONED, rack.get().getCurrentState().getMessage(), rack.get().getCurrentState().getUser()); } } }
int numRunningSlaves = 0; for (SingularitySlave slave : slaves) { if (slave.getCurrentState().getState() != MachineState.DEAD && slave.getCurrentState().getState() != MachineState.MISSING_ON_STARTUP) { numRunningSlaves++;
@Test public void testExpiringMachineState() { MesosMasterStateObject state = getMasterState(1); singularitySlaveAndRackManager.loadSlavesAndRacksFromMaster(state, true); SingularitySlave slave = slaveManager.getObjects().get(0); slaveResource.freezeSlave(singularityUser, slave.getId(), new SingularityMachineChangeRequest(Optional.of(1L), Optional.absent(), Optional.absent(), Optional.of(MachineState.ACTIVE), Optional.absent())); Assert.assertEquals(MachineState.FROZEN, slaveManager.getObjects().get(0).getCurrentState().getState()); expiringUserActionPoller.runActionOnPoll(); Assert.assertEquals(MachineState.ACTIVE, slaveManager.getObjects().get(0).getCurrentState().getState()); }
@Test public void testEmptyDecommissioning() { sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(StateChangeResult.SUCCESS, slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.absent(), Optional.of("user1"))); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(MachineState.DECOMMISSIONED, slaveManager.getObject("slave1").get().getCurrentState().getState()); }
@Test public void testLoadSlavesFromMasterDataOnStartup() { MesosMasterStateObject state = getMasterState(3); singularitySlaveAndRackManager.loadSlavesAndRacksFromMaster(state, true); List<SingularitySlave> slaves = slaveManager.getObjects(); Assert.assertEquals(3, slaves.size()); for (SingularitySlave slave : slaves) { Assert.assertEquals(MachineState.ACTIVE, slave.getCurrentState().getState()); } }
switch (slave.getCurrentState().getState()) { case ACTIVE: activeSlaves++;
@Test public void itShouldMarkSlavesFromInactiveHostAsDecommissioned() { inactiveSlaveManager.deactivateSlave("host1"); resourceOffers(); SingularitySlave slave = slaveManager.getObject("slave1").get(); Assert.assertTrue(slave.getCurrentState().getState().isDecommissioning()); } }
final String slaveId = offerHolder.getSlaveId(); final MachineState currentSlaveState = slaveManager.getSlave(slaveId).get().getCurrentState().getState();
@Test public void testReconcileSlaves() { // Load 3 slaves on startup MesosMasterStateObject state = getMasterState(3); singularitySlaveAndRackManager.loadSlavesAndRacksFromMaster(state, true); MesosMasterStateObject newState = getMasterState(2); // 2 slaves, third has died singularitySlaveAndRackManager.loadSlavesAndRacksFromMaster(newState, false); List<SingularitySlave> slaves = slaveManager.getObjects(); Assert.assertEquals(3, slaves.size()); for (SingularitySlave slave : slaves) { if (slave.getId().equals("2")) { Assert.assertEquals(MachineState.DEAD, slave.getCurrentState().getState()); } else { Assert.assertEquals(MachineState.ACTIVE, slave.getCurrentState().getState()); } } }
@Test public void testValidTransitionToDecommissioned() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); scheduler.drainPendingQueue(); resourceOffers(1); SingularitySlave slave = slaveManager.getObjects().get(0); slaveResource.decommissionSlave(singularityUser, slave.getId(), new SingularityMachineChangeRequest(Optional.of(1L), Optional.absent(), Optional.absent(), Optional.of(MachineState.DECOMMISSIONED), Optional.of(true))); Assert.assertEquals(MachineState.STARTING_DECOMMISSION, slaveManager.getObjects().get(0).getCurrentState().getState()); scheduler.checkForDecomissions(); scheduler.drainPendingQueue(); Assert.assertEquals(TaskCleanupType.DECOMISSIONING, taskManager.getCleanupTasks().get(0).getCleanupType()); expiringUserActionPoller.runActionOnPoll(); Assert.assertEquals(MachineState.DECOMMISSIONED, slaveManager.getObjects().get(0).getCurrentState().getState()); Assert.assertEquals(TaskCleanupType.DECOMMISSION_TIMEOUT, taskManager.getCleanupTasks().get(0).getCleanupType()); }
Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DECOMMISSIONING); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DECOMMISSIONED);
Assert.assertEquals(StateChangeResult.FAILURE_NOT_FOUND, slaveManager.changeState("slave9231", MachineState.STARTING_DECOMMISSION, Optional.absent(), Optional.of("user1"))); Assert.assertEquals(MachineState.STARTING_DECOMMISSION, slaveManager.getObject("slave1").get().getCurrentState().getState()); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getUser().get().equals("user1")); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DECOMMISSIONING); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getUser().get().equals("user1")); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DECOMMISSIONING); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getUser().get().equals("user1")); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DECOMMISSIONED); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getUser().get().equals("user1"));
public List<SingularitySlave> getSlavesToTrackUsageFor() { List<SingularitySlave> slaves = slaveManager.getObjects(); List<SingularitySlave> slavesToTrack = new ArrayList<>(slaves.size()); for (SingularitySlave slave : slaves) { if (slave.getCurrentState().getState().isInactive() || slave.getCurrentState().getState() == MachineState.DECOMMISSIONED) { continue; } slavesToTrack.add(slave); } return slavesToTrack; }
Assert.assertTrue(rackManager.getNumObjectsAtState(MachineState.ACTIVE) == 2); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().equals(slaveManager.getHistory("slave1").get(0))); Assert.assertTrue(rackManager.getNumObjectsAtState(MachineState.DEAD) == 1); Assert.assertTrue(slaveManager.getObject("slave1").get().getCurrentState().getState() == MachineState.DEAD); Assert.assertTrue(rackManager.getObject("rack1").get().getCurrentState().getState() == MachineState.DEAD);
@Test public void testEmptyDecommissioning() { sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(StateChangeResult.SUCCESS, slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.absent(), Optional.of("user1"))); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(MachineState.DECOMMISSIONED, slaveManager.getObject("slave1").get().getCurrentState().getState()); }