/** * Called the first time a ScalerThread is run. It will ensure the current number of instances does not exceed the max, and that the * minimum is also satisfied. If either of these cases are not true, it will trigger a scaling operation. * * @param instances information on the current number of instances of a service * @return the recommended action to take */ private ScalingAction handleFirstRun(final InstanceInfo instances) throws ScalerException { final ScalingAction action; if (instances.getTotalInstances() < minInstances) { action = new ScalingAction(ScalingOperation.SCALE_UP, minInstances - instances.getTotalInstances()); } else if (instances.getTotalInstances() > maxInstances) { action = new ScalingAction(ScalingOperation.SCALE_DOWN, instances.getTotalInstances() - maxInstances); } else { action = ScalingAction.NO_ACTION; } return action; }
/** * Perform a scale down, taking into account the minimum number of instances limitation. * * @param instances information on the current number of instances of a service * @param amount the requested number of instances to scale down by * @throws ScalerException if the scaling operation fails */ private void scaleDown(final InstanceInfo instances, final int amount) throws ScalerException { int downTarget = Math.max(0, Math.min(instances.getTotalInstances() - minInstances, Math.max(0, amount))); if (downTarget > 0) { LOG.debug("Triggering scale down of service {} by amount {}", serviceRef, downTarget); scaler.scaleDown(serviceRef, downTarget); backoff = true; } }
/** * Determine if the other services have met their minimum instances. * @param serviceRef the service identifier to exclude from the evaluation * @return True if other services have met their minimum instance requirement */ private boolean otherServicesMinimumInstancesMet(String serviceRef){ for(String key:scalingConfigurationMap.keySet()){ if(key.equals(serviceRef)){ continue; } ScalingConfiguration scalingConfiguration = scalingConfigurationMap.getOrDefault(key, null); InstanceInfo lastInstanceInfo = instanceInfoMap.getOrDefault(key, null); //If there are no instance info be cautious and assume minimum instances have not been met. if(lastInstanceInfo==null || lastInstanceInfo.getTotalInstances() < scalingConfiguration.getMinInstances()){ return false; } } return true; } }
/** * Perform a scale up, taking into account the maximum number of instances limitation. * * @param instances information on the current number of instances of a service * @param amount the requested number of instances to scale up by * @throws ScalerException if the scaling operation fails */ private void scaleUp(final InstanceInfo instances, final int amount) throws ScalerException { int upTarget = Math.min(maxInstances - instances.getTotalInstances(), Math.max(0, amount)); if (instances.getInstancesStaging() == 0 && upTarget > 0) { LOG.debug("Triggering scale up of service {} by amount {}", serviceRef, amount); scaler.scaleUp(serviceRef, upTarget); backoff = true; } }
@Override public ScalingAction govern(String serviceRef, ScalingAction action) { ScalingConfiguration scalingConfiguration = scalingConfigurationMap.getOrDefault(serviceRef, null); InstanceInfo lastInstanceInfo = instanceInfoMap.getOrDefault(serviceRef, null); if(scalingConfiguration==null){ throw new RuntimeException(String.format("Scaling configuration not found for {%s}", serviceRef)); } boolean otherServicesMinimumInstancesMet = otherServicesMinimumInstancesMet(serviceRef); switch(action.getOperation()){ case NONE: case SCALE_UP: { if(!otherServicesMinimumInstancesMet && lastInstanceInfo!=null){ if(lastInstanceInfo.getTotalInstances()==scalingConfiguration.getMinInstances()){ return new ScalingAction(ScalingOperation.NONE, 0); } else if(lastInstanceInfo.getTotalInstances()>scalingConfiguration.getMinInstances()){ //Gradually reduce the totalInstances by a percentage until Minimums are met. //This should be configurable, however there should be a Governor specific configuration //to allow different Governor implementations to be added without polluting the AutoscaleConfiguration int target = Math.max(scalingConfiguration.getMinInstances(), (int)Math.floor(lastInstanceInfo.getTotalInstances() * reduceToPercentage)); int amount = lastInstanceInfo.getTotalInstances() - target; return new ScalingAction(ScalingOperation.SCALE_DOWN, amount); } } break; } } return action; }
private boolean handleMemoryLoadIssues(final InstanceInfo instances) throws ScalerException { final double currentMemoryLoad = analyser.getCurrentMemoryLoad(); final int shutdownPriority = instances.getShutdownPriority(); if (shutdownPriority == -1) { return false; } handleAlerterDispatch(currentMemoryLoad); if (currentMemoryLoad >= resourceConfig.getResourceLimitOne() && shutdownPriority <= resourceConfig.getResourceLimitOneShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } else if (currentMemoryLoad >= resourceConfig.getResourceLimitTwo() && shutdownPriority <= resourceConfig.getResourceLimitTwoShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } else if (currentMemoryLoad >= resourceConfig.getResourceLimitThree() && shutdownPriority <= resourceConfig.getResourceLimitThreeShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } return false; }