@Override public InstanceInfo getInstanceInfo(final String service) throws ScalerException { return realScaler.getInstanceInfo(service); }
@Override public HealthResult healthCheck() { return realScaler.healthCheck(); } }
/** * Called the first time a ScalerThread is run. It will ensure the current number of instances does not exceed the max, and that the * minimum is also satisfied. If either of these cases are not true, it will trigger a scaling operation. * * @param instances information on the current number of instances of a service * @return the recommended action to take */ private ScalingAction handleFirstRun(final InstanceInfo instances) throws ScalerException { final ScalingAction action; if (instances.getTotalInstances() < minInstances) { action = new ScalingAction(ScalingOperation.SCALE_UP, minInstances - instances.getTotalInstances()); } else if (instances.getTotalInstances() > maxInstances) { action = new ScalingAction(ScalingOperation.SCALE_DOWN, instances.getTotalInstances() - maxInstances); } else { action = ScalingAction.NO_ACTION; } return action; }
/** * Each service to monitor, described by a ScalingConfiguration, should have a 'workload metric' which is a named * reference to a type of WorkloadAnalyser that will be used to monitor and scale the service. Here, try and acquire * the WorkloadAnalyser the ScalingConfiguration has requested from the available WorkloadAnalyserFactory objects * acquired from the classpath at boot time. * @param s the service to get a WorkloadAnalyser for * @return the WorkloadAnalyser for this service * @throws ScalerException if the requested WorkloadAnalyser is not available */ private WorkloadAnalyser getAnalyser(final ScalingConfiguration s) throws ScalerException { if ( analyserFactories.containsKey(s.getWorkloadMetric()) ) { return analyserFactories.get(s.getWorkloadMetric()).getAnalyser(s.getScalingTarget(), s.getScalingProfile()); } else { throw new ScalerException("Invalid workload metric " + s.getWorkloadMetric()); } }
private static <T extends Object> T getConfiguration(final ConfigurationSource configSource, final Class<T> type) throws ScalerException { try { return configSource.getConfiguration(type); } catch (final ConfigurationException ex) { throw new ScalerException("Unable to get configuration.", ex); } } }
@Override public ScalingAction govern(String serviceRef, ScalingAction action) { ScalingConfiguration scalingConfiguration = scalingConfigurationMap.getOrDefault(serviceRef, null); InstanceInfo lastInstanceInfo = instanceInfoMap.getOrDefault(serviceRef, null); if(scalingConfiguration==null){ throw new RuntimeException(String.format("Scaling configuration not found for {%s}", serviceRef)); } boolean otherServicesMinimumInstancesMet = otherServicesMinimumInstancesMet(serviceRef); switch(action.getOperation()){ case NONE: case SCALE_UP: { if(!otherServicesMinimumInstancesMet && lastInstanceInfo!=null){ if(lastInstanceInfo.getTotalInstances()==scalingConfiguration.getMinInstances()){ return new ScalingAction(ScalingOperation.NONE, 0); } else if(lastInstanceInfo.getTotalInstances()>scalingConfiguration.getMinInstances()){ //Gradually reduce the totalInstances by a percentage until Minimums are met. //This should be configurable, however there should be a Governor specific configuration //to allow different Governor implementations to be added without polluting the AutoscaleConfiguration int target = Math.max(scalingConfiguration.getMinInstances(), (int)Math.floor(lastInstanceInfo.getTotalInstances() * reduceToPercentage)); int amount = lastInstanceInfo.getTotalInstances() - target; return new ScalingAction(ScalingOperation.SCALE_DOWN, amount); } } break; } } return action; }
private void handleIntegers(final ScalingConfiguration sv, final Map<String, String> labels) { if ( labels.containsKey(ScalingConfiguration.KEY_INTERVAL) ) { sv.setInterval(Integer.parseInt(labels.get(ScalingConfiguration.KEY_INTERVAL))); } if ( labels.containsKey(ScalingConfiguration.KEY_MAX_INSTANCES) ) { sv.setMaxInstances(Integer.parseInt(labels.get(ScalingConfiguration.KEY_MAX_INSTANCES))); } if ( labels.containsKey(ScalingConfiguration.KEY_MIN_INSTANCES) ) { sv.setMinInstances(Integer.parseInt(labels.get(ScalingConfiguration.KEY_MIN_INSTANCES))); } if ( labels.containsKey(ScalingConfiguration.KEY_BACKOFF_AMOUNT) ) { sv.setBackoffAmount(Integer.parseInt(labels.get(ScalingConfiguration.KEY_BACKOFF_AMOUNT))); } }
/** * Perform a scale up, taking into account the maximum number of instances limitation. * * @param instances information on the current number of instances of a service * @param amount the requested number of instances to scale up by * @throws ScalerException if the scaling operation fails */ private void scaleUp(final InstanceInfo instances, final int amount) throws ScalerException { int upTarget = Math.min(maxInstances - instances.getTotalInstances(), Math.max(0, amount)); if (instances.getInstancesStaging() == 0 && upTarget > 0) { LOG.debug("Triggering scale up of service {} by amount {}", serviceRef, amount); scaler.scaleUp(serviceRef, upTarget); backoff = true; } }
private void handleStrings(final ScalingConfiguration sv, final Map<String, String> labels) { sv.setWorkloadMetric(labels.get(ScalingConfiguration.KEY_WORKLOAD_METRIC)); if ( labels.containsKey(ScalingConfiguration.KEY_SCALING_TARGET) ) { sv.setScalingTarget(labels.get(ScalingConfiguration.KEY_SCALING_TARGET)); } if ( labels.containsKey(ScalingConfiguration.KEY_SCALING_PROFILE) ) { sv.setScalingProfile(labels.get(ScalingConfiguration.KEY_SCALING_PROFILE)); } }
private boolean handleMemoryLoadIssues(final InstanceInfo instances) throws ScalerException { final double currentMemoryLoad = analyser.getCurrentMemoryLoad(); final int shutdownPriority = instances.getShutdownPriority(); if (shutdownPriority == -1) { return false; } handleAlerterDispatch(currentMemoryLoad); if (currentMemoryLoad >= resourceConfig.getResourceLimitOne() && shutdownPriority <= resourceConfig.getResourceLimitOneShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } else if (currentMemoryLoad >= resourceConfig.getResourceLimitTwo() && shutdownPriority <= resourceConfig.getResourceLimitTwoShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } else if (currentMemoryLoad >= resourceConfig.getResourceLimitThree() && shutdownPriority <= resourceConfig.getResourceLimitThreeShutdownThreshold()) { emergencyScaleDown(instances.getTotalInstances()); return true; } return false; }
private boolean validateMetric(final ScalingConfiguration s) { boolean ret = false; if ( metricNames.contains(s.getWorkloadMetric()) ) { ret = true; } else { LOG.warn("Service {} invalid: requested unknown workload metric {}", s.getId(), s.getWorkloadMetric()); } return ret; }
/** * Perform a scale down, taking into account the minimum number of instances limitation. * * @param instances information on the current number of instances of a service * @param amount the requested number of instances to scale down by * @throws ScalerException if the scaling operation fails */ private void scaleDown(final InstanceInfo instances, final int amount) throws ScalerException { int downTarget = Math.max(0, Math.min(instances.getTotalInstances() - minInstances, Math.max(0, amount))); if (downTarget > 0) { LOG.debug("Triggering scale down of service {} by amount {}", serviceRef, downTarget); scaler.scaleDown(serviceRef, downTarget); backoff = true; } }
/** * Determine if the other services have met their minimum instances. * @param serviceRef the service identifier to exclude from the evaluation * @return True if other services have met their minimum instance requirement */ private boolean otherServicesMinimumInstancesMet(String serviceRef){ for(String key:scalingConfigurationMap.keySet()){ if(key.equals(serviceRef)){ continue; } ScalingConfiguration scalingConfiguration = scalingConfigurationMap.getOrDefault(key, null); InstanceInfo lastInstanceInfo = instanceInfoMap.getOrDefault(key, null); //If there are no instance info be cautious and assume minimum instances have not been met. if(lastInstanceInfo==null || lastInstanceInfo.getTotalInstances() < scalingConfiguration.getMinInstances()){ return false; } } return true; } }
/** * Perform a scale down operation on the service bringing it to zero instances. * * @param instances information on the current number of instances of a service * @throws ScalerException if the scaling operation fails */ private void emergencyScaleDown(final int instances) throws ScalerException { LOG.info("Triggering emergency scale down of service {} to 0 due to low system resources.", serviceRef); scaler.scaleDown(serviceRef, instances); backoff = true; }
@Override public void register(ScalingConfiguration scalingConfiguration) { scalingConfigurationMap.put(scalingConfiguration.getId(), scalingConfiguration); }
/** * {@inheritDoc} * * If this instance is the master, perform scale up, otherwise ignore. */ @Override public void scaleUp(final String service, final int amount) throws ScalerException { if ( master.get() ) { realScaler.scaleUp(service, amount); } }
@Override public void run() { try { autoscaleScheduler.updateServices(source.getServices()); } catch (ScalerException e) { LOG.warn("Failed to retrieve services this run", e); } } }
private void handleAlerterDispatch(final double memLoad) throws ScalerException { if (memLoad > resourceConfig.getAlertDispatchThreshold()) { final String emailBody = analyser.getMemoryOverloadWarning(df.format(memLoad)); alertDispatcher.dispatchAlert(emailBody); } } }
@Override public AlertDispatcher getAlertDispatcher(final ConfigurationSource configs) throws ScalerException { try { return new EmailDispatcher(configs.getConfiguration(EmailDispatcherConfiguration.class)); } catch (final ConfigurationException ex) { throw new ScalerException("Unable to create email dispatcher, cannot find email dispatcher configuration.", ex); } } }
/** * {@inheritDoc} * * If this instance is the master, perform scale down, otherwise ignore. */ @Override public void scaleDown(final String service, final int amount) throws ScalerException { if ( master.get() ) { realScaler.scaleDown(service, amount); } }