int totalInstanceCount = getTotalInstanceCount(emrClusterDefinition); List<Subnet> subnets = getSubnets(emrClusterDefinition, awsParamsDto); for (Subnet subnet : subnets) removeSubnetsWithAvailableIpsLessThan(subnets, totalInstanceCount); InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition); InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition); InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition); for (AvailabilityZone availabilityZone : getAvailabilityZones(subnets, awsParamsDto)) Map<String, BigDecimal> instanceTypeOnDemandPrices = getInstanceTypeOnDemandPrices(availabilityZone, requestedInstanceTypes); Map<String, BigDecimal> instanceTypeSpotPrices = getInstanceTypeSpotPrices(availabilityZone, requestedInstanceTypes, awsParamsDto); Ec2PriceDto masterPrice = getBestInstancePrice(masterSpotPrice, masterOnDemandPrice, masterInstanceDefinition); BigDecimal coreSpotPrice = instanceTypeSpotPrices.get(coreInstanceType); BigDecimal coreOnDemandPrice = instanceTypeOnDemandPrices.get(coreInstanceType); corePrice = getBestInstancePrice(coreSpotPrice, coreOnDemandPrice, coreInstanceDefinition); BigDecimal taskSpotPrice = instanceTypeSpotPrices.get(taskInstanceType); BigDecimal taskOnDemandPrice = instanceTypeOnDemandPrices.get(taskInstanceType); taskPrice = getBestInstancePrice(taskSpotPrice, taskOnDemandPrice, taskInstanceDefinition); emrClusterPrices.add(createEmrClusterPrice(availabilityZone, masterPrice, corePrice, taskPrice));
/** * Selects the EMR cluster pricing with the lowest core instance price. We will select one pricing randomly if there are multiple pricings that meet the * lowest core price criteria. * <p> * Returns null if the given list is empty * * @param emrClusterPrices the list of pricing to select from * * @return the pricing with the lowest core price */ EmrClusterPriceDto getEmrClusterPriceWithLowestCoreInstancePrice(final List<EmrClusterPriceDto> emrClusterPrices) { final List<EmrClusterPriceDto> lowestCoreInstancePriceEmrClusters = getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(emrClusterPrices, configurationHelper.getNonNegativeBigDecimalRequiredProperty(ConfigurationValue.EMR_CLUSTER_LOWEST_CORE_INSTANCE_PRICE_PERCENTAGE)); if (!lowestCoreInstancePriceEmrClusters.isEmpty()) { // Pick one randomly from the lowest core instance price list final EmrClusterPriceDto selectedEmrClusterPriceDto = lowestCoreInstancePriceEmrClusters.get(new Random().nextInt(lowestCoreInstancePriceEmrClusters.size())); // Log the selected pricing as well as the pricing list LOGGER.info("selectedEmrCluster={} from lowestCoreInstancePriceEmrClusters={}", jsonHelper.objectToJson(selectedEmrClusterPriceDto), jsonHelper.objectToJson(lowestCoreInstancePriceEmrClusters)); return selectedEmrClusterPriceDto; } else { return null; } }
for (final EmrClusterPriceDto emrClusterPriceDto : emrClusterPrices) final BigDecimal coreInstancePrice = getEmrClusterCoreInstancePrice(emrClusterPriceDto); if (emrClusterPriceMapKeyedByCoreInstancePrice.containsKey(coreInstancePrice))
/** * Returns the total number of requested instances. Returns the sum of master, core, and task instance counts. Task instance is optional. * * @param emrClusterDefinition the EMR cluster definition containing the instance definitions * * @return the total instance count */ private int getTotalInstanceCount(EmrClusterDefinition emrClusterDefinition) { InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition); InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition); InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition); // Get total count of instances this definition will attempt to create int totalInstanceCount = masterInstanceDefinition.getInstanceCount(); if (coreInstanceDefinition != null) { totalInstanceCount += coreInstanceDefinition.getInstanceCount(); } if (taskInstanceDefinition != null) { totalInstanceCount += taskInstanceDefinition.getInstanceCount(); } return totalInstanceCount; }
verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); verify(emrHelper).buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName());
@Test public void testGetEmrClusterLowestCoreInstancePriceEmptyPricingList() { assertNull(emrPricingHelper.getEmrClusterPriceWithLowestCoreInstancePrice(Collections.emptyList())); }
verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); verify(emrHelper).buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName());
/** * Returns the total number of requested instances. Returns the sum of master, core, and task instance counts. Task instance is optional. * * @param emrClusterDefinition the EMR cluster definition containing the instance definitions * * @return the total instance count */ private int getTotalInstanceCount(EmrClusterDefinition emrClusterDefinition) { InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition); InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition); InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition); // Get total count of instances this definition will attempt to create int totalInstanceCount = masterInstanceDefinition.getInstanceCount(); if (coreInstanceDefinition != null) { totalInstanceCount += coreInstanceDefinition.getInstanceCount(); } if (taskInstanceDefinition != null) { totalInstanceCount += taskInstanceDefinition.getInstanceCount(); } return totalInstanceCount; }
int totalInstanceCount = getTotalInstanceCount(emrClusterDefinition); List<Subnet> subnets = getSubnets(emrClusterDefinition, awsParamsDto); for (Subnet subnet : subnets) removeSubnetsWithAvailableIpsLessThan(subnets, totalInstanceCount); InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition); InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition); InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition); for (AvailabilityZone availabilityZone : getAvailabilityZones(subnets, awsParamsDto)) Map<String, BigDecimal> instanceTypeOnDemandPrices = getInstanceTypeOnDemandPrices(availabilityZone, requestedInstanceTypes); Map<String, BigDecimal> instanceTypeSpotPrices = getInstanceTypeSpotPrices(availabilityZone, requestedInstanceTypes, awsParamsDto); Ec2PriceDto masterPrice = getBestInstancePrice(masterSpotPrice, masterOnDemandPrice, masterInstanceDefinition); BigDecimal coreSpotPrice = instanceTypeSpotPrices.get(coreInstanceType); BigDecimal coreOnDemandPrice = instanceTypeOnDemandPrices.get(coreInstanceType); corePrice = getBestInstancePrice(coreSpotPrice, coreOnDemandPrice, coreInstanceDefinition); BigDecimal taskSpotPrice = instanceTypeSpotPrices.get(taskInstanceType); BigDecimal taskOnDemandPrice = instanceTypeOnDemandPrices.get(taskInstanceType); taskPrice = getBestInstancePrice(taskSpotPrice, taskOnDemandPrice, taskInstanceDefinition); emrClusterPrices.add(createEmrClusterPrice(availabilityZone, masterPrice, corePrice, taskPrice));
verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); verify(emrHelper).buildEmrClusterName(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName());
/** * Selects the EMR cluster pricing with the lowest core instance price. We will select one pricing randomly if there are multiple pricings that meet the * lowest core price criteria. * <p> * Returns null if the given list is empty * * @param emrClusterPrices the list of pricing to select from * * @return the pricing with the lowest core price */ EmrClusterPriceDto getEmrClusterPriceWithLowestCoreInstancePrice(final List<EmrClusterPriceDto> emrClusterPrices) { final List<EmrClusterPriceDto> lowestCoreInstancePriceEmrClusters = getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(emrClusterPrices, configurationHelper.getNonNegativeBigDecimalRequiredProperty(ConfigurationValue.EMR_CLUSTER_LOWEST_CORE_INSTANCE_PRICE_PERCENTAGE)); if (!lowestCoreInstancePriceEmrClusters.isEmpty()) { // Pick one randomly from the lowest core instance price list final EmrClusterPriceDto selectedEmrClusterPriceDto = lowestCoreInstancePriceEmrClusters.get(new Random().nextInt(lowestCoreInstancePriceEmrClusters.size())); // Log the selected pricing as well as the pricing list LOGGER.info("selectedEmrCluster={} from lowestCoreInstancePriceEmrClusters={}", jsonHelper.objectToJson(selectedEmrClusterPriceDto), jsonHelper.objectToJson(lowestCoreInstancePriceEmrClusters)); return selectedEmrClusterPriceDto; } else { return null; } }
for (final EmrClusterPriceDto emrClusterPriceDto : emrClusterPrices) final BigDecimal coreInstancePrice = getEmrClusterCoreInstancePrice(emrClusterPriceDto); if (emrClusterPriceMapKeyedByCoreInstancePrice.containsKey(coreInstancePrice))
@Test public void testEmrCreateClusterAwsSpecificStepsImplDryRun() { // Create an AWS params DTO AwsParamsDto awsParamsDto = new AwsParamsDto(); // Create an EMR cluster definition object EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setAccountId(AWS_ACCOUNT_ID); emrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions()); // Create an EMR cluster create request EmrClusterCreateRequest emrClusterCreateRequest = new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, DRY_RUN, emrClusterDefinition); emrClusterCreateRequest.setEmrClusterDefinitionOverride(emrClusterDefinition); // Create an EMR cluster alternate key DTO EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME); // Create a cluster summary object ClusterSummary clusterSummary = new ClusterSummary(); clusterSummary.setId(EMR_CLUSTER_ID); // Mock the external calls. when(emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId())).thenReturn(awsParamsDto); when(emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())).thenReturn(false); // Call the method under test. emrHelperServiceImpl.emrCreateClusterAwsSpecificSteps(emrClusterCreateRequest, emrClusterDefinition, emrClusterAlternateKeyDto); // Verify the external calls. verify(emrHelper).getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId()); verify(emrHelper).isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions()); verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); verifyNoMoreInteractionsHelper(); }
@Test public void testGetEmrClusterPricesWithinLowestCoreInstancePriceThresholdSinglePricing() throws Exception { List<EmrClusterPriceDto> pricingList = Arrays.asList(createSimpleEmrClusterPrice(AVAILABILITY_ZONE_1, BigDecimal.ONE)); List<EmrClusterPriceDto> lowestCoreInstancePriceClusters = emrPricingHelper.getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(pricingList, TEN_PERCENT); assertEquals(1, lowestCoreInstancePriceClusters.size()); assertEquals(AVAILABILITY_ZONE_1, lowestCoreInstancePriceClusters.get(0).getAvailabilityZone()); }
emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
/** * Tests when one cluster does not have core instance. In this case this cluster will be picked since the price for the cluster is now zero (the lowest) * * @throws Exception */ @Test public void testGetEmrClusterPricesWithinLowestCoreInstancePriceEmptyCoreInstanceMultiplePricings() throws Exception { List<EmrClusterPriceDto> pricingList = Arrays.asList(createSimpleEmrClusterPrice(AVAILABILITY_ZONE_1, BigDecimal.ONE), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_4, null)); List<EmrClusterPriceDto> lowestCoreInstancePriceClusters = emrPricingHelper.getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(pricingList, TEN_PERCENT); assertEquals(1, lowestCoreInstancePriceClusters.size()); for (EmrClusterPriceDto emrClusterPriceDto : lowestCoreInstancePriceClusters) { assertTrue(Arrays.asList(AVAILABILITY_ZONE_4).contains(emrClusterPriceDto.getAvailabilityZone())); } }
emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
@Test public void testGetEmrClusterPricesWithinLowestCoreInstancePriceThresholdMultiplePricing() throws Exception { List<EmrClusterPriceDto> pricingList = Arrays .asList(createSimpleEmrClusterPrice(AVAILABILITY_ZONE_1, BigDecimal.ONE), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_2, BigDecimal.TEN), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_3, ONE_POINT_ONE), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_4, ONE_POINT_ONE_ONE)); List<EmrClusterPriceDto> lowestCoreInstancePriceClusters = emrPricingHelper.getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(pricingList, TEN_PERCENT); assertEquals(2, lowestCoreInstancePriceClusters.size()); for (EmrClusterPriceDto emrClusterPriceDto : lowestCoreInstancePriceClusters) { assertTrue(Arrays.asList(AVAILABILITY_ZONE_1, AVAILABILITY_ZONE_3).contains(emrClusterPriceDto.getAvailabilityZone())); } }
/** * Creates a new EMR cluster definition using the specified parameters, updates it with best price algorithm, and returns the definition. * * @param subnetId Subnet ID. Optional. * @param masterInstanceDefinition The master instance definition * @param coreInstanceDefinition The core instance definition * @param taskInstanceDefinition The task instance definition. Optional. * * @return Updated EMR cluster definition. */ private EmrClusterDefinition updateEmrClusterDefinitionWithBestPrice(String subnetId, MasterInstanceDefinition masterInstanceDefinition, InstanceDefinition coreInstanceDefinition, InstanceDefinition taskInstanceDefinition) { EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setSubnetId(subnetId); InstanceDefinitions instanceDefinitions = new InstanceDefinitions(); instanceDefinitions.setMasterInstances(masterInstanceDefinition); instanceDefinitions.setCoreInstances(coreInstanceDefinition); instanceDefinitions.setTaskInstances(taskInstanceDefinition); emrClusterDefinition.setInstanceDefinitions(instanceDefinitions); emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(new EmrClusterAlternateKeyDto(), emrClusterDefinition, new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, NO_HTTP_PROXY_HOST, NO_HTTP_PROXY_PORT, AWS_REGION_NAME_US_EAST_1)); return emrClusterDefinition; } }
/** * Tests when the threshold is set to zero. * * @throws Exception */ @Test public void testGetEmrClusterPricesWithinLowestCoreInstancePriceZeroThresholdMultiplePricings() throws Exception { List<EmrClusterPriceDto> pricingList = Arrays .asList(createSimpleEmrClusterPrice(AVAILABILITY_ZONE_1, BigDecimal.ONE), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_2, BigDecimal.TEN), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_3, BigDecimal.ONE.add(FIVE_UNIT)), createSimpleEmrClusterPrice(AVAILABILITY_ZONE_4, BigDecimal.ONE)); List<EmrClusterPriceDto> lowestCoreInstancePriceClusters = emrPricingHelper.getEmrClusterPricesWithinLowestCoreInstancePriceThreshold(pricingList, BigDecimal.ZERO); assertEquals(2, lowestCoreInstancePriceClusters.size()); for (EmrClusterPriceDto emrClusterPriceDto : lowestCoreInstancePriceClusters) { assertTrue(Arrays.asList(AVAILABILITY_ZONE_1, AVAILABILITY_ZONE_4).contains(emrClusterPriceDto.getAvailabilityZone())); } }