/** * Converts the given list of {@link EmrClusterDefinitionConfiguration} into a list of {@link Configuration}. * * @param emrClusterDefinitionConfigurations list of {@link EmrClusterDefinitionConfiguration} * * @return list of {@link Configuration} */ protected List<Configuration> getConfigurations(List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations) { List<Configuration> configurations = null; if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) { configurations = new ArrayList<>(); for (EmrClusterDefinitionConfiguration emrClusterDefinitionConfiguration : emrClusterDefinitionConfigurations) { if (emrClusterDefinitionConfiguration != null) { Configuration configuration = new Configuration(); configuration.setClassification(emrClusterDefinitionConfiguration.getClassification()); configuration.setConfigurations(getConfigurations(emrClusterDefinitionConfiguration.getConfigurations())); configuration.setProperties(getMap(emrClusterDefinitionConfiguration.getProperties())); configurations.add(configuration); } } } return configurations; }
@Override public void terminateEmrCluster(String clusterId, boolean overrideTerminationProtection, AwsParamsDto awsParams) { emrOperations.terminateEmrCluster(getEmrClient(awsParams), clusterId, overrideTerminationProtection); }
@Override public String createEmrCluster(String clusterName, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParams) { RunJobFlowRequest runJobFlowRequest = getRunJobFlowRequest(clusterName, emrClusterDefinition); LOGGER.info("runJobFlowRequest={}", jsonHelper.objectToJson(runJobFlowRequest)); String clusterId = emrOperations.runEmrJobFlow(getEmrClient(awsParams), runJobFlowRequest); LOGGER.info("EMR cluster started. emrClusterId=\"{}\"", clusterId); return clusterId; }
RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); if (!getBootstrapActionConfigList(emrClusterDefinition).isEmpty()) runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition)); runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition)); runJobFlowRequest.setKerberosAttributes(getKerberosAttributes(emrClusterDefinition.getKerberosAttributes()));
/** * Creates an instance fleet configuration that describes the EC2 instances and instance configurations for clusters that use this feature. * * @param emrClusterDefinitionInstanceFleets the list of instance fleet configurations from the EMR cluster definition * * @return the instance fleet configuration */ protected List<InstanceFleetConfig> getInstanceFleets(List<EmrClusterDefinitionInstanceFleet> emrClusterDefinitionInstanceFleets) { List<InstanceFleetConfig> instanceFleets = null; if (!CollectionUtils.isEmpty(emrClusterDefinitionInstanceFleets)) { instanceFleets = new ArrayList<>(); for (EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet : emrClusterDefinitionInstanceFleets) { if (emrClusterDefinitionInstanceFleet != null) { InstanceFleetConfig instanceFleetConfig = new InstanceFleetConfig(); instanceFleetConfig.setName(emrClusterDefinitionInstanceFleet.getName()); instanceFleetConfig.setInstanceFleetType(emrClusterDefinitionInstanceFleet.getInstanceFleetType()); instanceFleetConfig.setTargetOnDemandCapacity(emrClusterDefinitionInstanceFleet.getTargetOnDemandCapacity()); instanceFleetConfig.setTargetSpotCapacity(emrClusterDefinitionInstanceFleet.getTargetSpotCapacity()); instanceFleetConfig.setInstanceTypeConfigs(getInstanceTypeConfigs(emrClusterDefinitionInstanceFleet.getInstanceTypeConfigs())); instanceFleetConfig.setLaunchSpecifications(getLaunchSpecifications(emrClusterDefinitionInstanceFleet.getLaunchSpecifications())); instanceFleets.add(instanceFleetConfig); } } } return instanceFleets; }
/** * Creates a list of {@link InstanceTypeConfig} from a given list of {@link EmrClusterDefinitionInstanceTypeConfig}. * * @param emrClusterDefinitionInstanceTypeConfigs the list of {@link EmrClusterDefinitionInstanceTypeConfig} * * @return the list of {@link InstanceTypeConfig} */ protected List<InstanceTypeConfig> getInstanceTypeConfigs(List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs) { List<InstanceTypeConfig> instanceTypeConfigs = null; if (!CollectionUtils.isEmpty(emrClusterDefinitionInstanceTypeConfigs)) { instanceTypeConfigs = new ArrayList<>(); for (EmrClusterDefinitionInstanceTypeConfig emrClusterDefinitionInstanceTypeConfig : emrClusterDefinitionInstanceTypeConfigs) { if (emrClusterDefinitionInstanceTypeConfig != null) { InstanceTypeConfig instanceTypeConfig = new InstanceTypeConfig(); instanceTypeConfig.setInstanceType(emrClusterDefinitionInstanceTypeConfig.getInstanceType()); instanceTypeConfig.setWeightedCapacity(emrClusterDefinitionInstanceTypeConfig.getWeightedCapacity()); instanceTypeConfig.setBidPrice(emrClusterDefinitionInstanceTypeConfig.getBidPrice()); instanceTypeConfig.setBidPriceAsPercentageOfOnDemandPrice(emrClusterDefinitionInstanceTypeConfig.getBidPriceAsPercentageOfOnDemandPrice()); instanceTypeConfig.setEbsConfiguration(getEbsConfiguration(emrClusterDefinitionInstanceTypeConfig.getEbsConfiguration())); instanceTypeConfig.setConfigurations(getConfigurations(emrClusterDefinitionInstanceTypeConfig.getConfigurations())); instanceTypeConfigs.add(instanceTypeConfig); } } } return instanceTypeConfigs; }
jobFlowInstancesConfig.setInstanceGroups(getInstanceGroupConfigs(emrClusterDefinition.getInstanceDefinitions())); jobFlowInstancesConfig.setInstanceFleets(getInstanceFleets(emrClusterDefinition.getInstanceFleets()));
@Test public void testGetEbsBlockDeviceConfigsWhenInputListContainsNullElement() { // Create a list of emrClusterDefinitionEbsBlockDeviceConfigs with a null element. List<EmrClusterDefinitionEbsBlockDeviceConfig> emrClusterDefinitionEbsBlockDeviceConfigs = new ArrayList<>(); emrClusterDefinitionEbsBlockDeviceConfigs.add(null); // Call the method under test. List<EbsBlockDeviceConfig> result = emrDaoImpl.getEbsBlockDeviceConfigs(emrClusterDefinitionEbsBlockDeviceConfigs); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new ArrayList<>(), result); }
ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(getActiveEmrClusterStates()); emrOperations.listEmrClusters(getEmrClient(awsParams), listClustersRequest.withMarker(markerForListClusters));
@Test public void testGetInstanceGroupConfigWhenBidPriceIsNull() { // Call the method under test. InstanceGroupConfig result = emrDaoImpl .getInstanceGroupConfig(InstanceRoleType.MASTER, EC2_INSTANCE_TYPE, INSTANCE_COUNT, NO_BID_PRICE, NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new InstanceGroupConfig(InstanceRoleType.MASTER, EC2_INSTANCE_TYPE, INSTANCE_COUNT), result); }
@Test public void testGetMapWhenInputListContainsNullElement() { // Create objects required for testing with a list of parameters having a null element. List<Parameter> parameters = new ArrayList<>(); parameters.add(null); // Call the method under test. Map<String, String> result = emrDaoImpl.getMap(parameters); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertNotNull(result); assertEquals(0, result.size()); }
@Test public void testGetConfigurationsWhenInputListContainsNullElement() { // Create objects required for testing with a list of emrClusterDefinitionConfigurations having a null element. List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations = new ArrayList<>(); emrClusterDefinitionConfigurations.add(null); // Call the method under test. List<Configuration> result = emrDaoImpl.getConfigurations(emrClusterDefinitionConfigurations); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new ArrayList<>(), result); }
/** * Creates an instance group configuration. * * @param roleType role type for the instance group (MASTER/CORE/TASK) * @param instanceType EC2 instance type for the instance group * @param instanceCount number of instances for the instance group * @param bidPrice bid price in case of SPOT instance request * @param emrClusterDefinitionEbsConfiguration the instance of {@link EmrClusterDefinitionEbsConfiguration} that contains EBS configurations that will be * attached to each EC2 instance in this instance group * * @return the instance group config object */ protected InstanceGroupConfig getInstanceGroupConfig(InstanceRoleType roleType, String instanceType, Integer instanceCount, BigDecimal bidPrice, EmrClusterDefinitionEbsConfiguration emrClusterDefinitionEbsConfiguration) { // Create an instance group configuration with an optional EBS configuration. InstanceGroupConfig instanceGroup = new InstanceGroupConfig(roleType, instanceType, instanceCount).withEbsConfiguration(getEbsConfiguration(emrClusterDefinitionEbsConfiguration)); // Consider spot price, if specified. if (bidPrice != null) { instanceGroup.setMarket(MarketType.SPOT); instanceGroup.setBidPrice(bidPrice.toString()); } return instanceGroup; }
@Test public void testGetInstanceGroupConfigsWhenInstanceDefinitionsObjectIsEmpty() { // Create objects required for testing. final InstanceDefinitions instanceDefinitions = new InstanceDefinitions(); // Mock the external calls. when(emrHelper.isInstanceDefinitionsEmpty(instanceDefinitions)).thenReturn(true); // Call the method under test. List<InstanceGroupConfig> result = emrDaoImpl.getInstanceGroupConfigs(instanceDefinitions); // Verify the external calls. verify(emrHelper).isInstanceDefinitionsEmpty(instanceDefinitions); verifyNoMoreInteractionsHelper(); // Validate the results. assertNull(result); }
@Test public void testGetInstanceFleetsWhenInputListContainsNullElement() { // Create objects required for testing with a list of emrClusterDefinitionInstanceFleets having a null element. List<EmrClusterDefinitionInstanceFleet> emrClusterDefinitionInstanceFleets = new ArrayList<>(); emrClusterDefinitionInstanceFleets.add(null); // Call the method under test. List<InstanceFleetConfig> result = emrDaoImpl.getInstanceFleets(emrClusterDefinitionInstanceFleets); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new ArrayList<>(), result); }
@Test public void testGetInstanceTypeConfigsWhenInputListContainsNullElement() { // Create objects required for testing with a list of emrClusterDefinitionInstanceTypeConfigs having a null element. List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs = new ArrayList<>(); emrClusterDefinitionInstanceTypeConfigs.add(null); // Call the method under test. List<InstanceTypeConfig> result = emrDaoImpl.getInstanceTypeConfigs(emrClusterDefinitionInstanceTypeConfigs); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new ArrayList<>(), result); }
@Test public void testGetLaunchSpecifications() { // Create objects required for testing. final EmrClusterDefinitionSpotSpecification emrClusterDefinitionSpotSpecification = null; final EmrClusterDefinitionLaunchSpecifications emrClusterDefinitionLaunchSpecifications = new EmrClusterDefinitionLaunchSpecifications(emrClusterDefinitionSpotSpecification); // Call the method under test. InstanceFleetProvisioningSpecifications result = emrDaoImpl.getLaunchSpecifications(emrClusterDefinitionLaunchSpecifications); // Verify the external calls. verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(new InstanceFleetProvisioningSpecifications().withSpotSpecification(null), result); }
RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); if (!getBootstrapActionConfigList(emrClusterDefinition).isEmpty()) runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition)); runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition)); runJobFlowRequest.setKerberosAttributes(getKerberosAttributes(emrClusterDefinition.getKerberosAttributes()));
/** * Creates an instance fleet configuration that describes the EC2 instances and instance configurations for clusters that use this feature. * * @param emrClusterDefinitionInstanceFleets the list of instance fleet configurations from the EMR cluster definition * * @return the instance fleet configuration */ protected List<InstanceFleetConfig> getInstanceFleets(List<EmrClusterDefinitionInstanceFleet> emrClusterDefinitionInstanceFleets) { List<InstanceFleetConfig> instanceFleets = null; if (!CollectionUtils.isEmpty(emrClusterDefinitionInstanceFleets)) { instanceFleets = new ArrayList<>(); for (EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet : emrClusterDefinitionInstanceFleets) { if (emrClusterDefinitionInstanceFleet != null) { InstanceFleetConfig instanceFleetConfig = new InstanceFleetConfig(); instanceFleetConfig.setName(emrClusterDefinitionInstanceFleet.getName()); instanceFleetConfig.setInstanceFleetType(emrClusterDefinitionInstanceFleet.getInstanceFleetType()); instanceFleetConfig.setTargetOnDemandCapacity(emrClusterDefinitionInstanceFleet.getTargetOnDemandCapacity()); instanceFleetConfig.setTargetSpotCapacity(emrClusterDefinitionInstanceFleet.getTargetSpotCapacity()); instanceFleetConfig.setInstanceTypeConfigs(getInstanceTypeConfigs(emrClusterDefinitionInstanceFleet.getInstanceTypeConfigs())); instanceFleetConfig.setLaunchSpecifications(getLaunchSpecifications(emrClusterDefinitionInstanceFleet.getLaunchSpecifications())); instanceFleets.add(instanceFleetConfig); } } } return instanceFleets; }
/** * Creates a list of {@link InstanceTypeConfig} from a given list of {@link EmrClusterDefinitionInstanceTypeConfig}. * * @param emrClusterDefinitionInstanceTypeConfigs the list of {@link EmrClusterDefinitionInstanceTypeConfig} * * @return the list of {@link InstanceTypeConfig} */ protected List<InstanceTypeConfig> getInstanceTypeConfigs(List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs) { List<InstanceTypeConfig> instanceTypeConfigs = null; if (!CollectionUtils.isEmpty(emrClusterDefinitionInstanceTypeConfigs)) { instanceTypeConfigs = new ArrayList<>(); for (EmrClusterDefinitionInstanceTypeConfig emrClusterDefinitionInstanceTypeConfig : emrClusterDefinitionInstanceTypeConfigs) { if (emrClusterDefinitionInstanceTypeConfig != null) { InstanceTypeConfig instanceTypeConfig = new InstanceTypeConfig(); instanceTypeConfig.setInstanceType(emrClusterDefinitionInstanceTypeConfig.getInstanceType()); instanceTypeConfig.setWeightedCapacity(emrClusterDefinitionInstanceTypeConfig.getWeightedCapacity()); instanceTypeConfig.setBidPrice(emrClusterDefinitionInstanceTypeConfig.getBidPrice()); instanceTypeConfig.setBidPriceAsPercentageOfOnDemandPrice(emrClusterDefinitionInstanceTypeConfig.getBidPriceAsPercentageOfOnDemandPrice()); instanceTypeConfig.setEbsConfiguration(getEbsConfiguration(emrClusterDefinitionInstanceTypeConfig.getEbsConfiguration())); instanceTypeConfig.setConfigurations(getConfigurations(emrClusterDefinitionInstanceTypeConfig.getConfigurations())); instanceTypeConfigs.add(instanceTypeConfig); } } } return instanceTypeConfigs; }