if (emrClusterDefinitionOverride.getReleaseLabel() != null) emrClusterDefinition.setReleaseLabel(emrClusterDefinitionOverride.getReleaseLabel()); if (emrClusterDefinitionOverride.getApplications() != null) emrClusterDefinition.setApplications(emrClusterDefinitionOverride.getApplications()); if (emrClusterDefinitionOverride.getConfigurations() != null) emrClusterDefinition.setConfigurations(emrClusterDefinitionOverride.getConfigurations()); if (emrClusterDefinitionOverride.getSshKeyPairName() != null) emrClusterDefinition.setSshKeyPairName(emrClusterDefinitionOverride.getSshKeyPairName()); if (emrClusterDefinitionOverride.getSubnetId() != null) emrClusterDefinition.setSubnetId(emrClusterDefinitionOverride.getSubnetId()); if (emrClusterDefinitionOverride.getLogBucket() != null) emrClusterDefinition.setLogBucket(emrClusterDefinitionOverride.getLogBucket()); if (emrClusterDefinitionOverride.isKeepAlive() != null) emrClusterDefinition.setKeepAlive(emrClusterDefinitionOverride.isKeepAlive()); if (emrClusterDefinitionOverride.isVisibleToAll() != null)
/** * Create an instance of {@link EmrClusterDefinition } * */ public EmrClusterDefinition createEmrClusterDefinition() { return new EmrClusterDefinition(); }
jobFlowInstancesConfig.setEmrManagedMasterSecurityGroup(emrClusterDefinition.getMasterSecurityGroup()); jobFlowInstancesConfig.setEmrManagedSlaveSecurityGroup(emrClusterDefinition.getSlaveSecurityGroup()); jobFlowInstancesConfig.setAdditionalMasterSecurityGroups(emrClusterDefinition.getAdditionalMasterSecurityGroups()); jobFlowInstancesConfig.setAdditionalSlaveSecurityGroups(emrClusterDefinition.getAdditionalSlaveSecurityGroups()); if (StringUtils.isNotBlank(emrClusterDefinition.getSshKeyPairName())) jobFlowInstancesConfig.setEc2KeyName(emrClusterDefinition.getSshKeyPairName()); jobFlowInstancesConfig.setInstanceGroups(getInstanceGroupConfigs(emrClusterDefinition.getInstanceDefinitions())); jobFlowInstancesConfig.setInstanceFleets(getInstanceFleets(emrClusterDefinition.getInstanceFleets())); if (StringUtils.isNotBlank(emrClusterDefinition.getSubnetId())) jobFlowInstancesConfig.setEc2SubnetIds(herdStringHelper.splitAndTrim(emrClusterDefinition.getSubnetId(), ",")); jobFlowInstancesConfig.setEc2SubnetId(emrClusterDefinition.getSubnetId()); if (emrClusterDefinition.isKeepAlive() != null) jobFlowInstancesConfig.setKeepJobFlowAliveWhenNoSteps(emrClusterDefinition.isKeepAlive()); if (emrClusterDefinition.isTerminationProtection() != null) jobFlowInstancesConfig.setTerminationProtected(emrClusterDefinition.isTerminationProtection());
Assert.isTrue(StringUtils.isNotBlank(emrClusterDefinition.getSubnetId()), "Subnet ID must be specified"); for (String token : emrClusterDefinition.getSubnetId().split(",")) Assert.isTrue(!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions()) || CollectionUtils.isNotEmpty(emrClusterDefinition.getInstanceFleets()), "Instance group definitions or instance fleets must be specified."); if (!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())) Assert.notNull(emrClusterDefinition.getInstanceDefinitions().getMasterInstances(), "Master instances must be specified."); validateMasterInstanceDefinition(emrClusterDefinition.getInstanceDefinitions().getMasterInstances()); if (emrClusterDefinition.getInstanceDefinitions().getCoreInstances() != null) validateInstanceDefinition("core", emrClusterDefinition.getInstanceDefinitions().getCoreInstances(), 0); if (emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceCount() <= 0) emrClusterDefinition.getInstanceDefinitions().setCoreInstances(null); if (emrClusterDefinition.getInstanceDefinitions().getTaskInstances() != null) validateInstanceDefinition("task", emrClusterDefinition.getInstanceDefinitions().getTaskInstances(), 1); int instancesRequested = emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceCount(); if (emrClusterDefinition.getInstanceDefinitions().getCoreInstances() != null) instancesRequested += emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceCount(); if (emrClusterDefinition.getInstanceDefinitions().getTaskInstances() != null)
theSshKeyPairName = this.getSshKeyPairName(); strategy.appendField(locator, this, "sshKeyPairName", buffer, theSshKeyPairName, (this.sshKeyPairName!= null)); theSubnetId = this.getSubnetId(); strategy.appendField(locator, this, "subnetId", buffer, theSubnetId, (this.subnetId!= null)); theLogBucket = this.getLogBucket(); strategy.appendField(locator, this, "logBucket", buffer, theLogBucket, (this.logBucket!= null)); theKeepAlive = this.isKeepAlive(); strategy.appendField(locator, this, "keepAlive", buffer, theKeepAlive, (this.keepAlive!= null)); theVisibleToAll = this.isVisibleToAll(); strategy.appendField(locator, this, "visibleToAll", buffer, theVisibleToAll, (this.visibleToAll!= null)); theTerminationProtection = this.isTerminationProtection(); strategy.appendField(locator, this, "terminationProtection", buffer, theTerminationProtection, (this.terminationProtection!= null)); theEncryptionEnabled = this.isEncryptionEnabled(); strategy.appendField(locator, this, "encryptionEnabled", buffer, theEncryptionEnabled, (this.encryptionEnabled!= null)); theAccountId = this.getAccountId(); strategy.appendField(locator, this, "accountId", buffer, theAccountId, (this.accountId!= null)); theServiceIamRole = this.getServiceIamRole(); strategy.appendField(locator, this, "serviceIamRole", buffer, theServiceIamRole, (this.serviceIamRole!= null)); theEc2NodeIamProfileName = this.getEc2NodeIamProfileName(); strategy.appendField(locator, this, "ec2NodeIamProfileName", buffer, theEc2NodeIamProfileName, (this.ec2NodeIamProfileName!= null));
EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); emrClusterDefinitionOverride.setReleaseLabel("4.8.0." + Math.random()); // Security configuration is not supported prior to EMR version 4.8.0. expectedEmrClusterDefinition.setReleaseLabel(emrClusterDefinitionOverride.getReleaseLabel()); emrClusterDefinitionOverride.setAdditionalInfo("test" + Math.random()); expectedEmrClusterDefinition.setAdditionalInfo(emrClusterDefinitionOverride.getAdditionalInfo()); emrClusterDefinitionOverride.setAmiVersion("test" + Math.random()); expectedEmrClusterDefinition.setAmiVersion(emrClusterDefinitionOverride.getAmiVersion()); emrClusterDefinitionOverride.setEc2NodeIamProfileName("test" + Math.random()); expectedEmrClusterDefinition.setEc2NodeIamProfileName(emrClusterDefinitionOverride.getEc2NodeIamProfileName()); emrClusterDefinitionOverride.setEncryptionEnabled(!expectedEmrClusterDefinition.isEncryptionEnabled()); expectedEmrClusterDefinition.setEncryptionEnabled(emrClusterDefinitionOverride.isEncryptionEnabled()); emrClusterDefinitionOverride.setHadoopVersion("test" + Math.random()); expectedEmrClusterDefinition.setHadoopVersion(emrClusterDefinitionOverride.getHadoopVersion()); emrClusterDefinitionOverride.setHiveVersion("test" + Math.random()); expectedEmrClusterDefinition.setHiveVersion(emrClusterDefinitionOverride.getHiveVersion()); emrClusterDefinitionOverride.setInstallOozie(!expectedEmrClusterDefinition.isInstallOozie()); expectedEmrClusterDefinition.setInstallOozie(emrClusterDefinitionOverride.isInstallOozie()); emrClusterDefinitionOverride.setKeepAlive(!expectedEmrClusterDefinition.isKeepAlive()); expectedEmrClusterDefinition.setKeepAlive(emrClusterDefinitionOverride.isKeepAlive()); emrClusterDefinitionOverride.setLogBucket("test" + Math.random()); expectedEmrClusterDefinition.setLogBucket(emrClusterDefinitionOverride.getLogBucket()); emrClusterDefinitionOverride.setPigVersion("test" + Math.random()); expectedEmrClusterDefinition.setPigVersion(emrClusterDefinitionOverride.getPigVersion()); emrClusterDefinitionOverride.setServiceIamRole("test" + Math.random()); expectedEmrClusterDefinition.setServiceIamRole(emrClusterDefinitionOverride.getServiceIamRole()); emrClusterDefinitionOverride.setSshKeyPairName("test" + Math.random()); expectedEmrClusterDefinition.setSshKeyPairName(emrClusterDefinitionOverride.getSshKeyPairName()); emrClusterDefinitionOverride.setSubnetId(MockEc2OperationsImpl.SUBNET_1); expectedEmrClusterDefinition.setSubnetId(emrClusterDefinitionOverride.getSubnetId());
EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); InstanceDefinitions instanceDefinitions = new InstanceDefinitions(); instanceDefinitions.setMasterInstances( new InstanceDefinition(30, "taskInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD)); emrClusterDefinition.setInstanceDefinitions(instanceDefinitions); emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue"), new NodeTag("", "tagValue"), new NodeTag("tagName", ""))); emrClusterDefinition.setSshKeyPairName("sshKeyPairName"); emrClusterDefinition.setSubnetId("subnetId"); emrClusterDefinition.setKeepAlive(true); emrClusterDefinition.setTerminationProtection(true); emrClusterDefinition.setHadoopVersion("hadoopVersion"); emrClusterDefinition.setReleaseLabel("releaseLabel"); emrClusterDefinition.setApplications(new ArrayList<>()); emrClusterDefinitionApplication.setVersion("applicationVersion1"); emrClusterDefinitionApplication.setArgs(Arrays.asList("applicationArg1")); emrClusterDefinition.getApplications().add(emrClusterDefinitionApplication); emrClusterDefinitionApplication .setAdditionalInfoList(Arrays.asList(new Parameter("applicationAdditionalInfoName2", "applicationAdditionalInfoValue2"))); emrClusterDefinition.getApplications().add(emrClusterDefinitionApplication); emrClusterDefinition.setConfigurations(new ArrayList<>()); emrClusterDefinitionConfiguration.setConfigurations(Arrays.asList(emrClusterDefinitionConfigurationInner));
EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); emrClusterDefinitionOverride.setApplications(Collections.<EmrClusterDefinitionApplication>emptyList()); expectedEmrClusterDefinition.setApplications(emrClusterDefinitionOverride.getApplications()); emrClusterDefinitionOverride.setConfigurations(Collections.<EmrClusterDefinitionConfiguration>emptyList()); expectedEmrClusterDefinition.setConfigurations(emrClusterDefinitionOverride.getConfigurations()); emrClusterDefinitionOverride.setCustomBootstrapActionAll(Collections.<ScriptDefinition>emptyList()); expectedEmrClusterDefinition.setCustomBootstrapActionAll(emrClusterDefinitionOverride.getCustomBootstrapActionAll()); emrClusterDefinitionOverride.setCustomBootstrapActionMaster(Collections.<ScriptDefinition>emptyList()); expectedEmrClusterDefinition.setCustomBootstrapActionMaster(emrClusterDefinitionOverride.getCustomBootstrapActionMaster()); emrClusterDefinitionOverride.setDaemonConfigurations(Collections.<Parameter>emptyList()); expectedEmrClusterDefinition.setDaemonConfigurations(emrClusterDefinitionOverride.getDaemonConfigurations()); emrClusterDefinitionOverride.setHadoopConfigurations(Collections.<Serializable>emptyList()); expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations()); emrClusterDefinitionOverride.setHadoopJarSteps(Collections.<HadoopJarStep>emptyList()); expectedEmrClusterDefinition.setHadoopJarSteps(emrClusterDefinitionOverride.getHadoopJarSteps()); emrClusterDefinitionOverride.setAdditionalMasterSecurityGroups(Collections.emptyList()); expectedEmrClusterDefinition.setAdditionalMasterSecurityGroups(emrClusterDefinitionOverride.getAdditionalMasterSecurityGroups()); emrClusterDefinitionOverride.setAdditionalSlaveSecurityGroups(Collections.emptyList()); expectedEmrClusterDefinition.setAdditionalSlaveSecurityGroups(emrClusterDefinitionOverride.getAdditionalSlaveSecurityGroups()); List<NodeTag> nodeTags = expectedEmrClusterDefinition.getNodeTags(); nodeTags.add(new NodeTag("testTag", "test")); emrClusterDefinitionOverride.setNodeTags(nodeTags); expectedEmrClusterDefinition.setNodeTags(nodeTags); request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
if (StringUtils.isNotBlank(emrClusterDefinition.getReleaseLabel())) runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel()); List<EmrClusterDefinitionApplication> emrClusterDefinitionApplications = emrClusterDefinition.getApplications(); if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations = emrClusterDefinition.getConfigurations(); if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) if (StringUtils.isNotBlank(emrClusterDefinition.getLogBucket())) runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket()); if (emrClusterDefinition.isVisibleToAll() != null) runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll()); if (StringUtils.isNotBlank(emrClusterDefinition.getEc2NodeIamProfileName())) runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName()); if (StringUtils.isNotBlank(emrClusterDefinition.getServiceIamRole())) runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole()); if (StringUtils.isNotBlank(emrClusterDefinition.getAmiVersion())) runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion());
/** * This method tests the blank values for various parameters */ @Test public void testCreateEmrClusterBlankParams() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()); EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml); List<ScriptDefinition> scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionAll(); scriptDefinitions.get(0).setScriptArguments(null); emrClusterDefinition.setCustomBootstrapActionAll(scriptDefinitions); scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionMaster(); scriptDefinitions.get(0).setScriptArguments(null); emrClusterDefinition.setCustomBootstrapActionMaster(scriptDefinitions); emrClusterDefinition.setVisibleToAll(null); emrClusterDefinition.setServiceIamRole(null); emrClusterDefinition.setAmiVersion(null); emrClusterDefinition.setServiceIamRole(emrClusterDefinition.getEc2NodeIamProfileName()); emrClusterDefinition.setScaleDownBehavior(null); emrClusterDefinition.setKerberosAttributes(null); configXml = xmlHelper.objectToXml(emrClusterDefinition); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml); // Create a new EMR cluster create request EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); }
@Test public void testCreateEmrClusterOverrideExistingCoreInstanceToNullAssertSuccess() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()); EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml); emrClusterDefinition.getInstanceDefinitions().getCoreInstances().setInstanceCount(1); emrClusterDefinitionDaoTestHelper .createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, xmlHelper.objectToXml(emrClusterDefinition)); // Create a new EMR cluster create request EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); emrClusterDefinitionOverride.setInstanceDefinitions(emrClusterDefinition.getInstanceDefinitions()); emrClusterDefinitionOverride.getInstanceDefinitions().setCoreInstances(null); request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride); EmrCluster emrClusterCreateResponse = emrService.createCluster(request); assertNull(emrClusterCreateResponse.getEmrClusterDefinition().getInstanceDefinitions().getCoreInstances()); }
@Test public void testEmrCreateClusterAwsSpecificStepsImplDryRun() { // Create an AWS params DTO AwsParamsDto awsParamsDto = new AwsParamsDto(); // Create an EMR cluster definition object EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setAccountId(AWS_ACCOUNT_ID); emrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions()); // Create an EMR cluster create request EmrClusterCreateRequest emrClusterCreateRequest = new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, DRY_RUN, emrClusterDefinition); emrClusterCreateRequest.setEmrClusterDefinitionOverride(emrClusterDefinition); // Create an EMR cluster alternate key DTO EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME); // Create a cluster summary object ClusterSummary clusterSummary = new ClusterSummary(); clusterSummary.setId(EMR_CLUSTER_ID); // Mock the external calls. when(emrHelper.getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId())).thenReturn(awsParamsDto); when(emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())).thenReturn(false); // Call the method under test. emrHelperServiceImpl.emrCreateClusterAwsSpecificSteps(emrClusterCreateRequest, emrClusterDefinition, emrClusterAlternateKeyDto); // Verify the external calls. verify(emrHelper).getAwsParamsDtoByAccountId(emrClusterDefinition.getAccountId()); verify(emrHelper).isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions()); verify(emrPricingHelper).updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto); verifyNoMoreInteractionsHelper(); }
emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice()); assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice()); assertEquals("task instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getTaskInstances().getInstanceSpotPrice()); SUBNET_1.equals(emrClusterDefinition.getSubnetId()) || SUBNET_4.equals(emrClusterDefinition.getSubnetId()));
/** * Returns the task instance definition. Returns null if no task definition is specified. * * @param emrClusterDefinition the EMR cluster definition * * @return the task instance definition, or null */ private InstanceDefinition getTaskInstanceDefinition(EmrClusterDefinition emrClusterDefinition) { return emrClusterDefinition.getInstanceDefinitions().getTaskInstances(); }
EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setInstanceFleets(Arrays.asList(emrClusterDefinitionInstanceFleet)); emrClusterDefinition.setSubnetId(String.format("%s , %s ", EC2_SUBNET, EC2_SUBNET_2)); emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setSubnetId(MockEc2OperationsImpl.SUBNET_1); InstanceDefinitions instanceDefinitions = new InstanceDefinitions(); instanceDefinitions.setTaskInstances(taskInstanceDefinition); emrClusterDefinition.setInstanceDefinitions(instanceDefinitions); emrClusterDefinition.setNodeTags(nodeTags);
/** * Creates a new EMR cluster definition using the specified parameters, updates it with best price algorithm, and returns the definition. * * @param subnetId Subnet ID. Optional. * @param masterInstanceDefinition The master instance definition * @param coreInstanceDefinition The core instance definition * @param taskInstanceDefinition The task instance definition. Optional. * * @return Updated EMR cluster definition. */ private EmrClusterDefinition updateEmrClusterDefinitionWithBestPrice(String subnetId, MasterInstanceDefinition masterInstanceDefinition, InstanceDefinition coreInstanceDefinition, InstanceDefinition taskInstanceDefinition) { EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition(); emrClusterDefinition.setSubnetId(subnetId); InstanceDefinitions instanceDefinitions = new InstanceDefinitions(); instanceDefinitions.setMasterInstances(masterInstanceDefinition); instanceDefinitions.setCoreInstances(coreInstanceDefinition); instanceDefinitions.setTaskInstances(taskInstanceDefinition); emrClusterDefinition.setInstanceDefinitions(instanceDefinitions); emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(new EmrClusterAlternateKeyDto(), emrClusterDefinition, new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, NO_HTTP_PROXY_HOST, NO_HTTP_PROXY_PORT, AWS_REGION_NAME_US_EAST_1)); return emrClusterDefinition; } }
@Test public void testCreateEmrClusterOverrideSecurityGroups() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()); EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml); expectedEmrClusterDefinition.setMasterSecurityGroup(EMR_MASTER_SECURITY_GROUP + "override"); expectedEmrClusterDefinition.setSlaveSecurityGroup(EMR_SLAVE_SECURITY_GROUP + "override"); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml); // Create a new EMR cluster create request EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); emrClusterDefinitionOverride.setMasterSecurityGroup(EMR_MASTER_SECURITY_GROUP + "override"); emrClusterDefinitionOverride.setSlaveSecurityGroup(EMR_SLAVE_SECURITY_GROUP + "override"); request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride); EmrCluster emrCluster = emrService.createCluster(request); // Validate the returned object against the input. assertNotNull(emrCluster); assertTrue(emrCluster.getNamespace().equals(request.getNamespace())); assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName())); assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName())); assertNotNull(emrCluster.getId()); assertNull(emrCluster.isDryRun()); assertTrue(emrCluster.isEmrClusterCreated()); assertNotNull(emrCluster.getEmrClusterDefinition()); assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition()); validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition); }
EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); emrClusterDefinitionOverride.setInstanceDefinitions(new InstanceDefinitions()); emrClusterDefinitionOverride.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet())); request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride); expectedEmrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions()); expectedEmrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
@Test public void testCreateEmrClusterOverrideHadoopConfigurations() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()); EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml); // Create a new EMR cluster create request EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition(); ArrayList<Serializable> hadoopConfigurations = new ArrayList<>(); hadoopConfigurations.add(new ConfigurationFiles()); hadoopConfigurations.add(new KeyValuePairConfigurations()); emrClusterDefinitionOverride.setHadoopConfigurations(hadoopConfigurations); expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations()); request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride); EmrCluster emrCluster = emrService.createCluster(request); // Validate the returned object against the input. assertNotNull(emrCluster); assertTrue(emrCluster.getNamespace().equals(request.getNamespace())); assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName())); assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName())); assertNotNull(emrCluster.getId()); assertNull(emrCluster.isDryRun()); assertTrue(emrCluster.isEmrClusterCreated()); assertNotNull(emrCluster.getEmrClusterDefinition()); assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition()); validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition); }