@Test public void testGetS3StagingLocation() throws Exception { String s3StagingLocation = emrHelper.getS3StagingLocation(); assertNotNull("s3 staging location is null", s3StagingLocation); }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrHiveStepAddRequest emrHiveStepAddRequest = (EmrHiveStepAddRequest) stepRequest; EmrHiveStep step = new EmrHiveStep(); step.setNamespace(emrHiveStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrHiveStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrHiveStepAddRequest.getEmrClusterName()); step.setStepName(emrHiveStepAddRequest.getStepName().trim()); step.setScriptLocation(emrHiveStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrHiveStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrHiveStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrHiveStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrShellStepAddRequest emrShellStepAddRequest = (EmrShellStepAddRequest) stepRequest; EmrShellStep step = new EmrShellStep(); step.setNamespace(emrShellStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrShellStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrShellStepAddRequest.getEmrClusterName()); step.setStepName(emrShellStepAddRequest.getStepName().trim()); step.setScriptLocation(emrShellStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrShellStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrShellStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrShellStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrPigStepAddRequest emrPigStepAddRequest = (EmrPigStepAddRequest) stepRequest; EmrPigStep step = new EmrPigStep(); step.setNamespace(emrPigStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrPigStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrPigStepAddRequest.getEmrClusterName()); step.setStepName(emrPigStepAddRequest.getStepName().trim()); step.setScriptLocation(emrPigStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrPigStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrPigStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrPigStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrShellStepAddRequest emrShellStepAddRequest = (EmrShellStepAddRequest) stepRequest; EmrShellStep step = new EmrShellStep(); step.setNamespace(emrShellStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrShellStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrShellStepAddRequest.getEmrClusterName()); step.setStepName(emrShellStepAddRequest.getStepName().trim()); step.setScriptLocation(emrShellStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrShellStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrShellStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrShellStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrHiveStepAddRequest emrHiveStepAddRequest = (EmrHiveStepAddRequest) stepRequest; EmrHiveStep step = new EmrHiveStep(); step.setNamespace(emrHiveStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrHiveStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrHiveStepAddRequest.getEmrClusterName()); step.setStepName(emrHiveStepAddRequest.getStepName().trim()); step.setScriptLocation(emrHiveStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrHiveStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrHiveStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrHiveStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrPigStepAddRequest emrPigStepAddRequest = (EmrPigStepAddRequest) stepRequest; EmrPigStep step = new EmrPigStep(); step.setNamespace(emrPigStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrPigStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrPigStepAddRequest.getEmrClusterName()); step.setStepName(emrPigStepAddRequest.getStepName().trim()); step.setScriptLocation(emrPigStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); // Add the script arguments if (!CollectionUtils.isEmpty(emrPigStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrPigStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrPigStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrHadoopJarStepAddRequest emrHadoopJarStepAddRequest = (EmrHadoopJarStepAddRequest) stepRequest; EmrHadoopJarStep step = new EmrHadoopJarStep(); step.setNamespace(emrHadoopJarStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrHadoopJarStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrHadoopJarStepAddRequest.getEmrClusterName()); step.setStepName(emrHadoopJarStepAddRequest.getStepName().trim()); step.setJarLocation(emrHadoopJarStepAddRequest.getJarLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); if (emrHadoopJarStepAddRequest.getMainClass() != null) { step.setMainClass(emrHadoopJarStepAddRequest.getMainClass().trim()); } // Add the script arguments if (!CollectionUtils.isEmpty(emrHadoopJarStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrHadoopJarStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrHadoopJarStepAddRequest.isContinueOnError()); return step; }
@Override public Object buildResponseFromRequest(Object stepRequest) { EmrHadoopJarStepAddRequest emrHadoopJarStepAddRequest = (EmrHadoopJarStepAddRequest) stepRequest; EmrHadoopJarStep step = new EmrHadoopJarStep(); step.setNamespace(emrHadoopJarStepAddRequest.getNamespace()); step.setEmrClusterDefinitionName(emrHadoopJarStepAddRequest.getEmrClusterDefinitionName()); step.setEmrClusterName(emrHadoopJarStepAddRequest.getEmrClusterName()); step.setStepName(emrHadoopJarStepAddRequest.getStepName().trim()); step.setJarLocation(emrHadoopJarStepAddRequest.getJarLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation())); if (emrHadoopJarStepAddRequest.getMainClass() != null) { step.setMainClass(emrHadoopJarStepAddRequest.getMainClass().trim()); } // Add the script arguments if (!CollectionUtils.isEmpty(emrHadoopJarStepAddRequest.getScriptArguments())) { List<String> scriptArguments = new ArrayList<>(); step.setScriptArguments(scriptArguments); for (String argument : emrHadoopJarStepAddRequest.getScriptArguments()) { scriptArguments.add(argument.trim()); } } step.setContinueOnError(emrHadoopJarStepAddRequest.isContinueOnError()); return step; }
/** * The implementation of the EMR pre cluster steps. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key data transfer object * @param request the EMR cluster create request * * @return the EMR cluster definition * @throws Exception Exception when the original EMR cluster definition XML is malformed */ EmrClusterDefinition emrPreCreateClusterStepsImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, EmrClusterCreateRequest request) throws Exception { // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity( new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName())); // Replace all S3 managed location variables in xml String toReplace = getS3ManagedReplaceString(); String replacedConfigXml = emrClusterDefinitionEntity.getConfiguration().replaceAll(toReplace, emrHelper.getS3StagingLocation()); // Unmarshal definition xml into JAXB object. EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, replacedConfigXml); // Perform override if override is set. overrideEmrClusterDefinition(emrClusterDefinition, request.getEmrClusterDefinitionOverride()); // Perform the EMR cluster definition configuration validation. emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(emrClusterDefinition); // Check permissions. namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), emrClusterDefinition.getServiceIamRole(), emrClusterDefinition.getEc2NodeIamProfileName()); return emrClusterDefinition; }
when(configurationHelper.getProperty(ConfigurationValue.S3_STAGING_RESOURCE_LOCATION)) .thenReturn((String) ConfigurationValue.S3_STAGING_RESOURCE_LOCATION.getDefaultValue()); when(emrHelper.getS3StagingLocation()).thenReturn(S3_STAGING_LOCATION); when(xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, EMR_CLUSTER_CONFIGURATION)).thenReturn(emrClusterDefinition); verify(emrHelper).getS3StagingLocation(); verify(xmlHelper).unmarshallXmlToObject(EmrClusterDefinition.class, EMR_CLUSTER_CONFIGURATION); verify(emrClusterDefinitionHelper).validateEmrClusterDefinitionConfiguration(emrClusterDefinition);
/** * The implementation of the EMR pre cluster steps. * * @param emrClusterAlternateKeyDto the EMR cluster alternate key data transfer object * @param request the EMR cluster create request * * @return the EMR cluster definition * @throws Exception Exception when the original EMR cluster definition XML is malformed */ EmrClusterDefinition emrPreCreateClusterStepsImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, EmrClusterCreateRequest request) throws Exception { // Get the EMR cluster definition and ensure it exists. EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity( new EmrClusterDefinitionKey(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName())); // Replace all S3 managed location variables in xml String toReplace = getS3ManagedReplaceString(); String replacedConfigXml = emrClusterDefinitionEntity.getConfiguration().replaceAll(toReplace, emrHelper.getS3StagingLocation()); // Unmarshal definition xml into JAXB object. EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, replacedConfigXml); // Perform override if override is set. overrideEmrClusterDefinition(emrClusterDefinition, request.getEmrClusterDefinitionOverride()); // Perform the EMR cluster definition configuration validation. emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(emrClusterDefinition); // Check permissions. namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), emrClusterDefinition.getServiceIamRole(), emrClusterDefinition.getEc2NodeIamProfileName()); return emrClusterDefinition; }