/** * Adds a Hadoop Jar step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add Hadoop Jar step to the EMR cluster. * * @return the created EMR Hadoop Jar step. */ @RequestMapping(value = "/emrHadoopJarSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_HADOOP_JAR_STEPS_POST) public EmrHadoopJarStep addHadoopJarStepToEmrCluster(@RequestBody EmrHadoopJarStepAddRequest request) throws Exception { return (EmrHadoopJarStep) emrService.addStepToCluster(request); }
/** * Adds a hive step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add hive step to the EMR cluster. * * @return the created EMR hive step. */ @RequestMapping(value = "/emrHiveSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_HIVE_STEPS_POST) public EmrHiveStep addHiveStepToEmrCluster(@RequestBody EmrHiveStepAddRequest request) throws Exception { return (EmrHiveStep) emrService.addStepToCluster(request); }
/** * Adds a Pig step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add Pig step to the EMR cluster. * * @return the created EMR Pig step. */ @RequestMapping(value = "/emrPigSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_PIG_STEPS_POST) public EmrPigStep addPigStepToEmrCluster(@RequestBody EmrPigStepAddRequest request) throws Exception { return (EmrPigStep) emrService.addStepToCluster(request); }
/** * Adds a hive step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add hive step to the EMR cluster. * * @return the created EMR hive step. */ @RequestMapping(value = "/emrHiveSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_HIVE_STEPS_POST) public EmrHiveStep addHiveStepToEmrCluster(@RequestBody EmrHiveStepAddRequest request) throws Exception { return (EmrHiveStep) emrService.addStepToCluster(request); }
/** * Adds a shell step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add shell step to the EMR cluster. * * @return the created EMR shell step. * @throws Exception if a shell step couldn't be added to the EMR cluster. */ @RequestMapping(value = "/emrShellSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_SHELL_STEPS_POST) public EmrShellStep addShellStepToEmrCluster(@RequestBody EmrShellStepAddRequest request) throws Exception { return (EmrShellStep) emrService.addStepToCluster(request); }
/** * Adds a Hadoop Jar step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add Hadoop Jar step to the EMR cluster. * * @return the created EMR Hadoop Jar step. */ @RequestMapping(value = "/emrHadoopJarSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_HADOOP_JAR_STEPS_POST) public EmrHadoopJarStep addHadoopJarStepToEmrCluster(@RequestBody EmrHadoopJarStepAddRequest request) throws Exception { return (EmrHadoopJarStep) emrService.addStepToCluster(request); }
/** * Adds a Pig step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add Pig step to the EMR cluster. * * @return the created EMR Pig step. */ @RequestMapping(value = "/emrPigSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_PIG_STEPS_POST) public EmrPigStep addPigStepToEmrCluster(@RequestBody EmrPigStepAddRequest request) throws Exception { return (EmrPigStep) emrService.addStepToCluster(request); }
/** * Adds a shell step to the existing cluster <p>Requires EXECUTE permission on namespace</p> * * @param request the information needed to add shell step to the EMR cluster. * * @return the created EMR shell step. * @throws Exception if a shell step couldn't be added to the EMR cluster. */ @RequestMapping(value = "/emrShellSteps", method = RequestMethod.POST, consumes = {"application/xml", "application/json"}) @Secured(SecurityFunctions.FN_EMR_SHELL_STEPS_POST) public EmrShellStep addShellStepToEmrCluster(@RequestBody EmrShellStepAddRequest request) throws Exception { return (EmrShellStep) emrService.addStepToCluster(request); }
/** * Adds the Step to EMR cluster, and sets the step id as workflow variable. * * @param request the EmrStepsAddRequest * @param execution the DelegateExecution * * @throws Exception if any problems were encountered. */ protected void addEmrStepAndSetWorkflowVariables(Object request, DelegateExecution execution) throws Exception { // Add the step. Object emrStep = emrService.addStepToCluster(request); EmrStepHelper stepHelper = emrStepHelperFactory.getStepHelper(request.getClass().getName()); String stepId = stepHelper.getStepId(emrStep); // Set workflow variables based on the new step. // Set the stepId in workflow variable. setTaskWorkflowVariable(execution, VARIABLE_EMR_STEP_ID, stepId); }
/** * Adds the Step to EMR cluster, and sets the step id as workflow variable. * * @param request the EmrStepsAddRequest * @param execution the DelegateExecution * * @throws Exception if any problems were encountered. */ protected void addEmrStepAndSetWorkflowVariables(Object request, DelegateExecution execution) throws Exception { // Add the step. Object emrStep = emrService.addStepToCluster(request); EmrStepHelper stepHelper = emrStepHelperFactory.getStepHelper(request.getClass().getName()); String stepId = stepHelper.getStepId(emrStep); // Set workflow variables based on the new step. // Set the stepId in workflow variable. setTaskWorkflowVariable(execution, VARIABLE_EMR_STEP_ID, stepId); }
@Test public void testAddHadoopJarStepToEmrCluster() throws Exception { // Create an add step request. EmrHadoopJarStepAddRequest emrHadoopJarStepAddRequest = new EmrHadoopJarStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_JAR_LOCATION, EMR_STEP_MAIN_CLASS, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID); // Create an add step response. EmrHadoopJarStep emrHadoopJarStep = new EmrHadoopJarStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_JAR_LOCATION, EMR_STEP_MAIN_CLASS, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID); // Mock the external calls. when(emrService.addStepToCluster(emrHadoopJarStepAddRequest)).thenReturn(emrHadoopJarStep); // Call the method under test. EmrHadoopJarStep result = emrRestController.addHadoopJarStepToEmrCluster(emrHadoopJarStepAddRequest); // Verify the external calls. verify(emrService).addStepToCluster(emrHadoopJarStepAddRequest); verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(emrHadoopJarStep, result); }
@Test public void testAddShellStepToEmrCluster() throws Exception { // Create an add step request. EmrShellStepAddRequest emrShellStepAddRequest = new EmrShellStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID); // Create an add step response. EmrShellStep emrShellStep = new EmrShellStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID); // Mock the external calls. when(emrService.addStepToCluster(emrShellStepAddRequest)).thenReturn(emrShellStep); // Call the method under test. EmrShellStep result = emrRestController.addShellStepToEmrCluster(emrShellStepAddRequest); // Verify the external calls. verify(emrService).addStepToCluster(emrShellStepAddRequest); verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(emrShellStep, result); }
@Test public void testAddPigStepToEmrCluster() throws Exception { // Create an add step request. EmrPigStepAddRequest emrPigStepAddRequest = new EmrPigStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID); // Create an add step response. EmrPigStep emrPigStep = new EmrPigStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID); // Mock the external calls. when(emrService.addStepToCluster(emrPigStepAddRequest)).thenReturn(emrPigStep); // Call the method under test. EmrPigStep result = emrRestController.addPigStepToEmrCluster(emrPigStepAddRequest); // Verify the external calls. verify(emrService).addStepToCluster(emrPigStepAddRequest); verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(emrPigStep, result); }
@Test public void testAddHiveStepToEmrCluster() throws Exception { // Create an add step request. EmrHiveStepAddRequest emrHiveStepAddRequest = new EmrHiveStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID); // Create an add step response. EmrHiveStep emrHiveStep = new EmrHiveStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID); // Mock the external calls. when(emrService.addStepToCluster(emrHiveStepAddRequest)).thenReturn(emrHiveStep); // Call the method under test. EmrHiveStep result = emrRestController.addHiveStepToEmrCluster(emrHiveStepAddRequest); // Verify the external calls. verify(emrService).addStepToCluster(emrHiveStepAddRequest); verifyNoMoreInteractionsHelper(); // Validate the results. assertEquals(emrHiveStep, result); }
/** * This method tests the error cases for AmazonExceptions for Illegal Argument */ @Test(expected = ObjectNotFoundException.class) public void testEmrAddStepsAmazonObjectNotFound() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // Create a new EMR cluster create request. EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName()); stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_NOT_FOUND); emrService.addStepToCluster(stepRequest); }
/** * This method tests the Amazon exceptions */ @Test(expected = AmazonServiceException.class) public void testEmrAddStepsAmazonOtherException() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // Create a new EMR cluster create request. EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName()); stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION); emrService.addStepToCluster(stepRequest); }
/** * This method tests the happy path scenario by providing all the parameters */ @Test(expected = IllegalArgumentException.class) public void testEmrAddStepsInvalidCluster() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // Create a new EMR cluster create request. EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName()); stepRequest.setEmrClusterName("InvalidName"); emrService.addStepToCluster(stepRequest); }
/** * This method tests the Amazon BAD_REQUEST exception */ @Test(expected = IllegalArgumentException.class) public void testEmrAddStepsAmazonBadRequest() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // Create a new EMR cluster create request. EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName()); stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_BAD_REQUEST); emrService.addStepToCluster(stepRequest); }
/** * This method tests the happy path scenario by providing all the parameters */ @Test public void testEmrAddSteps() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // Create the Add step request. EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName()); EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest); // Validate the returned object against the input. assertNotNull(emrShellStep); assertTrue(emrShellStep.getNamespace().equals(request.getNamespace())); assertTrue(emrShellStep.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName())); assertTrue(emrShellStep.getEmrClusterName().equals(request.getEmrClusterName())); }
/** * This method tests the happy path scenario by testing all the step types */ @Test public void testEmrAddStepsHadoopNoMainClass() throws Exception { // Create the namespace entity. NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE); emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream())); EmrClusterCreateRequest request = getNewEmrClusterCreateRequest(); emrService.createCluster(request); // TODO: Why are we adding EMR steps to a list, but not doing anything with them once they're added? List<Serializable> emrSteps = new ArrayList<>(); EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest(); hadoopJarStepRequest.setNamespace(request.getNamespace()); hadoopJarStepRequest.setEmrClusterDefinitionName(request.getEmrClusterDefinitionName()); hadoopJarStepRequest.setEmrClusterName(request.getEmrClusterName()); hadoopJarStepRequest.setStepName("Hadoop Jar"); hadoopJarStepRequest.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar"); emrSteps.add(hadoopJarStepRequest); EmrHadoopJarStep emrHadoopJarStep = (EmrHadoopJarStep) emrService.addStepToCluster(hadoopJarStepRequest); assertNotNull(emrHadoopJarStep); assertNotNull(emrHadoopJarStep.getId()); }