@Bean public EmrOperations emrOperations() { return new MockEmrOperationsImpl(); }
private List<MockEmrJobFlow> getStepsByClusterId(String jobFlowId) { MockEmrJobFlow cluster = getClusterById(jobFlowId); if (cluster != null) { return cluster.getSteps(); } return null; }
private MockEmrJobFlow addClusterStep(String jobFlowId, StepConfig step) { List<MockEmrJobFlow> mockSteps = getStepsByClusterId(jobFlowId); if (mockSteps == null) { mockSteps = new ArrayList<>(); } MockEmrJobFlow mockStep = new MockEmrJobFlow(); if (!step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) { mockStep.setJobFlowId(getNewJobFlowId()); } mockStep.setJobFlowName(step.getName()); if (step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_NAME) || step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) { mockStep.setStatus(StepState.RUNNING.toString()); } else { mockStep.setStatus(StepState.PENDING.toString()); } mockStep.setJarLocation(step.getHadoopJarStep().getJar()); mockSteps.add(mockStep); setStepsByClusterId(jobFlowId, mockSteps); return mockStep; }
@Override public ListInstancesResult listClusterInstancesRequest(AmazonElasticMapReduceClient emrClient, ListInstancesRequest listInstancesRequest) { MockEmrJobFlow cluster = getClusterByName(buildEmrClusterName(AbstractDaoTest.NAMESPACE, AbstractDaoTest.EMR_CLUSTER_DEFINITION_NAME, MOCK_CLUSTER_NOT_PROVISIONED_NAME)); if (cluster != null && listInstancesRequest.getClusterId().equals(cluster.getJobFlowId())) { return new ListInstancesResult(); } Instance instance = new Instance().withEc2InstanceId("EC2_EMR_MASTER_INSTANCE").withPrivateIpAddress("INSTANCE_IP_ADDRESS"); return new ListInstancesResult().withInstances(instance); }
private MockEmrJobFlow createNewCluster(RunJobFlowRequest jobFlowRequest, String status, StatusChangeReason reason, StatusTimeline timeline) { MockEmrJobFlow cluster = new MockEmrJobFlow(); cluster.setJobFlowId(getNewJobFlowId()); cluster.setJobFlowName(jobFlowRequest.getName()); cluster.setStatus(status); cluster.setStatusTimeline(timeline); cluster.setStatusChangeReason(reason); emrClusters.put(cluster.getJobFlowId(), cluster); // Add the steps for (StepConfig stepConfig : jobFlowRequest.getSteps()) { addClusterStep(cluster.getJobFlowId(), stepConfig); } return cluster; }
MockEmrJobFlow cluster = getClusterById(addJobFlowStepsRequest.getJobFlowId()); if (cluster == null) for (StepConfig step : addJobFlowStepsRequest.getSteps()) jobIds.add(addClusterStep(cluster.getJobFlowId(), step).getJobFlowId());
return createNewCluster(jobFlowRequest, clusterStatus, reason, timeline).getJobFlowId();
private void setStepsByClusterId(String jobFlowId, List<MockEmrJobFlow> steps) { MockEmrJobFlow cluster = getClusterById(jobFlowId); if (cluster != null) { cluster.setSteps(steps); } }
@Override public void terminateEmrCluster(AmazonElasticMapReduceClient emrClient, String clusterId, boolean overrideTerminationProtection) { MockEmrJobFlow cluster = getClusterById(clusterId); if (cluster.getJobFlowName().endsWith(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION)) { throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION); } cluster.setStatus(ClusterState.TERMINATED.toString()); }
@Override public ListStepsResult listStepsRequest(AmazonElasticMapReduceClient emrClient, ListStepsRequest listStepsRequest) { MockEmrJobFlow cluster = getClusterById(listStepsRequest.getClusterId()); if (cluster == null) { throw new AmazonServiceException("No cluster found with jobFlowId: " + listStepsRequest.getClusterId()); } List<StepSummary> steps = new ArrayList<>(); // Add steps that are in these states for (MockEmrJobFlow step : cluster.getSteps()) { if ((listStepsRequest.getStepStates() == null || listStepsRequest.getStepStates().isEmpty()) || listStepsRequest.getStepStates().contains(step.getStatus())) { StepSummary stepSummary = new StepSummary().withId(step.getJobFlowId()).withName(step.getJobFlowName()).withStatus(new StepStatus().withState(step.getStatus())) .withConfig(new HadoopStepConfig().withJar(step.getJarLocation())); steps.add(stepSummary); } } return new ListStepsResult().withSteps(steps); }
@Override public DescribeClusterResult describeClusterRequest(AmazonElasticMapReduceClient emrClient, DescribeClusterRequest describeClusterRequest) { if (describeClusterRequest.getClusterId().equalsIgnoreCase(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION)) { throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION); } MockEmrJobFlow cluster = getClusterById(describeClusterRequest.getClusterId()); if (cluster != null) { return new DescribeClusterResult().withCluster(new Cluster().withId(cluster.getJobFlowId()).withName(cluster.getJobFlowName()).withStatus( new ClusterStatus().withState(cluster.getStatus()).withStateChangeReason( new ClusterStateChangeReason().withCode(cluster.getStatusChangeReason().getCode()) .withMessage(cluster.getStatusChangeReason().getMessage())).withTimeline(new ClusterTimeline().withCreationDateTime( cluster.getStatusTimeline().getCreationTime() != null ? cluster.getStatusTimeline().getCreationTime().toGregorianCalendar().getTime() : null).withEndDateTime( cluster.getStatusTimeline().getEndTime() != null ? cluster.getStatusTimeline().getEndTime().toGregorianCalendar().getTime() : null) .withReadyDateTime( cluster.getStatusTimeline().getReadyTime() != null ? cluster.getStatusTimeline().getReadyTime().toGregorianCalendar().getTime() : null)))); } else { return null; } }
@Override public DescribeStepResult describeStepRequest(AmazonElasticMapReduceClient emrClient, DescribeStepRequest describeStepRequest) { MockEmrJobFlow cluster = getClusterById(describeStepRequest.getClusterId()); if (cluster == null) { throw new AmazonServiceException("No cluster found with jobFlowId: " + describeStepRequest.getClusterId()); } Step stepResult = null; // Add steps that are in these states for (MockEmrJobFlow step : cluster.getSteps()) { if (describeStepRequest.getStepId().equalsIgnoreCase(step.getJobFlowId())) { HadoopStepConfig hadoopStepConfig = new HadoopStepConfig().withJar(step.getJarLocation()); stepResult = new Step().withId(step.getJobFlowId()).withName(step.getJobFlowName()).withStatus(new StepStatus().withState(step.getStatus())) .withConfig(hadoopStepConfig); if (stepResult.getName().equalsIgnoreCase(MOCK_STEP_WITHOUT_ID_NAME)) { stepResult.setId(null); } break; } } return new DescribeStepResult().withStep(stepResult); }