@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getName() == null) ? 0 : getName().hashCode()); hashCode = prime * hashCode + ((getActionOnFailure() == null) ? 0 : getActionOnFailure().hashCode()); hashCode = prime * hashCode + ((getHadoopJarStep() == null) ? 0 : getHadoopJarStep().hashCode()); return hashCode; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getName() != null) sb.append("Name: ").append(getName()).append(","); if (getActionOnFailure() != null) sb.append("ActionOnFailure: ").append(getActionOnFailure()).append(","); if (getHadoopJarStep() != null) sb.append("HadoopJarStep: ").append(getHadoopJarStep()); sb.append("}"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof StepConfig == false) return false; StepConfig other = (StepConfig) obj; if (other.getName() == null ^ this.getName() == null) return false; if (other.getName() != null && other.getName().equals(this.getName()) == false) return false; if (other.getActionOnFailure() == null ^ this.getActionOnFailure() == null) return false; if (other.getActionOnFailure() != null && other.getActionOnFailure().equals(this.getActionOnFailure()) == false) return false; if (other.getHadoopJarStep() == null ^ this.getHadoopJarStep() == null) return false; if (other.getHadoopJarStep() != null && other.getHadoopJarStep().equals(this.getHadoopJarStep()) == false) return false; return true; }
/** * Marshall the given parameter object. */ public void marshall(StepConfig stepConfig, ProtocolMarshaller protocolMarshaller) { if (stepConfig == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(stepConfig.getName(), NAME_BINDING); protocolMarshaller.marshall(stepConfig.getActionOnFailure(), ACTIONONFAILURE_BINDING); protocolMarshaller.marshall(stepConfig.getHadoopJarStep(), HADOOPJARSTEP_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
@Override public boolean apply(@Nullable StepDetail input) { return input.getStepConfig().getName().equals(stepName); } }),null);
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getName() != null) sb.append("Name: ").append(getName()).append(","); if (getActionOnFailure() != null) sb.append("ActionOnFailure: ").append(getActionOnFailure()).append(","); if (getHadoopJarStep() != null) sb.append("HadoopJarStep: ").append(getHadoopJarStep()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getName() == null) ? 0 : getName().hashCode()); hashCode = prime * hashCode + ((getActionOnFailure() == null) ? 0 : getActionOnFailure().hashCode()); hashCode = prime * hashCode + ((getHadoopJarStep() == null) ? 0 : getHadoopJarStep().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(StepConfig stepConfig, ProtocolMarshaller protocolMarshaller) { if (stepConfig == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(stepConfig.getName(), NAME_BINDING); protocolMarshaller.marshall(stepConfig.getActionOnFailure(), ACTIONONFAILURE_BINDING); protocolMarshaller.marshall(stepConfig.getHadoopJarStep(), HADOOPJARSTEP_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof StepConfig == false) return false; StepConfig other = (StepConfig) obj; if (other.getName() == null ^ this.getName() == null) return false; if (other.getName() != null && other.getName().equals(this.getName()) == false) return false; if (other.getActionOnFailure() == null ^ this.getActionOnFailure() == null) return false; if (other.getActionOnFailure() != null && other.getActionOnFailure().equals(this.getActionOnFailure()) == false) return false; if (other.getHadoopJarStep() == null ^ this.getHadoopJarStep() == null) return false; if (other.getHadoopJarStep() != null && other.getHadoopJarStep().equals(this.getHadoopJarStep()) == false) return false; return true; }
private Submitter existingClusterSubmitter(AmazonElasticMapReduce emr, String tag, StepCompiler stepCompiler, String clusterId, Filer filer) { return () -> { List<String> stepIds = pollingRetryExecutor(state, "submission") .retryUnless(AmazonServiceException.class, Aws::isDeterministicException) .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) .runOnce(new TypeReference<List<String>>() {}, s -> { RemoteFile runner = prepareRunner(filer, tag); // Compile steps stepCompiler.compile(runner); // Stage files to S3 filer.stageFiles(); AddJobFlowStepsRequest request = new AddJobFlowStepsRequest() .withJobFlowId(clusterId) .withSteps(stepCompiler.stepConfigs()); int steps = request.getSteps().size(); logger.info("Submitting {} EMR step(s) to {}", steps, clusterId); AddJobFlowStepsResult result = emr.addJobFlowSteps(request); logSubmittedSteps(clusterId, steps, i -> request.getSteps().get(i).getName(), i -> result.getStepIds().get(i)); return ImmutableList.copyOf(result.getStepIds()); }); return SubmissionResult.ofExistingCluster(clusterId, stepIds); }; }
private MockEmrJobFlow addClusterStep(String jobFlowId, StepConfig step) { List<MockEmrJobFlow> mockSteps = getStepsByClusterId(jobFlowId); if (mockSteps == null) { mockSteps = new ArrayList<>(); } MockEmrJobFlow mockStep = new MockEmrJobFlow(); if (!step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) { mockStep.setJobFlowId(getNewJobFlowId()); } mockStep.setJobFlowName(step.getName()); if (step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_NAME) || step.getName().equalsIgnoreCase(MOCK_STEP_RUNNING_WITHOUT_ID_NAME)) { mockStep.setStatus(StepState.RUNNING.toString()); } else { mockStep.setStatus(StepState.PENDING.toString()); } mockStep.setJarLocation(step.getHadoopJarStep().getJar()); mockSteps.add(mockStep); setStepsByClusterId(jobFlowId, mockSteps); return mockStep; }
@Test public void testEmrHadoopJarStepConfigWithArguments() throws Exception { List<String> arguments = new ArrayList<>(); arguments.add("arg1"); StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, arguments, false); assertNotNull("step not retuned", stepConfig); assertEquals("name not found", "step_name", stepConfig.getName()); assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar()); assertNotNull("arguments not found", stepConfig.getHadoopJarStep().getArgs()); }
@Test public void testEmrHadoopJarStepConfigNoContinueOnError() throws Exception { StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, null); assertNotNull("step not retuned", stepConfig); assertEquals("name not found", "step_name", stepConfig.getName()); assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar()); }
@Test public void testEmrHadoopJarStepConfig() throws Exception { StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, false); assertNotNull("step not retuned", stepConfig); assertEquals("name not found", "step_name", stepConfig.getName()); assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar()); }
@Test public void testEmrHadoopJarStepConfigContinueOnError() throws Exception { StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, true); assertNotNull("step not retuned", stepConfig); assertEquals("name not found", "step_name", stepConfig.getName()); assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar()); }