@Override public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception { List<StepConfig> steps = new ArrayList<>(); steps.add(emrStepConfig); // Add the job flow request AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps); List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest); return emrStepIds.get(0); }
@Override public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception { List<StepConfig> steps = new ArrayList<>(); steps.add(emrStepConfig); // Add the job flow request AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps); List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest); return emrStepIds.get(0); }
@Override public void runJob(MavenManagedJar defaultJar, List<String> jarArgs) throws Exception { String jarLocation=defaultJar.s3JarLocation(awsSoftwareBucket); count++; String stepName = UUID.randomUUID().toString(); StepConfig step = new StepConfig(stepName,new HadoopJarStepConfig(jarLocation).withArgs(jarArgs)); emrClient.addJobFlowSteps(new AddJobFlowStepsRequest() .withJobFlowId(runningCluster) .withSteps(step)); Thread.sleep(5000); // Enough time to transition out of WAITING? pollClusterForCompletion(runningCluster, stepName); }
"-d", "windowTime=tablaparametro")); AddJobFlowStepsRequest jobFlow = new AddJobFlowStepsRequest().withJobFlowId(rb.getString("jobflowID")) .withSteps(stepHive);
private Submitter existingClusterSubmitter(AmazonElasticMapReduce emr, String tag, StepCompiler stepCompiler, String clusterId, Filer filer) { return () -> { List<String> stepIds = pollingRetryExecutor(state, "submission") .retryUnless(AmazonServiceException.class, Aws::isDeterministicException) .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) .runOnce(new TypeReference<List<String>>() {}, s -> { RemoteFile runner = prepareRunner(filer, tag); // Compile steps stepCompiler.compile(runner); // Stage files to S3 filer.stageFiles(); AddJobFlowStepsRequest request = new AddJobFlowStepsRequest() .withJobFlowId(clusterId) .withSteps(stepCompiler.stepConfigs()); int steps = request.getSteps().size(); logger.info("Submitting {} EMR step(s) to {}", steps, clusterId); AddJobFlowStepsResult result = emr.addJobFlowSteps(request); logSubmittedSteps(clusterId, steps, i -> request.getSteps().get(i).getName(), i -> result.getStepIds().get(i)); return ImmutableList.copyOf(result.getStepIds()); }); return SubmissionResult.ofExistingCluster(clusterId, stepIds); }; }
@Override public void addStepToExistingJobFlow( String stagingS3FileUrl, String stagingS3BucketUrl, String stepType, String mainClass, AbstractAmazonJobEntry jobEntry ) { this.alive = jobEntry.getAlive(); this.hadoopJobFlowId = jobEntry.getHadoopJobFlowId(); setStepsFromCluster(); List<StepConfig> steps = initSteps( stagingS3FileUrl, stepType, mainClass, jobEntry ); AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest(); addJobFlowStepsRequest.setJobFlowId( hadoopJobFlowId ); addJobFlowStepsRequest.setSteps( steps ); emrClient.addJobFlowSteps( addJobFlowStepsRequest ); stepId = getSpecifiedRunningStep(); }