protected void executeProgram(PackagedProgram program, ClusterClient<?> client, int parallelism) throws ProgramMissingJobException, ProgramInvocationException { logAndSysout("Starting execution of program"); final JobSubmissionResult result = client.run(program, parallelism); if (null == result) { throw new ProgramMissingJobException("No JobSubmissionResult returned, please make sure you called " + "ExecutionEnvironment.execute()"); } if (result.isJobExecutionResult()) { logAndSysout("Program execution finished"); JobExecutionResult execResult = result.getJobExecutionResult(); System.out.println("Job with JobID " + execResult.getJobID() + " has finished."); System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms"); Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults(); if (accumulatorsResult.size() > 0) { System.out.println("Accumulator Results: "); System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult)); } } else { logAndSysout("Job has been submitted with JobID " + result.getJobID()); } }
private static void assertAccumulatorsShouldFail(JobExecutionResult result) { try { result.getAllAccumulatorResults(); fail("Should have failed"); } catch (Exception ex) { assertTrue(ExceptionUtils.findThrowable(ex, CustomException.class).isPresent()); } } }
private void executeAndRunAssertions(ExecutionEnvironment env) throws Exception { try { JobExecutionResult result = env.execute(); assertTrue(result.getNetRuntime() >= 0); assertNotNull(result.getAllAccumulatorResults()); assertTrue(result.getAllAccumulatorResults().isEmpty()); } catch (JobExecutionException e) { fail("The program should have succeeded on the second run"); } }
@Override protected void testProgram() throws Exception { // test verifying the number of records read and written vs the accumulator counts readCalls = new ConcurrentLinkedQueue<Integer>(); writeCalls = new ConcurrentLinkedQueue<Integer>(); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.createInput(new TestInputFormat(new Path(inputPath))).output(new TestOutputFormat()); JobExecutionResult result = env.execute(); Object a = result.getAllAccumulatorResults().get("DATA_SOURCE_ACCUMULATOR"); Object b = result.getAllAccumulatorResults().get("DATA_SINK_ACCUMULATOR"); long recordsRead = (Long) a; long recordsWritten = (Long) b; assertEquals(recordsRead, readCalls.size()); assertEquals(recordsWritten, writeCalls.size()); }
Map<String, Object> accumulators = env.getLastJobExecutionResult().getAllAccumulatorResults(); LOGGER.info("== COUNTERS =="); for (Map.Entry<String, Object> e : accumulators.entrySet()) {
@Override protected void postSubmit() throws Exception { compareResultsByLinesInMemory(EXPECTED, resultPath); // Test accumulator results System.out.println("Accumulator results:"); JobExecutionResult res = this.result; System.out.println(AccumulatorHelper.getResultsFormatted(res.getAllAccumulatorResults())); Assert.assertEquals(Integer.valueOf(3), res.getAccumulatorResult("num-lines")); Assert.assertEquals(Integer.valueOf(3), res.getIntCounterResult("num-lines")); Assert.assertEquals(Double.valueOf(getParallelism()), res.getAccumulatorResult("open-close-counter")); // Test histogram (words per line distribution) Map<Integer, Integer> dist = new HashMap<>(); dist.put(1, 1); dist.put(2, 1); dist.put(3, 1); Assert.assertEquals(dist, res.getAccumulatorResult("words-per-line")); // Test distinct words (custom accumulator) Set<StringValue> distinctWords = new HashSet<>(); distinctWords.add(new StringValue("one")); distinctWords.add(new StringValue("two")); distinctWords.add(new StringValue("three")); Assert.assertEquals(distinctWords, res.getAccumulatorResult("distinct-words")); }
for (Map.Entry<String, Object> entry : result.getAllAccumulatorResults().entrySet()) { json.writeStringField(entry.getKey(), entry.getValue().toString());
public static void main(String[] args) throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>()); env.execute().getAllAccumulatorResults(); } }
assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
assertThat(jobExecutionResult.getNetRuntime(), equalTo(Long.MAX_VALUE)); assertThat( jobExecutionResult.getAllAccumulatorResults(), equalTo(Collections.singletonMap("testName", 1.0)));
@Override public FlinkRunnerResult run(Pipeline pipeline) { LOG.info("Executing pipeline using FlinkPipelineRunner."); LOG.info("Translating pipeline to Flink program."); this.flinkJobEnv.translate(pipeline); LOG.info("Starting execution of Flink program."); JobExecutionResult result; try { result = this.flinkJobEnv.executePipeline(); } catch (Exception e) { LOG.error("Pipeline execution failed", e); throw new RuntimeException("Pipeline execution failed", e); } LOG.info("Execution finished in {} msecs", result.getNetRuntime()); Map<String, Object> accumulators = result.getAllAccumulatorResults(); if (accumulators != null && !accumulators.isEmpty()) { LOG.info("Final aggregator values:"); for (Map.Entry<String, Object> entry : result.getAllAccumulatorResults().entrySet()) { LOG.info("{} : {}", entry.getKey(), entry.getValue()); } } return new FlinkRunnerResult(accumulators, result.getNetRuntime()); }
static PipelineResult createPipelineResult(JobExecutionResult result, PipelineOptions options) { if (result instanceof DetachedEnvironment.DetachedJobExecutionResult) { LOG.info("Pipeline submitted in Detached mode"); // no metricsPusher because metrics are not supported in detached mode return new FlinkDetachedRunnerResult(); } else { LOG.info("Execution finished in {} msecs", result.getNetRuntime()); Map<String, Object> accumulators = result.getAllAccumulatorResults(); if (accumulators != null && !accumulators.isEmpty()) { LOG.info("Final accumulator values:"); for (Map.Entry<String, Object> entry : result.getAllAccumulatorResults().entrySet()) { LOG.info("{} : {}", entry.getKey(), entry.getValue()); } } FlinkRunnerResult flinkRunnerResult = new FlinkRunnerResult(accumulators, result.getNetRuntime()); MetricsPusher metricsPusher = new MetricsPusher( flinkRunnerResult.getMetricsContainerStepMap(), options, flinkRunnerResult); metricsPusher.start(); return flinkRunnerResult; } }
static PipelineResult createPipelineResult(JobExecutionResult result, PipelineOptions options) { if (result instanceof DetachedEnvironment.DetachedJobExecutionResult) { LOG.info("Pipeline submitted in Detached mode"); // no metricsPusher because metrics are not supported in detached mode return new FlinkDetachedRunnerResult(); } else { LOG.info("Execution finished in {} msecs", result.getNetRuntime()); Map<String, Object> accumulators = result.getAllAccumulatorResults(); if (accumulators != null && !accumulators.isEmpty()) { LOG.info("Final accumulator values:"); for (Map.Entry<String, Object> entry : result.getAllAccumulatorResults().entrySet()) { LOG.info("{} : {}", entry.getKey(), entry.getValue()); } } FlinkRunnerResult flinkRunnerResult = new FlinkRunnerResult(accumulators, result.getNetRuntime()); MetricsPusher metricsPusher = new MetricsPusher( flinkRunnerResult.getMetricsContainerStepMap(), options.as(MetricsOptions.class), flinkRunnerResult); metricsPusher.start(); return flinkRunnerResult; } }
} else { LOG.info("Execution finished in {} msecs", result.getNetRuntime()); Map<String, Object> accumulators = result.getAllAccumulatorResults(); if (accumulators != null && !accumulators.isEmpty()) { LOG.info("Final accumulator values:"); for (Map.Entry<String, Object> entry : result.getAllAccumulatorResults().entrySet()) { LOG.info("{} : {}", entry.getKey(), entry.getValue());
protected void executeProgram(PackagedProgram program, ClusterClient<?> client, int parallelism) throws ProgramMissingJobException, ProgramInvocationException { logAndSysout("Starting execution of program"); final JobSubmissionResult result = client.run(program, parallelism); if (null == result) { throw new ProgramMissingJobException("No JobSubmissionResult returned, please make sure you called " + "ExecutionEnvironment.execute()"); } if (result.isJobExecutionResult()) { logAndSysout("Program execution finished"); JobExecutionResult execResult = result.getJobExecutionResult(); System.out.println("Job with JobID " + execResult.getJobID() + " has finished."); System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms"); Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults(); if (accumulatorsResult.size() > 0) { System.out.println("Accumulator Results: "); System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult)); } } else { logAndSysout("Job has been submitted with JobID " + result.getJobID()); } }
protected void executeProgram(PackagedProgram program, ClusterClient<?> client, int parallelism) throws ProgramMissingJobException, ProgramInvocationException { logAndSysout("Starting execution of program"); final JobSubmissionResult result = client.run(program, parallelism); if (null == result) { throw new ProgramMissingJobException("No JobSubmissionResult returned, please make sure you called " + "ExecutionEnvironment.execute()"); } if (result.isJobExecutionResult()) { logAndSysout("Program execution finished"); JobExecutionResult execResult = result.getJobExecutionResult(); System.out.println("Job with JobID " + execResult.getJobID() + " has finished."); System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms"); Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults(); if (accumulatorsResult.size() > 0) { System.out.println("Accumulator Results: "); System.out.println(AccumulatorHelper.getResultsFormatted(accumulatorsResult)); } } else { logAndSysout("Job has been submitted with JobID " + result.getJobID()); } }
Map<String, Object> accumulators = env.getLastJobExecutionResult().getAllAccumulatorResults(); LOGGER.info("== COUNTERS =="); for (Map.Entry<String, Object> e : accumulators.entrySet()) {
System.out.println("Job with JobID " + execResult.getJobID() + " has finished."); System.out.println("Job Runtime: " + execResult.getNetRuntime() + " ms"); Map<String, Object> accumulatorsResult = execResult.getAllAccumulatorResults(); if (accumulatorsResult.size() > 0) { System.out.println("Accumulator Results: ");
Map<String, Object> accumulators = env.getLastJobExecutionResult().getAllAccumulatorResults(); LOGGER.info("== COUNTERS =="); for (Map.Entry<String, Object> e : accumulators.entrySet()) {
Map<String, Object> accumulators = env.getLastJobExecutionResult().getAllAccumulatorResults(); LOGGER.info("== COUNTERS =="); for (Map.Entry<String, Object> e : accumulators.entrySet()) {