/** * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long)}. * * @param interval Time interval between state checkpoints in milliseconds. * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller */ public PythonStreamExecutionEnvironment enable_checkpointing(long interval) { this.env.enableCheckpointing(interval); return this; }
/** * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long, CheckpointingMode)}. * * @param interval Time interval between state checkpoints in milliseconds. * @param mode The checkpointing mode, selecting between "exactly once" and "at least once" guaranteed. * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller */ public PythonStreamExecutionEnvironment enable_checkpointing(long interval, CheckpointingMode mode) { this.env.enableCheckpointing(interval, mode); return this; }
public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool) throws Exception { if (parameterTool.getNumberOfParameters() < 5) { System.out.println("Missing parameters!\n" + "Usage: Kafka --input-topic <topic> --output-topic <topic> " + "--bootstrap.servers <kafka brokers> " + "--zookeeper.connect <zk quorum> --group.id <some id>"); throw new Exception("Missing parameters!\n" + "Usage: Kafka --input-topic <topic> --output-topic <topic> " + "--bootstrap.servers <kafka brokers> " + "--zookeeper.connect <zk quorum> --group.id <some id>"); } StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)); env.enableCheckpointing(5000); // create a checkpoint every 5 seconds env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); return env; }
public static void main(final String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final String outputPath = params.getRequired("outputPath"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); env.enableCheckpointing(5000L); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(10L, TimeUnit.SECONDS))); final StreamingFileSink<Tuple2<Integer, Integer>> sink = StreamingFileSink .forRowFormat(new Path(outputPath), (Encoder<Tuple2<Integer, Integer>>) (element, stream) -> { PrintStream out = new PrintStream(stream); out.println(element.f1); }) .withBucketAssigner(new KeyBucketAssigner()) .withRollingPolicy(OnCheckpointRollingPolicy.build()) .build(); // generate data, shuffle, sink env.addSource(new Generator(10, 10, 60)) .keyBy(0) .addSink(sink); env.execute("StreamingFileSinkProgram"); }
/** * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the * client side. */ @Test public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500); env.fromElements(1).print(); StreamGraph graph = env.getStreamGraph(); JobGraph jobGraph = graph.getJobGraph(); RestartStrategies.RestartStrategyConfiguration restartStrategy = jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy(); Assert.assertNotNull(restartStrategy); Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration); }
/** * Checks that in a streaming use case where checkpointing is enabled and the number * of execution retries is set to 0, restarting is deactivated. */ @Test public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500); env.setNumberOfExecutionRetries(0); env.fromElements(1).print(); StreamGraph graph = env.getStreamGraph(); JobGraph jobGraph = graph.getJobGraph(); RestartStrategies.RestartStrategyConfiguration restartStrategy = jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy(); Assert.assertNotNull(restartStrategy); Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration); }
/** * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)} * followed by the checks in {@link #postSubmit}. */ @Test public void runCheckpointedProgram() throws Exception { try { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.enableCheckpointing(500); env.getConfig().disableSysoutLogging(); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L)); testProgram(env); TestUtils.tryExecute(env, "Fault Tolerance Test"); postSubmit(); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); String outputPath = params.getRequired("outputPath"); StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment(); sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart( 3, Time.of(10, TimeUnit.SECONDS) )); sEnv.enableCheckpointing(4000); final int idlenessMs = 10; // define bucketing sink to emit the result BucketingSink<Tuple4<Integer, Long, Integer, String>> sink = new BucketingSink<Tuple4<Integer, Long, Integer, String>>(outputPath) .setBucketer(new KeyBucketer()); // generate data, shuffle, perform stateful operation, sink sEnv.addSource(new Generator(10, idlenessMs, 60)) .keyBy(0) .map(new SubtractingMapper(-1L * idlenessMs)) .addSink(sink); sEnv.execute(); }
private JobGraph createJobGraph(ExecutionMode mode) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend((StateBackend) new MemoryStateBackend()); switch (mode) { case MIGRATE: createMigrationJob(env); break; case RESTORE: createRestoredJob(env); break; } return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph()); }
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final String checkpointDir = pt.getRequired("checkpoint.dir"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(new FsStateBackend(checkpointDir)); env.setRestartStrategy(RestartStrategies.noRestart()); env.enableCheckpointing(1000L); env.getConfig().disableGenericTypes(); env.addSource(new MySource()).uid("my-source") .keyBy(anInt -> 0) .map(new MyStatefulFunction()).uid("my-map") .addSink(new DiscardingSink<>()).uid("my-sink"); env.execute(); } }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); String outputPath = params.getRequired("outputPath"); int recordsPerSecond = params.getInt("recordsPerSecond", 10); int duration = params.getInt("durationInSecond", 60); int offset = params.getInt("offsetInSecond", 0); StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment(); sEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); sEnv.enableCheckpointing(4000); sEnv.getConfig().setAutoWatermarkInterval(1000); // execute a simple pass through program. PeriodicSourceGenerator generator = new PeriodicSourceGenerator( recordsPerSecond, duration, offset); DataStream<Tuple> rows = sEnv.addSource(generator); DataStream<Tuple> result = rows .keyBy(1) .timeWindow(Time.seconds(5)) .sum(0); result.writeAsText(outputPath + "/result.txt", FileSystem.WriteMode.OVERWRITE) .setParallelism(1); sEnv.execute(); }
/** * Runs the following program. * <pre> * [ (source)->(filter)] -> [ (map) -> (map) ] -> [ (groupBy/reduce)->(sink) ] * </pre> */ @Override public void testProgram(StreamExecutionEnvironment env) { assertTrue("Broken test setup", NUM_STRINGS % 40 == 0); final long failurePosMin = (long) (0.4 * NUM_STRINGS / PARALLELISM); final long failurePosMax = (long) (0.7 * NUM_STRINGS / PARALLELISM); final long failurePos = (new Random().nextLong() % (failurePosMax - failurePosMin)) + failurePosMin; env.enableCheckpointing(200); DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS)); stream // first vertex, chained to the source // this filter throttles the flow until at least one checkpoint // is complete, to make sure this program does not run without .filter(new StringRichFilterFunction()) // -------------- seconds vertex - one-to-one connected ---------------- .map(new StringPrefixCountRichMapFunction()) .startNewChain() .map(new StatefulCounterFunction()) // -------------- third vertex - reducer and the sink ---------------- .keyBy("prefix") .flatMap(new OnceFailingAggregator(failurePos)) .addSink(new ValidatingSink()); }
private static JobGraph createJobGraphWithOperatorState( int parallelism, int maxParallelism, OperatorCheckpointMethod checkpointMethod) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.getConfig().setMaxParallelism(maxParallelism); env.enableCheckpointing(Long.MAX_VALUE); env.setRestartStrategy(RestartStrategies.noRestart()); StateSourceBase.workStartedLatch = new CountDownLatch(parallelism); SourceFunction<Integer> src; switch (checkpointMethod) { case CHECKPOINTED_FUNCTION: src = new PartitionedStateSource(false); break; case CHECKPOINTED_FUNCTION_BROADCAST: src = new PartitionedStateSource(true); break; case LIST_CHECKPOINTED: src = new PartitionedStateSourceListCheckpointed(); break; case NON_PARTITIONED: src = new NonPartitionedStateSource(); break; default: throw new IllegalArgumentException(); } DataStream<Integer> input = env.addSource(src); input.addSink(new DiscardingSink<Integer>()); return env.getStreamGraph().getJobGraph(); }
@Test public void testCheckpointModeTranslation() { try { // with deactivated fault tolerance, the checkpoint mode should be at-least-once StreamExecutionEnvironment deactivated = getSimpleJob(); for (JobVertex vertex : deactivated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // with activated fault tolerance, the checkpoint mode should be by default exactly once StreamExecutionEnvironment activated = getSimpleJob(); activated.enableCheckpointing(1000L); for (JobVertex vertex : activated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.EXACTLY_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // explicitly setting the mode StreamExecutionEnvironment explicit = getSimpleJob(); explicit.enableCheckpointing(1000L, CheckpointingMode.AT_LEAST_ONCE); for (JobVertex vertex : explicit.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
/** * Checks that in a streaming use case where checkpointing is enabled and the number * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used. */ @Test public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500); env.setNumberOfExecutionRetries(42); env.getConfig().setExecutionRetryDelay(1337); env.fromElements(1).print(); StreamGraph graph = env.getStreamGraph(); JobGraph jobGraph = graph.getJobGraph(); RestartStrategies.RestartStrategyConfiguration restartStrategy = jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy(); Assert.assertNotNull(restartStrategy); Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration); Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts()); Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds()); } }
private static void runPartitioningProgram(int parallelism) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.getConfig().enableObjectReuse(); env.setBufferTimeout(5L); env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE); env .addSource(new TimeStampingSource()) .map(new IdMapper<Tuple2<Long, Long>>()) .keyBy(0) .addSink(new TimestampingSink()); env.execute("Partitioning Program"); }
private static JobGraph createJobGraphWithKeyedAndNonPartitionedOperatorState( int parallelism, int maxParallelism, int fixedParallelism, int numberKeys, int numberElements, boolean terminateAfterEmission, int checkpointingInterval) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.getConfig().setMaxParallelism(maxParallelism); env.enableCheckpointing(checkpointingInterval); env.setRestartStrategy(RestartStrategies.noRestart()); DataStream<Integer> input = env.addSource(new SubtaskIndexNonPartitionedStateSource( numberKeys, numberElements, terminateAfterEmission)) .setParallelism(fixedParallelism) .keyBy(new KeySelector<Integer, Integer>() { private static final long serialVersionUID = -7952298871120320940L; @Override public Integer getKey(Integer value) throws Exception { return value; } }); SubtaskIndexFlatMapper.workCompletedLatch = new CountDownLatch(numberKeys); DataStream<Tuple2<Integer, Integer>> result = input.flatMap(new SubtaskIndexFlatMapper(numberElements)); result.addSink(new CollectionSink<Tuple2<Integer, Integer>>()); return env.getStreamGraph().getJobGraph(); }
public static void main(String[] args) throws Exception { ParameterTool pt = ParameterTool.fromArgs(args); String savepointsPath = pt.getRequired("savepoint-path"); Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath); StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend(new MemoryStateBackend()); /** * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2) */ SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = createIntegerTupleSource(env, ExecutionMode.GENERATE); SingleOutputStreamOperator<Integer> window = createWindowFunction(ExecutionMode.GENERATE, source); SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, window); SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first); env.execute("job"); }
@Override public void testProgram(StreamExecutionEnvironment env) { // set the restart strategy. env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(NO_OF_RETRIES, 0)); env.enableCheckpointing(10); // create and start the file creating thread. fc = new FileCreator(); fc.start(); // create the monitoring source along with the necessary readers. TextInputFormat format = new TextInputFormat(new org.apache.flink.core.fs.Path(localFsURI)); format.setFilesFilter(FilePathFilter.createDefaultFilter()); DataStream<String> inputStream = env.readFile(format, localFsURI, FileProcessingMode.PROCESS_CONTINUOUSLY, INTERVAL); TestingSinkFunction sink = new TestingSinkFunction(); inputStream.flatMap(new FlatMapFunction<String, String>() { @Override public void flatMap(String value, Collector<String> out) throws Exception { out.collect(value); } }).addSink(sink).setParallelism(1); }
public static void main(String[] args) throws Exception { ParameterTool pt = ParameterTool.fromArgs(args); String savepointsPath = pt.getRequired("savepoint-path"); Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath); StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend(new MemoryStateBackend()); /** * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3) */ DataStream<Integer> source = createSource(env, ExecutionMode.GENERATE); SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, source); first.startNewChain(); SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first); second.startNewChain(); SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second); SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.GENERATE, stateless); env.execute("job"); }