public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final String checkpointDir = pt.getRequired("checkpoint.dir"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(new FsStateBackend(checkpointDir)); env.setRestartStrategy(RestartStrategies.noRestart()); env.enableCheckpointing(1000L); env.getConfig().disableGenericTypes(); env.addSource(new MySource()).uid("my-source") .keyBy(anInt -> 0) .map(new MyStatefulFunction()).uid("my-map") .addSink(new DiscardingSink<>()).uid("my-sink"); env.execute(); } }
private StreamExecutionEnvironment createJobGraph(int parallelism, Function<DataStream<String>, DataStream<String>> pipelinerBuilder) throws Exception { final Path checkpointDir = getCheckpointDir(); final Path savepointRootDir = getSavepointDir(); checkpointDir.getFileSystem().mkdirs(checkpointDir); savepointRootDir.getFileSystem().mkdirs(savepointRootDir); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getCheckpointConfig().enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setBufferTimeout(0); env.setParallelism(parallelism); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir.toString(), true)); DataStream<String> sourceData = env .addSource(new TestPipelineSource()) .uid("TestSource") .name("TestSource") .setParallelism(1); pipelinerBuilder.apply(sourceData) .addSink(new CollectingSink()).name("Output").uid("Output") .setParallelism(1); return env; }