public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final String checkpointDir = pt.getRequired("checkpoint.dir"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(new FsStateBackend(checkpointDir)); env.setRestartStrategy(RestartStrategies.noRestart()); env.enableCheckpointing(1000L); env.getConfig().disableGenericTypes(); env.addSource(new MySource()).uid("my-source") .keyBy(anInt -> 0) .map(new MyStatefulFunction()).uid("my-map") .addSink(new DiscardingSink<>()).uid("my-sink"); env.execute(); } }
/** * Tests that a manual hash at the beginning of a chain is accepted. */ @Test public void testManualHashAssignmentForStartNodeInInChain() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.addSource(new NoOpSourceFunction()).uid("source") .map(new NoOpMapFunction()) .addSink(new NoOpSinkFunction()); env.getStreamGraph().getJobGraph(); }
.addSource(nonParallelSource).uid("CheckpointingSource1") .keyBy(new KeySelector<Tuple2<Long, Long>, Long>() { .addSource(parallelSource).uid("CheckpointingSource2") .keyBy(new KeySelector<Tuple2<Long, Long>, Long>() { .addSource(nonParallelSourceB).uid("BrCheckpointingSource1") .broadcast(firstBroadcastStateDesc, secondBroadcastStateDesc); .addSource(parallelSourceB).uid("BrCheckpointingSource2") .broadcast(thirdBroadcastStateDesc);
/** * Tests that a collision on the manual hash throws an Exception. */ @Test(expected = IllegalArgumentException.class) public void testManualHashAssignmentCollisionThrowsException() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); env.addSource(new NoOpSourceFunction()).uid("source") .map(new NoOpMapFunction()).uid("source") // Collision .addSink(new NoOpSinkFunction()); // This call is necessary to generate the job graph env.getStreamGraph().getJobGraph(); }
DataStream<Event> eventStream = env.addSource(createEventSource(pt)).uid("0001") .assignTimestampsAndWatermarks(createTimestampExtractor(pt)) .keyBy(Event::getKey)
private StreamExecutionEnvironment createJobGraph(int parallelism, Function<DataStream<String>, DataStream<String>> pipelinerBuilder) throws Exception { final Path checkpointDir = getCheckpointDir(); final Path savepointRootDir = getSavepointDir(); checkpointDir.getFileSystem().mkdirs(checkpointDir); savepointRootDir.getFileSystem().mkdirs(savepointRootDir); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getCheckpointConfig().enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setBufferTimeout(0); env.setParallelism(parallelism); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir.toString(), true)); DataStream<String> sourceData = env .addSource(new TestPipelineSource()) .uid("TestSource") .name("TestSource") .setParallelism(1); pipelinerBuilder.apply(sourceData) .addSink(new CollectingSink()).name("Output").uid("Output") .setParallelism(1); return env; }