.flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override public void flatMap(Long value, Collector<Tuple2<String, String>> out) {
.flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override public void flatMap(Long value, Collector<Tuple2<String, String>> out) {
.flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override public void flatMap(Long value, Collector<Tuple2<String, String>> out) {
.flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override public void flatMap(Long value, Collector<Tuple2<String, String>> out) {
@Test public void testStreamingDistributedCache() throws Exception { String textPath = createTempFile("count.txt", DATA); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.registerCachedFile(textPath, "cache_test"); env.readTextFile(textPath).flatMap(new WordChecker()); env.execute(); }
.flatMap(new FlatMapFunction<Tuple2<Integer, Integer>, Object>() { @Override public void flatMap(Tuple2<Integer, Integer> value, Collector<Object> out) throws Exception {
final IntegerStreamSource source = new IntegerStreamSource(); IterativeStream<Integer> iteration = env.addSource(source) .flatMap(new RichFlatMapFunction<Integer, Integer>() {
DataStream<Integer> flatMap = src.flatMap(flatMapFunction); flatMap.addSink(new DiscardingSink<Integer>()); assertEquals(flatMapFunction, getFunctionForDataStream(flatMap));
env.addSource(srcFun).flatMap(new FlatMapFunction<Integer, Object>() {
@SuppressWarnings("serial") private static DataStream<Edge<Long, NullValue>> getEdgesDataSet(StreamExecutionEnvironment env) { if (fileOutput) { return env.readTextFile(edgeInputPath) .map(new MapFunction<String, Edge<Long, NullValue>>() { @Override public Edge<Long, NullValue> map(String s) throws Exception { String[] fields = s.split("\\t"); long src = Long.parseLong(fields[0]); long trg = Long.parseLong(fields[1]); return new Edge<>(src, trg, NullValue.getInstance()); } }); } return env.generateSequence(0, 999).flatMap( new FlatMapFunction<Long, Edge<Long, NullValue>>() { @Override public void flatMap(Long key, Collector<Edge<Long, NullValue>> out) throws Exception { out.collect(new Edge<>(key, (key + 2) % 1000, NullValue.getInstance())); out.collect(new Edge<>(key, (key + 4) % 1000, NullValue.getInstance())); } }); }
@SuppressWarnings("serial") private static DataStream<Edge<Long, NullValue>> getEdgesDataSet(StreamExecutionEnvironment env) { if (fileOutput) { return env.readTextFile(edgeInputPath) .map(new MapFunction<String, Edge<Long, NullValue>>() { @Override public Edge<Long, NullValue> map(String s) throws Exception { String[] fields = s.split("\\t"); long src = Long.parseLong(fields[0]); long trg = Long.parseLong(fields[1]); return new Edge<>(src, trg, NullValue.getInstance()); } }); } return env.generateSequence(0, 999).flatMap( new FlatMapFunction<Long, Edge<Long, NullValue>>() { @Override public void flatMap(Long key, Collector<Edge<Long, NullValue>> out) throws Exception { out.collect(new Edge<>(key, (key + 2) % 1000, NullValue.getInstance())); out.collect(new Edge<>(key, (key + 4) % 1000, NullValue.getInstance())); } }); }
@SuppressWarnings("serial") private static DataStream<Tuple2<Long, Long>> getEdgesDataSet(StreamExecutionEnvironment env) { if (fileOutput) { return env.readTextFile(edgeInputPath) .map(new MapFunction<String, Tuple2<Long, Long>>() { @Override public Tuple2<Long, Long> map(String s) { String[] fields = s.split("\\t"); long src = Long.parseLong(fields[0]); long trg = Long.parseLong(fields[1]); return new Tuple2<>(src, trg); } }); } return env.generateSequence(1, 10).flatMap( new FlatMapFunction<Long, Tuple2<Long, Long>>() { @Override public void flatMap(Long key, Collector<Tuple2<Long, Long>> out) throws Exception { for (int i = 1; i < 3; i++) { long target = key + i; out.collect(new Tuple2<>(key, target)); } } }); }
@SuppressWarnings("serial") private static DataStream<Edge<Long, NullValue>> getEdgesDataSet(StreamExecutionEnvironment env) { if (fileOutput) { return env.readTextFile(edgeInputPath) .map(new MapFunction<String, Edge<Long, NullValue>>() { @Override public Edge<Long, NullValue> map(String s) throws Exception { String[] fields = s.split("\\t"); long src = Long.parseLong(fields[0]); long trg = Long.parseLong(fields[1]); return new Edge<>(src, trg, NullValue.getInstance()); } }); } return env.generateSequence(1, 100).flatMap( new FlatMapFunction<Long, Edge<Long, NullValue>>() { @Override public void flatMap(Long key, Collector<Edge<Long, NullValue>> out) throws Exception { for (int i = 0; i < 10; i++) { long target = key * 2 + 1; out.collect(new Edge<>(key, target, NullValue.getInstance())); } } }); }
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> dataStream = env .fromElements("Who's there?", "I think I hear them. Stand, ho! Who's there?") //.socketTextStream("localhost", 9999) .flatMap(new Splitter()) .keyBy(0) .sum(1); dataStream.print(); env.execute(); //env.execute("Socket Stream WordCount"); }
@Override public void translateNode(Read.Unbounded<T> transform, FlinkStreamingTranslationContext context) { PCollection<T> output = context.getOutput(transform); DataStream<WindowedValue<T>> source; if (transform.getSource().getClass().equals(UnboundedFlinkSource.class)) { UnboundedFlinkSource flinkSource = (UnboundedFlinkSource) transform.getSource(); source = context.getExecutionEnvironment() .addSource(flinkSource.getFlinkSource()) .flatMap(new FlatMapFunction<String, WindowedValue<String>>() { @Override public void flatMap(String s, Collector<WindowedValue<String>> collector) throws Exception { collector.collect(WindowedValue.<String>of(s, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)); } }).assignTimestampsAndWatermarks(new IngestionTimeExtractor()); } else { source = context.getExecutionEnvironment() .addSource(new UnboundedSourceWrapper<>(context.getPipelineOptions(), transform)); } context.setOutputDataStream(output, source); } }
@Override public Object doOperation(final AddElementsFromSocket op, final Context context, final Store store) throws OperationException { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); if (null != op.getParallelism()) { env.setParallelism(op.getParallelism()); } final DataStream<Element> builder = env.socketTextStream(op.getHostname(), op.getPort(), op.getDelimiter()) .flatMap(new GafferMapFunction(String.class, op.getElementGenerator())); if (Boolean.parseBoolean(op.getOption(FlinkConstants.SKIP_REBALANCING))) { builder.addSink(new GafferSink(op, store)); } else { builder.rebalance().addSink(new GafferSink(op, store)); } try { env.execute(op.getClass().getSimpleName() + "-" + op.getHostname() + ":" + op.getPort()); } catch (final Exception e) { throw new OperationException("Failed to add elements from port: " + op.getPort(), e); } return null; } }
public static void main(String argsp[]) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); // 设置checkpoint env.enableCheckpointing(60000); CheckpointConfig checkpointConf = env.getCheckpointConfig(); checkpointConf.setMinPauseBetweenCheckpoints(30000L); checkpointConf.setCheckpointTimeout(10000L); checkpointConf.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); env.fromElements(1L,2L,3L,4L,5L,1L,3L,4L,5L,6L,7L,1L,4L,5L,3L,9L,9L,2L,1L) .flatMap(new CountWithOperatorState()) .addSink(new SinkFunction<String>() { @Override public void invoke(String s) throws Exception { } }); }
@SuppressWarnings("serial") private static SimpleEdgeStream<Long, NullValue> getGraphStream(StreamExecutionEnvironment env) { if (fileOutput) { return new SimpleEdgeStream<>(env.readTextFile(edgeInputPath) .map(new MapFunction<String, Edge<Long, Long>>() { @Override public Edge<Long, Long> map(String s) { String[] fields = s.split("\\s"); long src = Long.parseLong(fields[0]); long trg = Long.parseLong(fields[1]); long timestamp = Long.parseLong(fields[2]); return new Edge<>(src, trg, timestamp); } }), new EdgeValueTimestampExtractor(), env).mapEdges(new RemoveEdgeValue()); } return new SimpleEdgeStream<>(env.generateSequence(1, 10).flatMap( new FlatMapFunction<Long, Edge<Long, Long>>() { @Override public void flatMap(Long key, Collector<Edge<Long, Long>> out) throws Exception { for (int i = 1; i < 3; i++) { long target = key + i; out.collect(new Edge<>(key, target, key*100 + (i-1)*50)); } } }), new EdgeValueTimestampExtractor(), env).mapEdges(new RemoveEdgeValue()); }
@Override public Object doOperation(final AddElementsFromKafka op, final Context context, final Store store) throws OperationException { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); if (null != op.getParallelism()) { env.setParallelism(op.getParallelism()); } final GafferMapFunction function = new GafferMapFunction(op.getConsumeAs(), op.getElementGenerator()); final DataStream<Element> builder = env.addSource( new FlinkKafkaConsumer010<>( op.getTopic(), function.getSerialisationType(), createFlinkProperties(op))) .flatMap(function); if (Boolean.parseBoolean(op.getOption(FlinkConstants.SKIP_REBALANCING))) { builder.addSink(new GafferSink(op, store)); } else { builder.rebalance().addSink(new GafferSink(op, store)); } try { env.execute(op.getClass().getSimpleName() + "-" + op.getGroupId() + "-" + op.getTopic()); } catch (final Exception e) { throw new OperationException("Failed to add elements from Kafka topic: " + op.getTopic(), e); } return null; }
@SuppressWarnings("serial") private static SimpleEdgeStream<Integer, NullValue> getGraphStream(StreamExecutionEnvironment env) { if (fileOutput) { return new SimpleEdgeStream<>(env.readTextFile(edgeInputPath) .flatMap(new FlatMapFunction<String, Edge<Integer, NullValue>>() { @Override public void flatMap(String s, Collector<Edge<Integer, NullValue>> out) { String[] fields = s.split("\\s"); if (!fields[0].equals("%")) { int src = Integer.parseInt(fields[0]); int trg = Integer.parseInt(fields[1]); out.collect(new Edge<>(src, trg, NullValue.getInstance())); } } }), env); } return new SimpleEdgeStream<>(env.fromElements( new Edge<>(1, 2, NullValue.getInstance()), new Edge<>(2, 3, NullValue.getInstance()), new Edge<>(2, 6, NullValue.getInstance()), new Edge<>(5, 6, NullValue.getInstance()), new Edge<>(1, 4, NullValue.getInstance()), new Edge<>(5, 3, NullValue.getInstance()), new Edge<>(3, 4, NullValue.getInstance()), new Edge<>(3, 6, NullValue.getInstance()), new Edge<>(1, 3, NullValue.getInstance())), env); } }