out.collect(new Tuple2<>(value, 1)); }).keyBy(0).sum(1).print();
@Test(expected = CompositeType.InvalidFieldReferenceException.class) public void testFailOnNestedPojoFieldAccessor() throws Exception { StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Data> dataStream = see.fromCollection(elements); dataStream.keyBy("aaa", "stats.count").sum("stats.nonExistingField"); }
.keyBy(0).sum(1);
.keyBy(0).sum(1);
@Test public void testProgram() throws Exception { String resultPath = getTempDirPath("result"); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<String> text = env.fromElements(WordCountData.TEXT); DataStream<Tuple2<String, Integer>> counts = text .flatMap(new Tokenizer()) .keyBy(0).sum(1); counts.writeAsCsv(resultPath); env.execute("WriteAsCsvTest"); //Strip the parentheses from the expected text like output compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES .replaceAll("[\\\\(\\\\)]", ""), resultPath); }
@Test public void testProgram() throws Exception { String resultPath = getTempDirPath("result"); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<String> text = env.fromElements(WordCountData.TEXT); DataStream<Tuple2<String, Integer>> counts = text .flatMap(new Tokenizer()) .keyBy(0).sum(1); counts.writeAsText(resultPath); env.execute("WriteAsTextTest"); compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES, resultPath); }
@Test public void testNettyEpoll() throws Exception { MiniClusterWithClientResource cluster = trySetUpCluster(); try { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(NUM_TASK_MANAGERS); env.getConfig().disableSysoutLogging(); DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42); input.keyBy(new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) throws Exception { return value; } }) .sum(0) .print(); env.execute(); } finally { cluster.after(); } }
@Test public void testNestedPojoFieldAccessor() throws Exception { StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); see.getConfig().disableObjectReuse(); see.setParallelism(4); DataStream<Data> dataStream = see.fromCollection(elements); DataStream<Data> summedStream = dataStream .keyBy("aaa") .sum("stats.count") .keyBy("aaa") .flatMap(new FlatMapFunction<Data, Data>() { Data[] first = new Data[3]; @Override public void flatMap(Data value, Collector<Data> out) throws Exception { if (first[value.aaa] == null) { first[value.aaa] = value; if (value.stats.count != 123) { throw new RuntimeException("Expected stats.count to be 123"); } } else { if (value.stats.count != 2 * 123) { throw new RuntimeException("Expected stats.count to be 2 * 123"); } } } }); summedStream.print(); see.execute(); }
.sum("sum") .keyBy("aaa", "stats.count") .flatMap(new FlatMapFunction<Data, Data>() {
.sum("sum") .keyBy("aaa", "abc", "wxyz") .flatMap(new FlatMapFunction<Data, Data>() {
out.collect(new Tuple2<>(value, 1)); }).keyBy(0).sum(1).print();
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<String, Integer>> dataStream = env .fromElements("Who's there?", "I think I hear them. Stand, ho! Who's there?") //.socketTextStream("localhost", 9999) .flatMap(new Splitter()) .keyBy(0) .sum(1); dataStream.print(); env.execute(); //env.execute("Socket Stream WordCount"); }
.keyBy(0).sum(1);
DataStream<Tuple2<Long, Long>> rideCounts = keyedByDriverId.sum(1);
build()). keyBy(0, 2, 3). sum(1). setParallelism(groupbyParallelism). setResources(