return Functions.responseCodeCount(rdd); }}) .updateStateByKey(new Functions.ComputeRunningSum()); responseCodeCountDStream.foreachRDD(new Function<JavaPairRDD<Integer, Long>, Void>() { public Void call(JavaPairRDD<Integer, Long> rdd) { }}); JavaPairDStream<String, Long> ipCumDStream = ipRawDStream.updateStateByKey( new Functions.ComputeRunningSum()); .updateStateByKey(new Functions.ComputeRunningSum());
@SuppressWarnings("unchecked") @Test public void testUpdateStateByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) { out += state.get(); } for (Integer v : values) { out += v; } return Optional.of(out); }); JavaTestUtils.attachTestOutputStream(updated); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testUpdateStateByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) { out += state.get(); } for (Integer v : values) { out += v; } return Optional.of(out); }); JavaTestUtils.attachTestOutputStream(updated); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
@Test public void testUpdateStateByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) { out = out + state.get(); } for (Integer v : values) { out = out + v; } return Optional.of(out); }); JavaTestUtils.attachTestOutputStream(updated); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
@Test public void testUpdateStateByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9)), Arrays.asList(new Tuple2<>("california", 14), new Tuple2<>("new york", 9))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) { out = out + state.get(); } for (Integer v : values) { out = out + v; } return Optional.of(out); }); JavaTestUtils.attachTestOutputStream(updated); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) {
JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> updated = pairStream.updateStateByKey((values, state) -> { int out = 0; if (state.isPresent()) {