@Test public void testPairMap2() { // Maps pair -> single List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Integer>> expected = Arrays.asList( Arrays.asList(1, 3, 4, 1), Arrays.asList(5, 5, 3, 1)); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> reversed = pairStream.map(Tuple2::_2); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransformWith() { // tests whether all variations of transformWith can be called from Java List<List<Integer>> inputData1 = Arrays.asList(Arrays.asList(1)); List<List<String>> inputData2 = Arrays.asList(Arrays.asList("x")); JavaDStream<Integer> stream1 = JavaTestUtils.attachTestInputStream(ssc, inputData1, 1); JavaDStream<String> stream2 = JavaTestUtils.attachTestInputStream(ssc, inputData2, 1); List<List<Tuple2<String, Integer>>> pairInputData1 = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); List<List<Tuple2<Double, Character>>> pairInputData2 = Arrays.asList(Arrays.asList(new Tuple2<>(1.0, 'x'))); JavaPairDStream<String, Integer> pairStream1 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData1, 1)); JavaPairDStream<Double, Character> pairStream2 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData2, 1)); stream1.transformWith(stream2, (rdd1, rdd2, time) -> null); stream1.transformWith(pairStream1, (rdd1, rdd2, time) -> null); stream1.transformWithToPair(stream2, (rdd1, rdd2, time) -> null); stream1.transformWithToPair(pairStream1, (rdd1, rdd2, time) -> null); pairStream1.transformWith(stream2, (rdd1, rdd2, time) -> null); pairStream1.transformWith(pairStream1, (rdd1, rdd2, time) -> null); pairStream1.transformWithToPair(stream2, (rdd1, rdd2, time) -> null); pairStream1.transformWithToPair(pairStream2, (rdd1, rdd2, time) -> null); }
@Test public void testPairMap2() { // Maps pair -> single List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Integer>> expected = Arrays.asList( Arrays.asList(1, 3, 4, 1), Arrays.asList(5, 5, 3, 1)); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> reversed = pairStream.map(Tuple2::_2); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransformWith() { // tests whether all variations of transformWith can be called from Java List<List<Integer>> inputData1 = Arrays.asList(Arrays.asList(1)); List<List<String>> inputData2 = Arrays.asList(Arrays.asList("x")); JavaDStream<Integer> stream1 = JavaTestUtils.attachTestInputStream(ssc, inputData1, 1); JavaDStream<String> stream2 = JavaTestUtils.attachTestInputStream(ssc, inputData2, 1); List<List<Tuple2<String, Integer>>> pairInputData1 = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); List<List<Tuple2<Double, Character>>> pairInputData2 = Arrays.asList(Arrays.asList(new Tuple2<>(1.0, 'x'))); JavaPairDStream<String, Integer> pairStream1 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData1, 1)); JavaPairDStream<Double, Character> pairStream2 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData2, 1)); stream1.transformWith(stream2, (rdd1, rdd2, time) -> null); stream1.transformWith(pairStream1, (rdd1, rdd2, time) -> null); stream1.transformWithToPair(stream2, (rdd1, rdd2, time) -> null); stream1.transformWithToPair(pairStream1, (rdd1, rdd2, time) -> null); pairStream1.transformWith(stream2, (rdd1, rdd2, time) -> null); pairStream1.transformWith(pairStream1, (rdd1, rdd2, time) -> null); pairStream1.transformWithToPair(stream2, (rdd1, rdd2, time) -> null); pairStream1.transformWithToPair(pairStream2, (rdd1, rdd2, time) -> null); }
@Test public void testPairReduceByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList( new Tuple2<>("california", 10), new Tuple2<>("new york", 4))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> reduced = pairStream.reduceByKey((x, y) -> x + y); JavaTestUtils.attachTestOutputStream(reduced); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testVariousTransformWith() { // tests whether all variations of transformWith can be called from Java List<List<Integer>> inputData1 = Arrays.asList(Arrays.asList(1)); List<List<String>> inputData2 = Arrays.asList(Arrays.asList("x")); JavaDStream<Integer> stream1 = JavaTestUtils.attachTestInputStream(ssc, inputData1, 1); JavaDStream<String> stream2 = JavaTestUtils.attachTestInputStream(ssc, inputData2, 1); List<List<Tuple2<String, Integer>>> pairInputData1 = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); List<List<Tuple2<Double, Character>>> pairInputData2 = Arrays.asList(Arrays.asList(new Tuple2<>(1.0, 'x'))); JavaPairDStream<String, Integer> pairStream1 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData1, 1)); JavaPairDStream<Double, Character> pairStream2 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData2, 1)); JavaDStream<Double> transformed1 = stream1.transformWith(stream2, (x, y, z) -> null); JavaDStream<Double> transformed2 = stream1.transformWith(pairStream1,(x, y, z) -> null); JavaPairDStream<Double, Double> transformed3 = stream1.transformWithToPair(stream2,(x, y, z) -> null); JavaPairDStream<Double, Double> transformed4 = stream1.transformWithToPair(pairStream1,(x, y, z) -> null); JavaDStream<Double> pairTransformed1 = pairStream1.transformWith(stream2,(x, y, z) -> null); JavaDStream<Double> pairTransformed2_ = pairStream1.transformWith(pairStream1,(x, y, z) -> null); JavaPairDStream<Double, Double> pairTransformed3 = pairStream1.transformWithToPair(stream2,(x, y, z) -> null); JavaPairDStream<Double, Double> pairTransformed4 = pairStream1.transformWithToPair(pairStream2,(x, y, z) -> null); }
@SuppressWarnings("unchecked") @Test public void testPairMap2() { // Maps pair -> single List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Integer>> expected = Arrays.asList( Arrays.asList(1, 3, 4, 1), Arrays.asList(5, 5, 3, 1)); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> reversed = pairStream.map(in -> in._2()); JavaTestUtils.attachTestOutputStream(reversed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testVariousTransformWith() { // tests whether all variations of transformWith can be called from Java List<List<Integer>> inputData1 = Arrays.asList(Arrays.asList(1)); List<List<String>> inputData2 = Arrays.asList(Arrays.asList("x")); JavaDStream<Integer> stream1 = JavaTestUtils.attachTestInputStream(ssc, inputData1, 1); JavaDStream<String> stream2 = JavaTestUtils.attachTestInputStream(ssc, inputData2, 1); List<List<Tuple2<String, Integer>>> pairInputData1 = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); List<List<Tuple2<Double, Character>>> pairInputData2 = Arrays.asList(Arrays.asList(new Tuple2<>(1.0, 'x'))); JavaPairDStream<String, Integer> pairStream1 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData1, 1)); JavaPairDStream<Double, Character> pairStream2 = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData2, 1)); JavaDStream<Double> transformed1 = stream1.transformWith(stream2, (x, y, z) -> null); JavaDStream<Double> transformed2 = stream1.transformWith(pairStream1,(x, y, z) -> null); JavaPairDStream<Double, Double> transformed3 = stream1.transformWithToPair(stream2,(x, y, z) -> null); JavaPairDStream<Double, Double> transformed4 = stream1.transformWithToPair(pairStream1,(x, y, z) -> null); JavaDStream<Double> pairTransformed1 = pairStream1.transformWith(stream2,(x, y, z) -> null); JavaDStream<Double> pairTransformed2_ = pairStream1.transformWith(pairStream1,(x, y, z) -> null); JavaPairDStream<Double, Double> pairTransformed3 = pairStream1.transformWithToPair(stream2,(x, y, z) -> null); JavaPairDStream<Double, Double> pairTransformed4 = pairStream1.transformWithToPair(pairStream2,(x, y, z) -> null); }
@Test public void testPairReduceByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList( new Tuple2<>("california", 10), new Tuple2<>("new york", 4))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> reduced = pairStream.reduceByKey((x, y) -> x + y); JavaTestUtils.attachTestOutputStream(reduced); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testPairMap2() { // Maps pair -> single List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Integer>> expected = Arrays.asList( Arrays.asList(1, 3, 4, 1), Arrays.asList(5, 5, 3, 1)); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> reversed = pairStream.map(in -> in._2()); JavaTestUtils.attachTestOutputStream(reversed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testMapValues() { List<List<Tuple2<String, String>>> inputData = stringStringKVStream; List<List<Tuple2<String, String>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", "DODGERS"), new Tuple2<>("california", "GIANTS"), new Tuple2<>("new york", "YANKEES"), new Tuple2<>("new york", "METS")), Arrays.asList(new Tuple2<>("california", "SHARKS"), new Tuple2<>("california", "DUCKS"), new Tuple2<>("new york", "RANGERS"), new Tuple2<>("new york", "ISLANDERS"))); JavaDStream<Tuple2<String, String>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, String> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, String> mapped = pairStream.mapValues(s -> s.toUpperCase(Locale.ROOT)); JavaTestUtils.attachTestOutputStream(mapped); List<List<Tuple2<String, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@Test public void testCombineByKey() { List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<String, Integer>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>("california", 4), new Tuple2<>("new york", 5)), Arrays.asList( new Tuple2<>("california", 10), new Tuple2<>("new york", 4))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, Integer> combined = pairStream.combineByKey(i -> i, (x, y) -> x + y, (x, y) -> x + y, new HashPartitioner(2)); JavaTestUtils.attachTestOutputStream(combined); List<List<Tuple2<String, Integer>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Test public void testMapValues() { List<List<Tuple2<String, String>>> inputData = stringStringKVStream; List<List<Tuple2<String, String>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", "DODGERS"), new Tuple2<>("california", "GIANTS"), new Tuple2<>("new york", "YANKEES"), new Tuple2<>("new york", "METS")), Arrays.asList(new Tuple2<>("california", "SHARKS"), new Tuple2<>("california", "DUCKS"), new Tuple2<>("new york", "RANGERS"), new Tuple2<>("new york", "ISLANDERS"))); JavaDStream<Tuple2<String, String>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, String> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, String> mapped = pairStream.mapValues(s -> s.toUpperCase(Locale.ROOT)); JavaTestUtils.attachTestOutputStream(mapped); List<List<Tuple2<String, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testPairMap() { // Maps pair -> pair of different type List<List<Tuple2<String, Integer>>> inputData = stringIntKVStream; List<List<Tuple2<Integer, String>>> expected = Arrays.asList( Arrays.asList( new Tuple2<>(1, "california"), new Tuple2<>(3, "california"), new Tuple2<>(4, "new york"), new Tuple2<>(1, "new york")), Arrays.asList( new Tuple2<>(5, "california"), new Tuple2<>(5, "california"), new Tuple2<>(3, "new york"), new Tuple2<>(1, "new york"))); JavaDStream<Tuple2<String, Integer>> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<Integer, String> reversed = pairStream.mapToPair(Tuple2::swap); JavaTestUtils.attachTestOutputStream(reversed); List<List<Tuple2<Integer, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testMapValues() { List<List<Tuple2<String, String>>> inputData = stringStringKVStream; List<List<Tuple2<String, String>>> expected = Arrays.asList( Arrays.asList(new Tuple2<>("california", "DODGERS"), new Tuple2<>("california", "GIANTS"), new Tuple2<>("new york", "YANKEES"), new Tuple2<>("new york", "METS")), Arrays.asList(new Tuple2<>("california", "SHARKS"), new Tuple2<>("california", "DUCKS"), new Tuple2<>("new york", "RANGERS"), new Tuple2<>("new york", "ISLANDERS"))); JavaDStream<Tuple2<String, String>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, String> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaPairDStream<String, String> mapped = pairStream.mapValues(s -> s.toUpperCase(Locale.ROOT)); JavaTestUtils.attachTestOutputStream(mapped); List<List<Tuple2<String, String>>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }