public static <K, V1, V2> JavaPairDStream<K, Tuple2<V1, Optional<V2>>> leftOuterJoin(JavaPairDStream<K, V1> left, JavaPairDStream<K, V2> right, int numPartitions) { return left.leftOuterJoin(right, numPartitions).mapValues(new Compat.ConvertOptional<V1, V2>()); }
public static <K, V1, V2> JavaPairDStream<K, Tuple2<V1, Optional<V2>>> leftOuterJoin(JavaPairDStream<K, V1> left, JavaPairDStream<K, V2> right) { return left.leftOuterJoin(right).mapValues(new Compat.ConvertOptional<V1, V2>()); }
@SuppressWarnings("unchecked") @Test public void testLeftOuterJoin() { List<List<Tuple2<String, String>>> stringStringKVStream1 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "dodgers"), new Tuple2<>("new york", "yankees")), Arrays.asList(new Tuple2<>("california", "sharks") )); List<List<Tuple2<String, String>>> stringStringKVStream2 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "giants") ), Arrays.asList(new Tuple2<>("new york", "islanders") ) ); List<List<Long>> expected = Arrays.asList(Arrays.asList(2L), Arrays.asList(1L)); JavaDStream<Tuple2<String, String>> stream1 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream1, 1); JavaPairDStream<String, String> pairStream1 = JavaPairDStream.fromJavaDStream(stream1); JavaDStream<Tuple2<String, String>> stream2 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream2, 1); JavaPairDStream<String, String> pairStream2 = JavaPairDStream.fromJavaDStream(stream2); JavaPairDStream<String, Tuple2<String, Optional<String>>> joined = pairStream1.leftOuterJoin(pairStream2); JavaDStream<Long> counted = joined.count(); JavaTestUtils.attachTestOutputStream(counted); List<List<Long>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testLeftOuterJoin() { List<List<Tuple2<String, String>>> stringStringKVStream1 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "dodgers"), new Tuple2<>("new york", "yankees")), Arrays.asList(new Tuple2<>("california", "sharks") )); List<List<Tuple2<String, String>>> stringStringKVStream2 = Arrays.asList( Arrays.asList(new Tuple2<>("california", "giants") ), Arrays.asList(new Tuple2<>("new york", "islanders") ) ); List<List<Long>> expected = Arrays.asList(Arrays.asList(2L), Arrays.asList(1L)); JavaDStream<Tuple2<String, String>> stream1 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream1, 1); JavaPairDStream<String, String> pairStream1 = JavaPairDStream.fromJavaDStream(stream1); JavaDStream<Tuple2<String, String>> stream2 = JavaTestUtils.attachTestInputStream( ssc, stringStringKVStream2, 1); JavaPairDStream<String, String> pairStream2 = JavaPairDStream.fromJavaDStream(stream2); JavaPairDStream<String, Tuple2<String, Optional<String>>> joined = pairStream1.leftOuterJoin(pairStream2); JavaDStream<Long> counted = joined.count(); JavaTestUtils.attachTestOutputStream(counted); List<List<Long>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }