JavaDStream<String> ipAddressDStream = ipCumDStream.transform( new Function<JavaPairRDD<String, Long>, JavaRDD<String>>() { public JavaRDD<String> call(JavaPairRDD<String, Long> rdd) {
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Override protected SparkCollection<Object> mergeJoinResults( StageSpec stageSpec, SparkPairCollection<Object, List<JoinElement<Object>>> joinedInputs, StageStatisticsCollector collector) throws Exception { DynamicDriverContext dynamicDriverContext = new DynamicDriverContext(stageSpec, sec, collector); JavaPairDStream<Object, List<JoinElement<Object>>> pairDStream = joinedInputs.getUnderlying(); JavaDStream<Object> result = pairDStream.transform(new DynamicJoinMerge<>(dynamicDriverContext)); return new DStreamCollection<>(sec, result); } }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@Override public SparkCollection<RecordInfo<Object>> aggregate(StageSpec stageSpec, @Nullable Integer partitions, StageStatisticsCollector collector) { DynamicDriverContext dynamicDriverContext = new DynamicDriverContext(stageSpec, sec, collector); JavaPairDStream<Object, T> keyedCollection = stream.transformToPair(new DynamicAggregatorGroupBy<Object, T>(dynamicDriverContext)); JavaPairDStream<Object, Iterable<T>> groupedCollection = partitions == null ? keyedCollection.groupByKey() : keyedCollection.groupByKey(partitions); return wrap(groupedCollection.transform(new DynamicAggregatorAggregate<Object, T, Object>(dynamicDriverContext))); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testPairToNormalRDDTransform() { List<List<Tuple2<Integer, Integer>>> inputData = Arrays.asList( Arrays.asList( new Tuple2<>(3, 5), new Tuple2<>(1, 5), new Tuple2<>(4, 5), new Tuple2<>(2, 5)), Arrays.asList( new Tuple2<>(2, 5), new Tuple2<>(3, 5), new Tuple2<>(4, 5), new Tuple2<>(1, 5))); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3,1,4,2), Arrays.asList(2,3,4,1)); JavaDStream<Tuple2<Integer, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<Integer, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> firstParts = pairStream.transform(in -> in.map(in2 -> in2._1())); JavaTestUtils.attachTestOutputStream(firstParts); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testPairToNormalRDDTransform() { List<List<Tuple2<Integer, Integer>>> inputData = Arrays.asList( Arrays.asList( new Tuple2<>(3, 5), new Tuple2<>(1, 5), new Tuple2<>(4, 5), new Tuple2<>(2, 5)), Arrays.asList( new Tuple2<>(2, 5), new Tuple2<>(3, 5), new Tuple2<>(4, 5), new Tuple2<>(1, 5))); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3, 1, 4, 2), Arrays.asList(2, 3, 4, 1)); JavaDStream<Tuple2<Integer, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<Integer, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> firstParts = pairStream.transform(in -> in.map(x -> x._1())); JavaTestUtils.attachTestOutputStream(firstParts); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@Test public void testPairToNormalRDDTransform() { List<List<Tuple2<Integer, Integer>>> inputData = Arrays.asList( Arrays.asList( new Tuple2<>(3, 5), new Tuple2<>(1, 5), new Tuple2<>(4, 5), new Tuple2<>(2, 5)), Arrays.asList( new Tuple2<>(2, 5), new Tuple2<>(3, 5), new Tuple2<>(4, 5), new Tuple2<>(1, 5))); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3, 1, 4, 2), Arrays.asList(2, 3, 4, 1)); JavaDStream<Tuple2<Integer, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<Integer, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> firstParts = pairStream.transform(in -> in.map(x -> x._1())); JavaTestUtils.attachTestOutputStream(firstParts); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testPairToNormalRDDTransform() { List<List<Tuple2<Integer, Integer>>> inputData = Arrays.asList( Arrays.asList( new Tuple2<>(3, 5), new Tuple2<>(1, 5), new Tuple2<>(4, 5), new Tuple2<>(2, 5)), Arrays.asList( new Tuple2<>(2, 5), new Tuple2<>(3, 5), new Tuple2<>(4, 5), new Tuple2<>(1, 5))); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3,1,4,2), Arrays.asList(2,3,4,1)); JavaDStream<Tuple2<Integer, Integer>> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<Integer, Integer> pairStream = JavaPairDStream.fromJavaDStream(stream); JavaDStream<Integer> firstParts = pairStream.transform(in -> in.map(in2 -> in2._1())); JavaTestUtils.attachTestOutputStream(firstParts); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 2, 2); Assert.assertEquals(expected, result); }
.transform(new Function<JavaPairRDD<Integer, String>, JavaPairRDD<Integer, String>>() { @Override public JavaPairRDD<Integer, String> call(