@Override public SparkCollection<RecordInfo<Object>> multiOutputTransform(StageSpec stageSpec, StageStatisticsCollector collector) { return wrap(stream.transform(new DynamicTransform<T>(new DynamicDriverContext(stageSpec, sec, collector), true))); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); JavaDStream<Integer> transformed1 = stream.transform(in -> null); JavaDStream<Integer> transformed2 = stream.transform((x, time) -> null); JavaPairDStream<String, Integer> transformed3 = stream.transformToPair(x -> null); JavaPairDStream<String, Integer> transformed4 = stream.transformToPair((x, time) -> null); JavaDStream<Integer> pairTransformed1 = pairStream.transform(x -> null); JavaDStream<Integer> pairTransformed2 = pairStream.transform((x, time) -> null); JavaPairDStream<String, String> pairTransformed3 = pairStream.transformToPair(x -> null); JavaPairDStream<String, String> pairTransformed4 = pairStream.transformToPair((x, time) -> null); }
@Override public SparkCollection<RecordInfo<Object>> transform(StageSpec stageSpec, StageStatisticsCollector collector) { return wrap(stream.transform(new DynamicTransform<T>(new DynamicDriverContext(stageSpec, sec, collector), false))); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@Test public void testTransform() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6), Arrays.asList(7, 8, 9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3, 4, 5), Arrays.asList(6, 7, 8), Arrays.asList(9, 10, 11)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> transformed = stream.transform(in -> in.map(i -> i + 2)); JavaTestUtils.attachTestOutputStream(transformed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testVariousTransform() { // tests whether all variations of transform can be called from Java List<List<Integer>> inputData = Arrays.asList(Arrays.asList(1)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); List<List<Tuple2<String, Integer>>> pairInputData = Arrays.asList(Arrays.asList(new Tuple2<>("x", 1))); JavaPairDStream<String, Integer> pairStream = JavaPairDStream.fromJavaDStream( JavaTestUtils.attachTestInputStream(ssc, pairInputData, 1)); stream.transform(in -> null); stream.transform((in, time) -> null); stream.transformToPair(in -> null); stream.transformToPair((in, time) -> null); pairStream.transform(in -> null); pairStream.transform((in, time) -> null); pairStream.transformToPair(in -> null); pairStream.transformToPair((in, time) -> null); }
@SuppressWarnings("unchecked") @Test public void testTransform() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3,4,5), Arrays.asList(6,7,8), Arrays.asList(9,10,11)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> transformed = stream.transform(in -> in.map(i -> i + 2)); JavaTestUtils.attachTestOutputStream(transformed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); assertOrderInvariantEquals(expected, result); }
@Override public SparkCollection<T> window(StageSpec stageSpec, Windower windower) { String stageName = stageSpec.getName(); return wrap(stream.transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.in", null)) .window(Durations.seconds(windower.getWidth()), Durations.seconds(windower.getSlideInterval())) .transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.out", sec.getDataTracer(stageName)))); }
@Test public void testTransform() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6), Arrays.asList(7, 8, 9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3, 4, 5), Arrays.asList(6, 7, 8), Arrays.asList(9, 10, 11)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> transformed = stream.transform(in -> in.map(i -> i + 2)); JavaTestUtils.attachTestOutputStream(transformed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testTransform() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(3,4,5), Arrays.asList(6,7,8), Arrays.asList(9,10,11)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> transformed = stream.transform(in -> in.map(i -> i + 2)); JavaTestUtils.attachTestOutputStream(transformed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); assertOrderInvariantEquals(expected, result); }
@Override public <U> SparkCollection<U> compute(final StageSpec stageSpec, SparkCompute<T, U> compute) throws Exception { final SparkCompute<T, U> wrappedCompute = new DynamicSparkCompute<>(new DynamicDriverContext(stageSpec, sec, new NoopStageStatisticsCollector()), compute); Transactionals.execute(sec, new TxRunnable() { @Override public void run(DatasetContext datasetContext) throws Exception { PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec); SparkExecutionPluginContext sparkPluginContext = new BasicSparkExecutionPluginContext(sec, JavaSparkContext.fromSparkContext(stream.context().sparkContext()), datasetContext, pipelineRuntime, stageSpec); wrappedCompute.initialize(sparkPluginContext); } }, Exception.class); return wrap(stream.transform(new ComputeTransformFunction<>(sec, stageSpec, wrappedCompute))); }
if (dataTracer.isEnabled()) { javaDStream = javaDStream.transform(new LimitingFunction<>(spec.getNumOfRecordsPreview())); .transform(new CountingTransformFunction<>(stageSpec.getName(), sec.getMetrics(), "records.out", dataTracer)) .map(new WrapOutputTransformFunction<>(stageSpec.getName())); return new DStreamCollection<>(sec, outputDStream);