/** * Applies the given {@link ProcessFunction} on the input stream, thereby * creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. * * @param processFunction The {@link ProcessFunction} that is called for each element * in the stream. * @param outputType {@link TypeInformation} for the result type of the function. * * @param <R> The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( ProcessFunction<T, R> processFunction, TypeInformation<R> outputType) { ProcessOperator<T, R> operator = new ProcessOperator<>(clean(processFunction)); return transform("Process", outputType, operator); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
public void testSideOutput() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new SideOutputProcessFunction());
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Override public void open() throws Exception { super.open(); collector = new TimestampedCollector<>(output); context = new ContextImpl(userFunction, getProcessingTimeService()); }
@Test public void testNullOutputTagRefusal() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
/** * Applies the given {@link ProcessFunction} on the input stream, thereby * creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. * * @param processFunction The {@link ProcessFunction} that is called for each element * in the stream. * @param outputType {@link TypeInformation} for the result type of the function. * * @param <R> The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( ProcessFunction<T, R> processFunction, TypeInformation<R> outputType) { ProcessOperator<T, R> operator = new ProcessOperator<>(clean(processFunction)); return transform("Process", outputType, operator); }
/** * Applies the given {@link ProcessFunction} on the input stream, thereby * creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. * * @param processFunction The {@link ProcessFunction} that is called for each element * in the stream. * @param outputType {@link TypeInformation} for the result type of the function. * * @param <R> The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( ProcessFunction<T, R> processFunction, TypeInformation<R> outputType) { ProcessOperator<T, R> operator = new ProcessOperator<>(clean(processFunction)); return transform("Process", outputType, operator); }
/** * Applies the given {@link ProcessFunction} on the input stream, thereby * creating a transformed output stream. * * <p>The function will be called for every element in the input streams and can produce zero * or more output elements. * * @param processFunction The {@link ProcessFunction} that is called for each element * in the stream. * @param outputType {@link TypeInformation} for the result type of the function. * * @param <R> The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. */ @Internal public <R> SingleOutputStreamOperator<R> process( ProcessFunction<T, R> processFunction, TypeInformation<R> outputType) { ProcessOperator<T, R> operator = new ProcessOperator<>(clean(processFunction)); return transform("Process", outputType, operator); }