private static AbstractStreamOperatorTestHarness<Integer> createTestHarness(TestUserFunction userFunction) throws Exception { return new AbstractStreamOperatorTestHarness<>( new StreamMap<>(userFunction), 1, 1, 0); }
OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask); headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint()); headOperator.processElement(new StreamRecord<>(1)); headOperator.processElement(new StreamRecord<>(2)); headOperator.processElement(new StreamRecord<>(3));
/** * Applies a Map transformation on a {@link DataStream}. The transformation * calls a {@link MapFunction} for each element of the DataStream. Each * MapFunction call returns exactly one element. The user can also extend * {@link RichMapFunction} to gain access to other features provided by the * {@link org.apache.flink.api.common.functions.RichFunction} interface. * * @param mapper * The MapFunction that is called for each element of the * DataStream. * @param <R> * output type * @return The transformed {@link DataStream}. */ public <R> SingleOutputStreamOperator<R> map(MapFunction<T, R> mapper) { TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType(), Utils.getCallLocationName(), true); return transform("Map", outType, new StreamMap<>(clean(mapper))); }
OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask); headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint()); headOperator.processElement(new StreamRecord<>(1)); headOperator.processElement(new StreamRecord<>(2)); headOperator.processElement(new StreamRecord<>(3));
StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>()); streamConfig.setStreamOperator(mapOperator);
@Test public void testOpenClose() throws Exception { StreamMap<String, String> operator = new StreamMap<String, String>(new TestOpenCloseMapFunction()); OneInputStreamOperatorTestHarness<String, String> testHarness = new OneInputStreamOperatorTestHarness<String, String>(operator); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<String>("Hello", initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testMap() throws Exception { StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
/** * This test verifies that open() and close() are correctly called. This test also verifies * that timestamps of emitted elements are correct. {@link StreamMap} assigns the input * timestamp to emitted elements. */ @Test public void testOpenCloseAndTimestamps() throws Exception { final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>( OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); StreamConfig streamConfig = testHarness.getStreamConfig(); StreamMap<String, String> mapOperator = new StreamMap<String, String>(new TestOpenCloseMapFunction()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID()); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.invoke(); testHarness.waitForTaskRunning(); testHarness.processElement(new StreamRecord<String>("Hello", initialTime + 1)); testHarness.processElement(new StreamRecord<String>("Ciao", initialTime + 2)); expectedOutput.add(new StreamRecord<String>("Hello", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("Ciao", initialTime + 2)); testHarness.endInput(); testHarness.waitForTaskCompletion(); assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFun()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFun innerTestFun = new InnerTestFun(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
@Test public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFunList()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFunList innerTestFun = new InnerTestFunList(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
StreamMap<String, String> mapOperator = new StreamMap<>(new StreamTaskTimerTest.DummyMapFunction<String>()); streamConfig.setStreamOperator(mapOperator); streamConfig.setOperatorID(new OperatorID());
/** * Applies a Map transformation on a {@link DataStream}. The transformation * calls a {@link MapFunction} for each element of the DataStream. Each * MapFunction call returns exactly one element. The user can also extend * {@link RichMapFunction} to gain access to other features provided by the * {@link org.apache.flink.api.common.functions.RichFunction} interface. * * @param mapper * The MapFunction that is called for each element of the * DataStream. * @param <R> * output type * @return The transformed {@link DataStream}. */ public <R> SingleOutputStreamOperator<R> map(MapFunction<T, R> mapper) { TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType(), Utils.getCallLocationName(), true); return transform("Map", outType, new StreamMap<>(clean(mapper))); }
/** * Applies a Map transformation on a {@link DataStream}. The transformation * calls a {@link MapFunction} for each element of the DataStream. Each * MapFunction call returns exactly one element. The user can also extend * {@link RichMapFunction} to gain access to other features provided by the * {@link org.apache.flink.api.common.functions.RichFunction} interface. * * @param mapper * The MapFunction that is called for each element of the * DataStream. * @param <R> * output type * @return The transformed {@link DataStream}. */ public <R> SingleOutputStreamOperator<R> map(MapFunction<T, R> mapper) { TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType(), Utils.getCallLocationName(), true); return transform("Map", outType, new StreamMap<>(clean(mapper))); }
/** * Applies a Map transformation on a {@link DataStream}. The transformation * calls a {@link MapFunction} for each element of the DataStream. Each * MapFunction call returns exactly one element. The user can also extend * {@link RichMapFunction} to gain access to other features provided by the * {@link org.apache.flink.api.common.functions.RichFunction} interface. * * @param mapper * The MapFunction that is called for each element of the * DataStream. * @param <R> * output type * @return The transformed {@link DataStream}. */ public <R> SingleOutputStreamOperator<R> map(MapFunction<T, R> mapper) { TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType(), Utils.getCallLocationName(), true); return transform("Map", outType, new StreamMap<>(clean(mapper))); }