@Override public Tuple3<Integer, Point, Long> map(Tuple2<Integer, Point> t) { return new Tuple3<>(t.f0, t.f1, 1L); } }
@Override public void run(SourceContext<Tuple3<Integer, Long, String>> ctx) throws Exception { while (ms < durationMs) { synchronized (ctx.getCheckpointLock()) { for (int i = 0; i < numKeys; i++) { ctx.collect(Tuple3.of(i, ms, "Some payload...")); } ms += idlenessMs; } Thread.sleep(idlenessMs); } }
@Override protected void deepEquals( String message, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> should, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> is) { for (int x = 0; x < should.getArity(); x++) { // Check whether field is of type Tuple2 because assertEquals must be called on the non Tuple2 fields. if(should.getField(x) instanceof Tuple2) { this.deepEquals(message, (Tuple2<?,?>) should.getField(x), (Tuple2<?,?>)is.getField(x)); } else { assertEquals(message, should.getField(x), is.getField(x)); } }// For }
@Override public Tuple3<String, String, String> map(Tuple3<String, String, String> value) throws Exception { String tmp = value.f0; for (int i = 0; i < 2; i++) { value.setField("Test", i); } Tuple3<String, String, String> tuple; if (value.f0.equals("x")) { tuple = new Tuple3<String, String, String>(tmp, value.f0, null); } else { tuple = new Tuple3<String, String, String>(tmp, value.f0, ""); } return tuple; } }
@Test public void testAggregateEventTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple3<String, String, Integer>> source = env.fromElements( Tuple3.of("hello", "hallo", 1), Tuple3.of("hello", "hallo", 2)); DataStream<Integer> window1 = source .keyBy(new Tuple3KeySelector()) .window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .aggregate(new DummyAggregationFunction()); final OneInputTransformation<Tuple3<String, String, Integer>, Integer> transform = (OneInputTransformation<Tuple3<String, String, Integer>, Integer>) window1.getTransformation(); final OneInputStreamOperator<Tuple3<String, String, Integer>, Integer> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1)); }
@Override public String map(Tuple3<Integer, Long, String> value) throws Exception { return value.getField(2); } }
@Test public void testAggregateWithWindowFunctionProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple3<String, String, Integer>> source = env.fromElements( Tuple3.of("hello", "hallo", 1), Tuple3.of("hello", "hallo", 2)); DataStream<String> window = source .keyBy(new Tuple3KeySelector()) .window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .aggregate(new DummyAggregationFunction(), new TestWindowFunction()); final OneInputTransformation<Tuple3<String, String, Integer>, String> transform = (OneInputTransformation<Tuple3<String, String, Integer>, String>) window.getTransformation(); final OneInputStreamOperator<Tuple3<String, String, Integer>, String> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor); processElementAndEnsureOutput( operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1)); }
@Override public Tuple3<String, String, String> map(Tuple3<String, String, String> value) throws Exception { int i = 0; do { value.setField("", i); i++; } while (i >= 2); return value; } }
/** * Shallow tuple copy. * @return A new Tuple with the same fields as this. */ @Override @SuppressWarnings("unchecked") public Tuple3<T0, T1, T2> copy() { return new Tuple3<>(this.f0, this.f1, this.f2); }
@Test public void testAggregateWithProcessWindowFunctionProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple3<String, String, Integer>> source = env.fromElements( Tuple3.of("hello", "hallo", 1), Tuple3.of("hello", "hallo", 2)); DataStream<String> window = source .keyBy(new Tuple3KeySelector()) .window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))) .aggregate(new DummyAggregationFunction(), new TestProcessWindowFunction()); final OneInputTransformation<Tuple3<String, String, Integer>, String> transform = (OneInputTransformation<Tuple3<String, String, Integer>, String>) window.getTransformation(); final OneInputStreamOperator<Tuple3<String, String, Integer>, String> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor); processElementAndEnsureOutput( operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1)); }
private List<Tuple3<Integer, Integer, Integer>> getInputByList() { ArrayList<Tuple3<Integer, Integer, Integer>> inputList = new ArrayList<>(); for (int i = 0; i < 9; i++) { inputList.add(Tuple3.of(0, i % 3, i)); } return inputList; }
@Override protected void deepEquals( String message, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> should, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> is) { for (int x = 0; x < should.getArity(); x++) { // Check whether field is of type Tuple2 because assertEquals must be called on the non Tuple2 fields. if(should.getField(x) instanceof Tuple2) { this.deepEquals(message, (Tuple2<?,?>) should.getField(x), (Tuple2<?,?>)is.getField(x)); } else { assertEquals(message, should.getField(x), is.getField(x)); } }// For }
@Override public Tuple3<String, Integer, Integer> join( Tuple2<String, Integer> first, Tuple2<String, Integer> second) { return new Tuple3<String, Integer, Integer>(first.f0, first.f1, second.f1); } });
@Test public void testAggregateProcessingTime() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); DataStream<Tuple3<String, String, Integer>> source = env.fromElements( Tuple3.of("hello", "hallo", 1), Tuple3.of("hello", "hallo", 2)); DataStream<Integer> window1 = source .keyBy(new Tuple3KeySelector()) .window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .aggregate(new DummyAggregationFunction()); final OneInputTransformation<Tuple3<String, String, Integer>, Integer> transform = (OneInputTransformation<Tuple3<String, String, Integer>, Integer>) window1.getTransformation(); final OneInputStreamOperator<Tuple3<String, String, Integer>, Integer> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1)); }
@Override public void run(SourceContext<Tuple3<String, String, Integer>> ctx) throws Exception { ctx.collect(Tuple3.of("a", "x", 0)); ctx.collect(Tuple3.of("a", "y", 1)); ctx.collect(Tuple3.of("a", "z", 2)); ctx.collect(Tuple3.of("b", "u", 3)); ctx.collect(Tuple3.of("b", "w", 5)); ctx.collect(Tuple3.of("a", "i", 6)); ctx.collect(Tuple3.of("a", "j", 7)); ctx.collect(Tuple3.of("a", "k", 8)); // source is finite, so it will have an implicit MAX watermark when it finishes }
@Override protected void deepEquals( String message, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> should, Tuple3<Tuple2<String, Double>, Tuple2<Long, Long>, Tuple2<Integer, Long>> is) { for (int x = 0; x < should.getArity(); x++) { // Check whether field is of type Tuple2 because assertEquals must be called on the non Tuple2 fields. if(should.getField(x) instanceof Tuple2) { this.deepEquals(message, (Tuple2<?,?>) should.getField(x), (Tuple2<?,?>)is.getField(x)); } else { assertEquals(message, should.getField(x), is.getField(x)); } }// For }
@Override public Tuple3<Integer, Long, String> map(Tuple3<Integer, Long, String> value) throws Exception { Integer incr = Integer.valueOf(value.f0.intValue() + 1); value.setField(incr, 0); return value; } }
public Tuple3WrappingCollector(Tuple3UnwrappingIterator<IN, K1, K2> tui) { this.tui = tui; this.outTuple = new Tuple3<K1, K2, IN>(); }