@Override public void run(SourceContext<Integer> ctx) throws Exception { for (int i = 0; i < numWatermarks; i++) { ctx.collectWithTimestamp(i, initialTime + i); ctx.emitWatermark(new Watermark(initialTime + i)); } while (running) { Thread.sleep(20); } }
break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); break; case EMIT_WATERMARK: break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); break; case EMIT_WATERMARK:
@Override public void run(SourceContext<String> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(WORDS[random.nextInt(WORDS.length)], timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
@Override public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception { for (Tuple3<String, Long, Integer> value : input) { ctx.collectWithTimestamp(value, value.f1); ctx.emitWatermark(new Watermark(value.f1 - 1)); } ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<DataPoint<Long>> ctx) throws Exception { while (running) { synchronized (ctx.getCheckpointLock()) { ctx.collectWithTimestamp(new DataPoint<>(currentTimeMs, 0L), currentTimeMs); ctx.emitWatermark(new Watermark(currentTimeMs)); currentTimeMs += periodMs; } timeSync(); } }
@Override public void run(SourceContext<Event> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(Event.of(number.get(), "test_event", random.nextDouble(), timestamp), timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
@Override public void run(SourceContext<Tuple4<Integer, String, Double, Long>> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(Tuple4.of(number.get(), "test_tuple", random.nextDouble(), timestamp), timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
sourceContext.collectWithTimestamp(record, timestamp); partitionState.setOffset(offset);
/** * Record emission, if a timestamp will be attached from an assigner that is * also a punctuated watermark generator. */ private void emitRecordWithTimestampAndPunctuatedWatermark( T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) { @SuppressWarnings("unchecked") final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState = (KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState; // only one thread ever works on accessing timestamps and watermarks // from the punctuated extractor final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp); final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp); // emit the record with timestamp, using the usual checkpoint lock to guarantee // atomicity of record emission and offset state update synchronized (checkpointLock) { sourceContext.collectWithTimestamp(record, timestamp); partitionState.setOffset(offset); } // if we also have a new per-partition watermark, check if that is also a // new cross-partition watermark if (newWatermark != null) { updateMinPunctuatedWatermark(newWatermark); } }
/** * Record emission, if a timestamp will be attached from an assigner that is * also a periodic watermark generator. */ private void emitRecordWithTimestampAndPeriodicWatermark( T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) { @SuppressWarnings("unchecked") final KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH> withWatermarksState = (KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH>) partitionState; // extract timestamp - this accesses/modifies the per-partition state inside the // watermark generator instance, so we need to lock the access on the // partition state. concurrent access can happen from the periodic emitter final long timestamp; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (withWatermarksState) { timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp); } // emit the record with timestamp, using the usual checkpoint lock to guarantee // atomicity of record emission and offset state update synchronized (checkpointLock) { sourceContext.collectWithTimestamp(record, timestamp); partitionState.setOffset(offset); } }
@Override public void run(SourceContext<SessionEvent<Integer, TestEventPayload>> ctx) { ParallelSessionsEventGenerator<Integer, SessionEvent<Integer, TestEventPayload>> generator = createGenerator(); this.isRunning = true; //main data source driver loop while (isRunning) { synchronized (ctx.getCheckpointLock()) { SessionEvent<Integer, TestEventPayload> evt = generator.nextEvent(); if (evt != null) { ctx.collectWithTimestamp(evt, evt.getEventTimestamp()); ctx.emitWatermark(new Watermark(generator.getWatermark())); } else { break; } } } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { int index = 1; while (index <= numElements) { ctx.collectWithTimestamp(index, index); ctx.collectWithTimestamp(index - 1, index - 1); index++; ctx.emitWatermark(new Watermark(index - 2)); } // emit the final Long.MAX_VALUE watermark, do it twice and verify that // we only see one in the result ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Integer> ctx) throws Exception { int index = 1; while (index <= numElements) { ctx.collectWithTimestamp(index, index); ctx.collectWithTimestamp(index - 1, index - 1); index++; ctx.emitWatermark(new Watermark(index - 2)); } // emit the final Long.MAX_VALUE watermark, do it twice and verify that // we only see one in the result ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Tuple2<String, Integer>> ctx) { ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L); ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L); ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L); ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L); ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L); ctx.emitWatermark(new Watermark(5)); ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L); ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L); ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L); ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L); }
@Override public void run(SourceContext<Tuple2<String, Integer>> ctx) { ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L); ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L); ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L); ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L); ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L); ctx.emitWatermark(new Watermark(5)); ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L); ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L); ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L); ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L); }
@Override public void emitEvent(SourceFunction.SourceContext<Tuple2<Long, IntType>> ctx, int eventSequenceNo) { final IntType intTypeNext = new IntType(eventSequenceNo); for (long i = 0; i < keyUniverseSize; i++) { final Tuple2<Long, IntType> generatedEvent = new Tuple2<>(i, intTypeNext); ctx.collectWithTimestamp(generatedEvent, eventSequenceNo); } ctx.emitWatermark(new Watermark(eventSequenceNo - watermarkTrailing)); } }
@Override public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception { for (Tuple3<String, Long, Integer> value : input) { ctx.collectWithTimestamp(value, value.f1); ctx.emitWatermark(new Watermark(value.f1 - 1)); } ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void run(SourceContext<Integer> ctx) throws Exception { ctx.collectWithTimestamp(1, 0); ctx.emitWatermark(new Watermark(0)); ctx.collectWithTimestamp(2, 1); ctx.collectWithTimestamp(5, 2); ctx.emitWatermark(new Watermark(2)); ctx.collectWithTimestamp(3, 3); ctx.collectWithTimestamp(4, 4); }
@Override public void run(SourceContext<Integer> ctx) throws Exception { for (int i = 0; i < numWatermarks; i++) { ctx.collectWithTimestamp(i, initialTime + i); ctx.emitWatermark(new Watermark(initialTime + i)); } }