synchronized (ctx.getCheckpointLock()) { long value = randomKeyRangeStates.incrementAndGet(randomKey); StringUtils.getRandomString(random, payloadLength, payloadLength, 'A', 'z')); ctx.collect(event);
@Override public void onProcessingTime(long timestamp) throws Exception { long minAcrossAll = Long.MAX_VALUE; boolean isEffectiveMinAggregation = false; for (KafkaTopicPartitionState<?> state : allPartitions) { // we access the current watermark for the periodic assigners under the state // lock, to prevent concurrent modification to any internal variables final long curr; //noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (state) { curr = ((KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?>) state).getCurrentWatermarkTimestamp(); } minAcrossAll = Math.min(minAcrossAll, curr); isEffectiveMinAggregation = true; } // emit next watermark, if there is one if (isEffectiveMinAggregation && minAcrossAll > lastWatermarkTimestamp) { lastWatermarkTimestamp = minAcrossAll; emitter.emitWatermark(new Watermark(minAcrossAll)); } // schedule the next watermark timerService.registerTimer(timerService.getCurrentProcessingTime() + interval, this); } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { for (int i = 0; i < numWatermarks; i++) { ctx.collectWithTimestamp(i, initialTime + i); ctx.emitWatermark(new Watermark(initialTime + i)); } while (running) { Thread.sleep(20); } }
public void close() { this.shouldClose = true; } }
public void close() { this.shouldClose = true; } }
@Override public void withCheckpointLock(SourceContext<String> ctx) throws Exception { ctx.emitWatermark(new Watermark(watermark)); Thread.sleep(500); } }
@Override public void run(SourceContext<Event> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(Event.of(number.get(), "test_event", random.nextDouble(), timestamp), timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
@Override public void run(SourceContext<Tuple4<Integer, String, Double, Long>> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(Tuple4.of(number.get(), "test_tuple", random.nextDouble(), timestamp), timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
@Override public void withCheckpointLock(SourceContext<String> ctx) throws Exception { ctx.collectWithTimestamp(output, timestamp); } }
@Override public void run(SourceContext<DataPoint<Long>> ctx) throws Exception { while (running) { synchronized (ctx.getCheckpointLock()) { ctx.collectWithTimestamp(new DataPoint<>(currentTimeMs, 0L), currentTimeMs); ctx.emitWatermark(new Watermark(currentTimeMs)); currentTimeMs += periodMs; } timeSync(); } }
@Override public void run(SourceContext<String> ctx) throws Exception { while (isRunning) { long timestamp = initialTimestamp + 1000 * number.get(); ctx.collectWithTimestamp(WORDS[random.nextInt(WORDS.length)], timestamp); if (number.incrementAndGet() >= this.count) { cancel(); } } }
/** * To handle data with timestamp * * @param data data received from feeder actor * @param timestamp timestamp received from feeder actor */ private void collect(Object data, long timestamp) { ctx.collectWithTimestamp(data, timestamp); }
/** * To handle data with timestamp * * @param data data received from feeder actor * @param timestamp timestamp received from feeder actor */ private void collect(Object data, long timestamp) { ctx.collectWithTimestamp(data, timestamp); }
@Override public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception { for (Tuple3<String, Long, Integer> value : input) { ctx.collectWithTimestamp(value, value.f1); ctx.emitWatermark(new Watermark(value.f1 - 1)); } ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
@Override public void onClose(Session session, CloseReason reason) { super.onClose(session, reason); try { close(); } catch (Exception e1) { LOG.error("Error closing client", e1); } // Signal done sending data ctx.emitWatermark(new Watermark(Long.MAX_VALUE)); }
switch (testMethod) { case COLLECT: context.collect("msg"); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime())); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); assertEquals(expectedOutput, output); switch (testMethod) { case COLLECT: context.collect("msg"); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive()); break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive()); break;
switch (testMethod) { case COLLECT: context.collect("msg"); break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); break; case EMIT_WATERMARK: context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime())); break; switch (testMethod) { case COLLECT: context.collect("msg"); break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); break; case EMIT_WATERMARK: context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime())); break;
ctx.emitWatermark(Watermark.MAX_WATERMARK); ctx.close(); if (latencyEmitter != null) { latencyEmitter.close();
sourceContext.collectWithTimestamp(record, timestamp); partitionState.setOffset(offset);
@Override public void run(SourceContext<Tuple2<Long, IntType>> ctx) throws Exception { final RuntimeContext runtimeContext = getRuntimeContext(); // detect if this task is "the chosen one" and should fail (via subtaskidx), if it did not fail before (via attempt) final boolean failThisTask = runtimeContext.getAttemptNumber() == 0 && runtimeContext.getIndexOfThisSubtask() == 0; // we loop longer than we have elements, to permit delayed checkpoints // to still cause a failure while (running && emitCallCount < expectedEmitCalls) { // the function failed before, or we are in the elements before the failure synchronized (ctx.getCheckpointLock()) { eventEmittingGenerator.emitEvent(ctx, emitCallCount++); } if (emitCallCount < failureAfterNumElements) { Thread.sleep(1); } else if (failThisTask && emitCallCount == failureAfterNumElements) { // wait for a pending checkpoint that fulfills our requirements if needed while (checkpointStatus.get() != STATEFUL_CHECKPOINT_COMPLETED) { Thread.sleep(1); } throw new Exception("Artificial Failure"); } } if (usingProcessingTime) { while (running) { Thread.sleep(10); } } }