@Override protected void run() throws Exception { headOperator.run(getCheckpointLock(), getStreamStatusMaintainer()); }
@Override public void run(Object lockingObject, StreamStatusMaintainer streamStatusMaintainer, Output<StreamRecord<OUT>> collector) throws Exception { ACTUAL_ORDER_TRACKING.add("OPERATOR::run"); super.run(lockingObject, streamStatusMaintainer, collector); runStarted.trigger(); runFinish.await(); }
@Test public void testEmitMaxWatermarkForFiniteSource() throws Exception { // regular stream source operator StreamSource<String, FiniteSource<String>> operator = new StreamSource<>(new FiniteSource<String>()); final List<StreamElement> output = new ArrayList<>(); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); assertEquals(1, output.size()); assertEquals(Watermark.MAX_WATERMARK, output.get(0)); }
@Test public void testNoMaxWatermarkOnImmediateCancel() throws Exception { final List<StreamElement> output = new ArrayList<>(); // regular stream source operator final StreamSource<String, InfiniteSource<String>> operator = new StreamSource<>(new InfiniteSource<String>()); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); operator.cancel(); // run and exit operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); assertTrue(output.isEmpty()); }
@Test public void testNoMaxWatermarkOnAsyncCancel() throws Exception { final List<StreamElement> output = new ArrayList<>(); final Thread runner = Thread.currentThread(); // regular stream source operator final StreamSource<String, InfiniteSource<String>> operator = new StreamSource<>(new InfiniteSource<String>()); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); // trigger an async cancel in a bit new Thread("canceler") { @Override public void run() { try { Thread.sleep(200); } catch (InterruptedException ignored) {} operator.cancel(); runner.interrupt(); } }.start(); // run and wait to be canceled try { operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); } catch (InterruptedException ignored) {} assertTrue(output.isEmpty()); }
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception { final List<StreamElement> output = new ArrayList<>(); final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService(); testProcessingTimeService.setCurrentTime(0L); final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime); // regular stream source operator final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes)); operatorSetup.setupSourceOperator(operator, testProcessingTimeService); // run and wait to be stopped operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output)); assertEquals( numberLatencyMarkers + 1, // + 1 is the final watermark element output.size()); long timestamp = 0L; int i = 0; // verify that its only latency markers + a final watermark for (; i < numberLatencyMarkers; i++) { StreamElement se = output.get(i); Assert.assertTrue(se.isLatencyMarker()); Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId()); Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex()); Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp); timestamp += latencyMarkInterval; } Assert.assertTrue(output.get(i).isWatermark()); }
public void run(final Object lockingObject, final StreamStatusMaintainer streamStatusMaintainer, final Output<StreamRecord<OUT>> collector) throws Exception { final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval() : configuration.getLong(MetricOptions.LATENCY_INTERVAL); LatencyMarksEmitter<OUT> latencyEmitter = null; if (latencyTrackingInterval > 0) { latencyEmitter = new LatencyMarksEmitter<>( getProcessingTimeService(), collector, latencyTrackingInterval, this.getOperatorID(), getRuntimeContext().getIndexOfThisSubtask()); } final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.ctx = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), lockingObject, streamStatusMaintainer, collector, watermarkInterval,
public void run(final Object lockingObject, final StreamStatusMaintainer streamStatusMaintainer) throws Exception { run(lockingObject, streamStatusMaintainer, output); }
@Override protected void run() throws Exception { headOperator.run(getCheckpointLock(), getStreamStatusMaintainer()); }
@Override protected void run() throws Exception { headOperator.run(getCheckpointLock(), getStreamStatusMaintainer()); }
@Override protected void run() throws Exception { headOperator.run(getCheckpointLock(), getStreamStatusMaintainer()); }
try { testHarness.open(); sourceOperator.run(checkpointLock, new TestStreamStatusMaintainer(), new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
try { testHarness.open(); sourceOperator.run( checkpointLock, new TestStreamStatusMaintainer(),
sourceOperator.run( testHarness.getCheckpointLock(), new TestStreamStatusMaintainer(),
sourceOperator.run(checkpointLock, new TestStreamStatusMaintainer(), new Output<StreamRecord<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>>() {
sourceOperator.run( testHarness.getCheckpointLock(), new TestStreamStatusMaintainer(),
public void run(final Object lockingObject, final StreamStatusMaintainer streamStatusMaintainer, final Output<StreamRecord<OUT>> collector) throws Exception { final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval() : configuration.getLong(MetricOptions.LATENCY_INTERVAL); LatencyMarksEmitter<OUT> latencyEmitter = null; if (latencyTrackingInterval > 0) { latencyEmitter = new LatencyMarksEmitter<>( getProcessingTimeService(), collector, latencyTrackingInterval, this.getOperatorID(), getRuntimeContext().getIndexOfThisSubtask()); } final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.ctx = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), lockingObject, streamStatusMaintainer, collector, watermarkInterval,
public void run(final Object lockingObject, final StreamStatusMaintainer streamStatusMaintainer, final Output<StreamRecord<OUT>> collector) throws Exception { final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final Configuration configuration = this.getContainingTask().getEnvironment().getTaskManagerInfo().getConfiguration(); final long latencyTrackingInterval = getExecutionConfig().isLatencyTrackingConfigured() ? getExecutionConfig().getLatencyTrackingInterval() : configuration.getLong(MetricOptions.LATENCY_INTERVAL); LatencyMarksEmitter<OUT> latencyEmitter = null; if (latencyTrackingInterval > 0) { latencyEmitter = new LatencyMarksEmitter<>( getProcessingTimeService(), collector, latencyTrackingInterval, this.getOperatorID(), getRuntimeContext().getIndexOfThisSubtask()); } final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.ctx = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), lockingObject, streamStatusMaintainer, collector, watermarkInterval,