/** * Internally used to set the job ID after instantiation. */ public void setJobId(String id) throws Exception { super.setJobId(id); table += id; }
public void close() throws Exception { committer.close(); }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
public GenericWriteAheadSink( CheckpointCommitter committer, TypeSerializer<IN> serializer, String jobID) throws Exception { this.committer = Preconditions.checkNotNull(committer); this.serializer = Preconditions.checkNotNull(serializer); this.id = UUID.randomUUID().toString(); this.committer.setJobId(jobID); this.committer.createResource(); }
@Override public void open() throws Exception { super.open(); currentWatermark = Long.MIN_VALUE; watermarkInterval = getExecutionConfig().getAutoWatermarkInterval(); if (watermarkInterval > 0) { long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); } }
@Override public void open() throws Exception { super.open(); watermarkInterval = getExecutionConfig().getAutoWatermarkInterval(); if (watermarkInterval > 0) { long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); } currentWatermark = Long.MIN_VALUE; }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
@Test public void testZeroOnNegativeTimestamps() throws Exception { final AssignerWithPunctuatedWatermarks<Long> assigner = new NeverWatermarkExtractor(); final TimestampsAndPunctuatedWatermarksOperator<Long> operator = new TimestampsAndPunctuatedWatermarksOperator<Long>(assigner); OneInputStreamOperatorTestHarness<Long, Long> testHarness = new OneInputStreamOperatorTestHarness<Long, Long>(operator); testHarness.open(); long[] values = { Long.MIN_VALUE, -1L, 0L, 1L, 2L, 3L, Long.MAX_VALUE }; for (long value : values) { testHarness.processElement(new StreamRecord<>(value)); } ConcurrentLinkedQueue<Object> output = testHarness.getOutput(); for (long value: values) { assertEquals(value, ((StreamRecord<?>) output.poll()).getTimestamp()); } }
@Override public void onProcessingTime(long timestamp) throws Exception { // register next timer long newWatermark = userFunction.getCurrentWatermark(); if (newWatermark > currentWatermark) { currentWatermark = newWatermark; // emit watermark output.emitWatermark(new Watermark(currentWatermark)); } long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); }
@Override public void onProcessingTime(long timestamp) throws Exception { // register next timer Watermark newWatermark = userFunction.getCurrentWatermark(); if (newWatermark != null && newWatermark.getTimestamp() > currentWatermark) { currentWatermark = newWatermark.getTimestamp(); // emit watermark output.emitWatermark(newWatermark); } long now = getProcessingTimeService().getCurrentProcessingTime(); getProcessingTimeService().registerTimer(now + watermarkInterval, this); }
@Test public void testNoMaxWatermarkOnImmediateCancel() throws Exception { final List<StreamElement> output = new ArrayList<>(); // regular stream source operator final StreamSource<String, InfiniteSource<String>> operator = new StreamSource<>(new InfiniteSource<String>()); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); operator.cancel(); // run and exit operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); assertTrue(output.isEmpty()); }
@Test public void testEmitMaxWatermarkForFiniteSource() throws Exception { // regular stream source operator StreamSource<String, FiniteSource<String>> operator = new StreamSource<>(new FiniteSource<String>()); final List<StreamElement> output = new ArrayList<>(); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); assertEquals(1, output.size()); assertEquals(Watermark.MAX_WATERMARK, output.get(0)); }
@Test public void testMultiChainingWithObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableObjectReuse(); testMultiChaining(env); }
@Test public void testMultiChainingWithSplitWithObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableObjectReuse(); testMultiChainingWithSplit(env); }
@Override protected ListSink createSink() throws Exception { return new ListSink(); }
@Override public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException { FileInputSplit[] splits = new FileInputSplit[minNumSplits]; for (int i = 0; i < minNumSplits; i++) { splits[i] = new FileInputSplit(i, getFilePaths()[0], i * linesPerSplit + 1, linesPerSplit, null); } return splits; }
@Override public String nextRecord(String reuse) throws IOException { return reachedEnd() ? null : split.getSplitNumber() + ": test line " + state++; } }
@Test public void testNoMaxWatermarkOnImmediateStop() throws Exception { final List<StreamElement> output = new ArrayList<>(); // regular stream source operator final StoppableStreamSource<String, InfiniteSource<String>> operator = new StoppableStreamSource<>(new InfiniteSource<String>()); setupSourceOperator(operator, TimeCharacteristic.EventTime, 0); operator.stop(); // run and stop operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output)); assertTrue(output.isEmpty()); }
@Test public void testMultiChainingWithoutObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse(); testMultiChaining(env); }
@Test public void testMultiChainingWithSplitWithoutObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse(); testMultiChainingWithSplit(env); }