/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
/** * Returns the state handle from the {@link #secondaryOutputStream}. Also reports suppressed exceptions from earlier * interactions with that stream. */ public StreamStateHandle closeAndGetSecondaryHandle() throws IOException { if (secondaryStreamException == null) { flushInternalBuffer(); return secondaryOutputStream.closeAndGetHandle(); } else { throw new IOException("Secondary stream previously failed exceptionally", secondaryStreamException); } }
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { return SnapshotResult.of(outputStream.closeAndGetHandle()); }
/** * This method should not be public so as to not expose internals to user code. */ @Override OperatorStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); if (null == streamStateHandle) { return null; } if (partitionOffsets.isEmpty() && delegate.getPos() > initialPosition) { startNewPartition(); } Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(1); OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo( partitionOffsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE); offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); return new OperatorStreamStateHandle(offsetsMap, streamStateHandle); }
/** * This method should not be public so as to not expose internals to user code. */ @Override OperatorStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); if (null == streamStateHandle) { return null; } if (partitionOffsets.isEmpty() && delegate.getPos() > initialPosition) { startNewPartition(); } Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(1); OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo( partitionOffsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE); offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); return new OperatorStreamStateHandle(offsetsMap, streamStateHandle); }
/** * This method should not be public so as to not expose internals to user code. */ @Override OperatorStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); if (null == streamStateHandle) { return null; } if (partitionOffsets.isEmpty() && delegate.getPos() > initialPosition) { startNewPartition(); } Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(1); OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo( partitionOffsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE); offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); return new OperatorStateHandle(offsetsMap, streamStateHandle); }
private KeyGroupsStateHandle closeSnapshotStreamAndGetHandle() throws IOException { stateBackend.cancelStreamRegistry.unregisterClosable(outStream); StreamStateHandle stateHandle = outStream.closeAndGetHandle(); outStream = null; return stateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle) : null; }
/** * Returns the state handle from the {@link #secondaryOutputStream}. Also reports suppressed exceptions from earlier * interactions with that stream. */ public StreamStateHandle closeAndGetSecondaryHandle() throws IOException { if (secondaryStreamException == null) { flushInternalBuffer(); return secondaryOutputStream.closeAndGetHandle(); } else { throw new IOException("Secondary stream previously failed exceptionally", secondaryStreamException); } }
@Override KeyGroupsStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); return streamStateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, streamStateHandle) : null; } }
/** * Returns the state handle from the {@link #secondaryOutputStream}. Also reports suppressed exceptions from earlier * interactions with that stream. */ public StreamStateHandle closeAndGetSecondaryHandle() throws IOException { if (secondaryStreamException == null) { flushInternalBuffer(); return secondaryOutputStream.closeAndGetHandle(); } else { throw new IOException("Secondary stream previously failed exceptionally", secondaryStreamException); } }
protected StreamStateHandle closeStreamAndGetStateHandle() throws IOException { if (checkStreamOpenAndDoTransitionToClose()) { final CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); try { return stream.closeAndGetHandle(); } finally { closeStreamOnCancelRegistry.unregisterClosable(stream); } } else { throw new IOException("Checkpoint stream already closed."); } }
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { return SnapshotResult.of(outputStream.closeAndGetHandle()); }
@Nonnull @Override public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException { return SnapshotResult.of(outputStream.closeAndGetHandle()); }
/** * Returns the state handle from the {@link #primaryOutputStream}. */ public StreamStateHandle closeAndGetPrimaryHandle() throws IOException { flushInternalBuffer(); return primaryOutputStream.closeAndGetHandle(); }
@Override KeyGroupsStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); return streamStateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, streamStateHandle) : null; } }
/** * Returns the state handle from the {@link #primaryOutputStream}. */ public StreamStateHandle closeAndGetPrimaryHandle() throws IOException { flushInternalBuffer(); return primaryOutputStream.closeAndGetHandle(); }
@Override KeyGroupsStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); return streamStateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, streamStateHandle) : null; } }
@Override KeyGroupsStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); return streamStateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, streamStateHandle) : null; } }
/** * Returns the state handle from the {@link #primaryOutputStream}. */ public StreamStateHandle closeAndGetPrimaryHandle() throws IOException { flushInternalBuffer(); return primaryOutputStream.closeAndGetHandle(); }
result = outputStream.closeAndGetHandle(); outputStream = null;