/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
StreamStateHandle stateHandle = localOut.closeAndGetHandle();
StreamStateHandle stateHandle = localOut.closeAndGetHandle();
StreamStateHandle stateHandle = out.closeAndGetHandle();
long offset = localStream.getPos(); int numEntries = 0;
outputStream.write(buffer, 0, numBytes); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; outputStream.close();
@Override public KeyGroupsStateHandle performOperation() throws Exception { long asyncStartTime = System.currentTimeMillis(); CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(stream); serializationProxy.write(outView); long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) { int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos); keyGroupRangeOffsets[keyGroupPos] = stream.getPos(); outView.writeInt(keyGroupId); for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) { outView.writeShort(kVStateToId.get(kvState.getKey())); cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(outView, keyGroupId); } } final StreamStateHandle streamStateHandle = closeStreamAndGetStateHandle(); if (asynchronousSnapshots) { LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.", streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime)); } if (streamStateHandle == null) { return null; } KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets); final KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(offsets, streamStateHandle); return keyGroupsStateHandle; } };
@Override public void close() throws IOException { Exception exCollector = null; try { flushInternalBuffer(); } catch (Exception flushEx) { exCollector = flushEx; } try { primaryOutputStream.close(); } catch (Exception closeEx) { exCollector = ExceptionUtils.firstOrSuppressed(closeEx, exCollector); } if (secondaryStreamException == null) { try { secondaryOutputStream.close(); } catch (Exception closeEx) { handleSecondaryStreamOnException(closeEx); } } if (exCollector != null) { throw new IOException("Exception while closing duplicating stream.", exCollector); } }
@Override public void close() throws IOException { Exception exCollector = null; try { flushInternalBuffer(); } catch (Exception flushEx) { exCollector = flushEx; } try { primaryOutputStream.close(); } catch (Exception closeEx) { exCollector = ExceptionUtils.firstOrSuppressed(closeEx, exCollector); } if (secondaryStreamException == null) { try { secondaryOutputStream.close(); } catch (Exception closeEx) { handleSecondaryStreamOnException(closeEx); } } if (exCollector != null) { throw new IOException("Exception while closing duplicating stream.", exCollector); } }
/** * This is public so that we can use it when restoring a legacy snapshot * in {@code AbstractStreamOperatorTestHarness}. */ public static KeyGroupsStateHandle convertKeyedBackendState( HashMap<String, KvStateSnapshot<?, ?, ?, ?>> oldKeyedState, int parallelInstanceIdx, long checkpointID) throws Exception { if (null != oldKeyedState) { CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(MAX_SIZE); CheckpointStreamFactory.CheckpointStateOutputStream keyedStateOut = checkpointStreamFactory.createCheckpointStateOutputStream(checkpointID, 0L); try { final long offset = keyedStateOut.getPos(); InstantiationUtil.serializeObject(keyedStateOut, oldKeyedState); StreamStateHandle streamStateHandle = keyedStateOut.closeAndGetHandle(); keyedStateOut = null; // makes IOUtils.closeQuietly(...) ignore this if (null != streamStateHandle) { KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(parallelInstanceIdx, parallelInstanceIdx, new long[]{offset}); return new MigrationKeyGroupStateHandle(keyGroupRangeOffsets, streamStateHandle); } } finally { IOUtils.closeQuietly(keyedStateOut); } } return null; }
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @SuppressWarnings("deprecation") @Deprecated @Override public StreamStateHandle snapshotLegacyOperatorState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (this instanceof StreamCheckpointedOperator) { CheckpointStreamFactory factory = getCheckpointStreamFactory(checkpointOptions); final CheckpointStreamFactory.CheckpointStateOutputStream outStream = factory.createCheckpointStateOutputStream(checkpointId, timestamp); getContainingTask().getCancelables().registerClosable(outStream); try { ((StreamCheckpointedOperator) this).snapshotState(outStream, checkpointId, timestamp); return outStream.closeAndGetHandle(); } finally { getContainingTask().getCancelables().unregisterClosable(outStream); outStream.close(); } } else { return null; } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
@Override public void close() throws IOException { Exception exCollector = null; try { flushInternalBuffer(); } catch (Exception flushEx) { exCollector = flushEx; } try { primaryOutputStream.close(); } catch (Exception closeEx) { exCollector = ExceptionUtils.firstOrSuppressed(closeEx, exCollector); } if (secondaryStreamException == null) { try { secondaryOutputStream.close(); } catch (Exception closeEx) { handleSecondaryStreamOnException(closeEx); } } if (exCollector != null) { throw new IOException("Exception while closing duplicating stream.", exCollector); } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
private StreamStateHandle materializeMetaData() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; try { outputStream = checkpointStreamFactory .createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closeableRegistry.registerClosable(outputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.keySerializer, stateMetaInfoSnapshots); DataOutputView out = new DataOutputViewStreamWrapper(outputStream); serializationProxy.write(out); closeableRegistry.unregisterClosable(outputStream); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; return result; } finally { if (outputStream != null) { closeableRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
/** * This method should not be public so as to not expose internals to user code. */ @Override OperatorStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); if (null == streamStateHandle) { return null; } if (partitionOffsets.isEmpty() && delegate.getPos() > initialPosition) { startNewPartition(); } Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(1); OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo( partitionOffsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE); offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); return new OperatorStateHandle(offsetsMap, streamStateHandle); }
/** * This method should not be public so as to not expose internals to user code. */ @Override OperatorStateHandle closeAndGetHandle() throws IOException { StreamStateHandle streamStateHandle = delegate.closeAndGetHandle(); if (null == streamStateHandle) { return null; } if (partitionOffsets.isEmpty() && delegate.getPos() > initialPosition) { startNewPartition(); } Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(1); OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo( partitionOffsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE); offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); return new OperatorStreamStateHandle(offsetsMap, streamStateHandle); }