protected void closeCurrentStream() { closableRegistry.unregisterClosable(currentStream); IOUtils.closeQuietly(currentStream); currentStream = null; }
@Override public void done(boolean canceled) { if (checkStreamOpenAndDoTransitionToClose()) { CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); if (stream != null) { closeStreamOnCancelRegistry.unregisterClosable(stream); IOUtils.closeQuietly(stream); } } }
private <T extends StreamStateHandle> T closeAndUnregisterStreamToObtainStateHandle( NonClosingCheckpointOutputStream<T> stream) throws IOException { if (null == stream) { return null; } closableRegistry.unregisterClosable(stream.getDelegate()); // for now we only support synchronous writing return stream.closeAndGetHandle(); }
private KeyGroupsStateHandle closeSnapshotStreamAndGetHandle() throws IOException { stateBackend.cancelStreamRegistry.unregisterClosable(outStream); StreamStateHandle stateHandle = outStream.closeAndGetHandle(); outStream = null; return stateHandle != null ? new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle) : null; }
void releaseResources(boolean canceled) { stateBackend.cancelStreamRegistry.unregisterClosable(closeableRegistry); if (backupPath != null) { try { if (backupFileSystem.exists(backupPath)) { backupFileSystem.delete(backupPath, true); } } catch (Exception e) { LOG.warn("Could not properly delete the checkpoint directory.", e); } } if (canceled) { List<StateObject> statesToDiscard = new ArrayList<>(); statesToDiscard.add(metaStateHandle); statesToDiscard.addAll(miscFiles.values()); statesToDiscard.addAll(sstFiles.values()); try { StateUtil.bestEffortDiscardAllStateObjects(statesToDiscard); } catch (Exception e) { LOG.warn("Could not properly discard states.", e); } } } }
protected StreamStateHandle closeStreamAndGetStateHandle() throws IOException { if (checkStreamOpenAndDoTransitionToClose()) { final CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); try { return stream.closeAndGetHandle(); } finally { closeStreamOnCancelRegistry.unregisterClosable(stream); } } else { throw new IOException("Checkpoint stream already closed."); } }
@Override public boolean hasNext() { if (null != offsets && offPos < offsets.length) { return true; } closeCurrentStream(); while (stateHandleIterator.hasNext()) { currentStateHandle = stateHandleIterator.next(); OperatorStateHandle.StateMetaInfo metaInfo = currentStateHandle.getStateNameToPartitionOffsets().get(stateName); if (null != metaInfo) { long[] metaOffsets = metaInfo.getOffsets(); if (null != metaOffsets && metaOffsets.length > 0) { this.offsets = metaOffsets; this.offPos = 0; closableRegistry.unregisterClosable(currentStream); IOUtils.closeQuietly(currentStream); currentStream = null; return true; } } } return false; }
private <T extends StreamStateHandle> void closeAndUnregisterStream(NonClosingCheckpointOutputStream<T> stream) throws IOException { Preconditions.checkNotNull(stream); closableRegistry.unregisterClosable(stream.getDelegate()); stream.getDelegate().close(); }
private StreamStateHandle materializeMetaData() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; try { outputStream = checkpointStreamFactory .createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closeableRegistry.registerClosable(outputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.keySerializer, stateMetaInfoSnapshots); DataOutputView out = new DataOutputViewStreamWrapper(outputStream); serializationProxy.write(out); closeableRegistry.unregisterClosable(outputStream); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; return result; } finally { if (outputStream != null) { closeableRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
closeableRegistry.unregisterClosable(outputStream); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; } finally { if (inputStream != null) { closeableRegistry.unregisterClosable(inputStream); inputStream.close(); closeableRegistry.unregisterClosable(outputStream); outputStream.close();
closeStreamOnCancelRegistry.unregisterClosable(in); IOUtils.closeQuietly(in);
private void readStateData( Path restoreFilePath, StreamStateHandle remoteFileHandle) throws IOException { FileSystem restoreFileSystem = restoreFilePath.getFileSystem(); FSDataInputStream inputStream = null; FSDataOutputStream outputStream = null; try { inputStream = remoteFileHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE); stateBackend.cancelStreamRegistry.registerClosable(outputStream); byte[] buffer = new byte[8 * 1024]; while (true) { int numBytes = inputStream.read(buffer); if (numBytes == -1) { break; } outputStream.write(buffer, 0, numBytes); } } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } if (outputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @Deprecated private void restoreStreamCheckpointed(OperatorStateHandles stateHandles) throws Exception { StreamStateHandle state = stateHandles.getLegacyOperatorState(); if (null != state) { if (this instanceof CheckpointedRestoringOperator) { LOG.debug("Restore state of task {} in chain ({}).", stateHandles.getOperatorChainIndex(), getContainingTask().getName()); FSDataInputStream is = state.openInputStream(); try { getContainingTask().getCancelables().registerClosable(is); ((CheckpointedRestoringOperator) this).restoreState(is); } finally { getContainingTask().getCancelables().unregisterClosable(is); is.close(); } } else { throw new Exception( "Found legacy operator state for operator that does not implement StreamCheckpointedOperator."); } } }
/** * Restore one key groups state handle * * @throws IOException * @throws RocksDBException * @throws ClassNotFoundException */ private void restoreKeyGroupsInStateHandle() throws IOException, StateMigrationException, RocksDBException, ClassNotFoundException { try { currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); rocksDBKeyedStateBackend.cancelStreamRegistry.registerClosable(currentStateHandleInStream); currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); restoreKVStateMetaData(); restoreKVStateData(); } finally { if (currentStateHandleInStream != null) { rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterClosable(currentStateHandleInStream); IOUtils.closeQuietly(currentStateHandleInStream); } } }
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @SuppressWarnings("deprecation") @Deprecated @Override public StreamStateHandle snapshotLegacyOperatorState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (this instanceof StreamCheckpointedOperator) { CheckpointStreamFactory factory = getCheckpointStreamFactory(checkpointOptions); final CheckpointStreamFactory.CheckpointStateOutputStream outStream = factory.createCheckpointStateOutputStream(checkpointId, timestamp); getContainingTask().getCancelables().registerClosable(outStream); try { ((StreamCheckpointedOperator) this).snapshotState(outStream, checkpointId, timestamp); return outStream.closeAndGetHandle(); } finally { getContainingTask().getCancelables().unregisterClosable(outStream); outStream.close(); } } else { return null; } }
cancelStreamRegistry.unregisterClosable(fsDataInputStream); IOUtils.closeQuietly(fsDataInputStream);
private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( serializationProxy.getKeySerializer(), UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), stateBackend.keySerializer) .isRequiresMigration()) { // TODO replace with state migration; note that key hash codes need to remain the same after migration throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + "Aborting now since state migration is currently not available"); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } } }