protected void openCurrentStream() throws IOException { FSDataInputStream stream = currentStateHandle.openInputStream(); closableRegistry.registerClosable(stream); currentStream = stream; }
private CheckpointStreamFactory.CheckpointStateOutputStream openAndRegisterNewStream() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream cout = streamFactory.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closableRegistry.registerClosable(cout); return cout; }
public void runAsyncCheckpointingAndAcknowledge() throws IOException { AsyncCheckpointRunnable asyncCheckpointRunnable = new AsyncCheckpointRunnable( owner, nonPartitionedStates, snapshotInProgressList, checkpointMetaData, checkpointMetrics, startAsyncPartNano); owner.cancelables.registerClosable(asyncCheckpointRunnable); owner.asyncOperationsThreadPool.submit(asyncCheckpointRunnable); }
@Override public CheckpointStreamFactory.CheckpointStateOutputStream openIOHandle() throws Exception { if (checkStreamClosedAndDoTransitionToOpen()) { CheckpointStreamFactory.CheckpointStateOutputStream stream = streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp); try { closeStreamOnCancelRegistry.registerClosable(stream); return stream; } catch (Exception ex) { open.set(false); throw ex; } } else { throw new IOException("Async snapshot: a checkpoint stream was already opened."); } }
/** * 2) Open CheckpointStateOutputStream through the checkpointStreamFactory into which we will write. * * @throws Exception */ public void openCheckpointStream() throws Exception { Preconditions.checkArgument(outStream == null, "Output stream for snapshot is already set."); outStream = checkpointStreamFactory. createCheckpointStateOutputStream(checkpointId, checkpointTimeStamp); stateBackend.cancelStreamRegistry.registerClosable(outStream); outputView = new DataOutputViewStreamWrapper(outStream); }
KeyedStateHandle materializeSnapshot() throws Exception { stateBackend.cancelStreamRegistry.registerClosable(closeableRegistry);
public OperatorStateBackend createOperatorStateBackend( StreamOperator<?> op, Collection<OperatorStateHandle> restoreStateHandles) throws Exception { Environment env = getEnvironment(); String opId = createOperatorIdentifier(op, getConfiguration().getVertexID()); OperatorStateBackend operatorStateBackend = stateBackend.createOperatorStateBackend(env, opId); // let operator state backend participate in the operator lifecycle, i.e. make it responsive to cancelation cancelables.registerClosable(operatorStateBackend); // restore if we have some old state if (null != restoreStateHandles) { operatorStateBackend.restore(restoreStateHandles); } return operatorStateBackend; }
closeStreamOnCancelRegistry.registerClosable(in);
private void readStateData( Path restoreFilePath, StreamStateHandle remoteFileHandle) throws IOException { FileSystem restoreFileSystem = restoreFilePath.getFileSystem(); FSDataInputStream inputStream = null; FSDataOutputStream outputStream = null; try { inputStream = remoteFileHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE); stateBackend.cancelStreamRegistry.registerClosable(outputStream); byte[] buffer = new byte[8 * 1024]; while (true) { int numBytes = inputStream.read(buffer); if (numBytes == -1) { break; } outputStream.write(buffer, 0, numBytes); } } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } if (outputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @Deprecated private void restoreStreamCheckpointed(OperatorStateHandles stateHandles) throws Exception { StreamStateHandle state = stateHandles.getLegacyOperatorState(); if (null != state) { if (this instanceof CheckpointedRestoringOperator) { LOG.debug("Restore state of task {} in chain ({}).", stateHandles.getOperatorChainIndex(), getContainingTask().getName()); FSDataInputStream is = state.openInputStream(); try { getContainingTask().getCancelables().registerClosable(is); ((CheckpointedRestoringOperator) this).restoreState(is); } finally { getContainingTask().getCancelables().unregisterClosable(is); is.close(); } } else { throw new Exception( "Found legacy operator state for operator that does not implement StreamCheckpointedOperator."); } } }
/** * Restore one key groups state handle * * @throws IOException * @throws RocksDBException * @throws ClassNotFoundException */ private void restoreKeyGroupsInStateHandle() throws IOException, StateMigrationException, RocksDBException, ClassNotFoundException { try { currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); rocksDBKeyedStateBackend.cancelStreamRegistry.registerClosable(currentStateHandleInStream); currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); restoreKVStateMetaData(); restoreKVStateData(); } finally { if (currentStateHandleInStream != null) { rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterClosable(currentStateHandleInStream); IOUtils.closeQuietly(currentStateHandleInStream); } } }
closeableRegistry.registerClosable(inputStream); closeableRegistry.registerClosable(outputStream);
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @SuppressWarnings("deprecation") @Deprecated @Override public StreamStateHandle snapshotLegacyOperatorState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (this instanceof StreamCheckpointedOperator) { CheckpointStreamFactory factory = getCheckpointStreamFactory(checkpointOptions); final CheckpointStreamFactory.CheckpointStateOutputStream outStream = factory.createCheckpointStateOutputStream(checkpointId, timestamp); getContainingTask().getCancelables().registerClosable(outStream); try { ((StreamCheckpointedOperator) this).snapshotState(outStream, checkpointId, timestamp); return outStream.closeAndGetHandle(); } finally { getContainingTask().getCancelables().unregisterClosable(outStream); outStream.close(); } } else { return null; } }
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange) throws Exception { if (keyedStateBackend != null) { throw new RuntimeException("The keyed state backend can only be created once."); } String operatorIdentifier = createOperatorIdentifier( headOperator, configuration.getVertexID()); keyedStateBackend = stateBackend.createKeyedStateBackend( getEnvironment(), getEnvironment().getJobID(), operatorIdentifier, keySerializer, numberOfKeyGroups, keyGroupRange, getEnvironment().getTaskKvStateRegistry()); // let keyed state backend participate in the operator lifecycle, i.e. make it responsive to cancelation cancelables.registerClosable(keyedStateBackend); // restore if we have some old state Collection<KeyedStateHandle> restoreKeyedStateHandles = restoreStateHandles == null ? null : restoreStateHandles.getManagedKeyedState(); keyedStateBackend.restore(restoreKeyedStateHandles); @SuppressWarnings("unchecked") AbstractKeyedStateBackend<K> typedBackend = (AbstractKeyedStateBackend<K>) keyedStateBackend; return typedBackend; }
private StreamStateHandle materializeMetaData() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; try { outputStream = checkpointStreamFactory .createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closeableRegistry.registerClosable(outputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.keySerializer, stateMetaInfoSnapshots); DataOutputView out = new DataOutputViewStreamWrapper(outputStream); serializationProxy.write(out); closeableRegistry.unregisterClosable(outputStream); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; return result; } finally { if (outputStream != null) { closeableRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
cancelStreamRegistry.registerClosable(fsDataInputStream);
private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( serializationProxy.getKeySerializer(), UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), stateBackend.keySerializer) .isRequiresMigration()) { // TODO replace with state migration; note that key hash codes need to remain the same after migration throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + "Aborting now since state migration is currently not available"); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } } }