protected void openCurrentStream() throws IOException { Preconditions.checkState(currentStream == null); FSDataInputStream stream = currentStateHandle.openInputStream(); closableRegistry.registerCloseable(stream); currentStream = stream; }
/** * Called at {@link #open()} to clean-up the pending handle list. * It iterates over all restored pending handles, checks which ones are already * committed to the outside storage system and removes them from the list. */ private void cleanRestoredHandles() throws Exception { synchronized (pendingCheckpoints) { Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator(); while (pendingCheckpointIt.hasNext()) { PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next(); if (committer.isCheckpointCommitted(pendingCheckpoint.subtaskId, pendingCheckpoint.checkpointId)) { pendingCheckpoint.stateHandle.discardState(); pendingCheckpointIt.remove(); } } } }
try { if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( new ReusingMutableToRegularIteratorWrapper<>( streamHandle.discardState(); pendingCheckpointIt.remove(); streamHandle.discardState(); pendingCheckpointIt.remove();
@Override public long getStateSize() { return stateHandle.getStateSize(); }
public MultiFSDataInputStream(List<StreamStateHandle> stateHandles) throws IOException { this.stateHandleMap = new TreeMap<>(); this.totalPos = 0L; long calculateSize = 0L; for (StreamStateHandle stateHandle : stateHandles) { stateHandleMap.put(calculateSize, stateHandle); calculateSize += stateHandle.getStateSize(); } this.totalAvailable = calculateSize; if (totalAvailable > 0L) { StreamStateHandle first = stateHandleMap.firstEntry().getValue(); delegate = first.openInputStream(); } }
@Override public long getStateSize() { return wrappedStreamStateHandle.getStateSize(); }
inputStream = remoteFileHandle.openInputStream(); closeableRegistry.registerCloseable(inputStream);
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
@Override public long getStateSize() { return delegate.getStateSize(); }
inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerCloseable(inputStream);
@Override public void discardState() throws Exception { delegateStateHandle.discardState(); }
@Override public long getStateSize() { return snapshotHandle == null ? 0 : snapshotHandle.getStateSize(); }
@Override public FSDataInputStream openInputStream() throws IOException { return wrappedStreamStateHandle.openInputStream(); }
@Override public void discardState() throws Exception { if (snapshotHandle != null) { snapshotHandle.discardState(); } }
@Override public long getStateSize() { return stateHandle.getStateSize(); }
@Override public FSDataInputStream openInputStream() throws IOException { return delegateStateHandle.openInputStream(); }
@Override public void discardState() throws Exception { delegateStateHandle.discardState(); }
@Override public long getStateSize() { return wrappedStreamStateHandle.getStateSize(); }
@Override public FSDataInputStream openInputStream() throws IOException { return wrappedStreamStateHandle.openInputStream(); }
@Override public void discardState() throws Exception { delegateStateHandle.discardState(); }