public GenericWriteAheadSink( CheckpointCommitter committer, TypeSerializer<IN> serializer, String jobID) throws Exception { this.committer = Preconditions.checkNotNull(committer); this.serializer = Preconditions.checkNotNull(serializer); this.id = UUID.randomUUID().toString(); this.committer.setJobId(jobID); this.committer.createResource(); }
public void close() throws Exception { committer.close(); }
/** * Called at {@link #open()} to clean-up the pending handle list. * It iterates over all restored pending handles, checks which ones are already * committed to the outside storage system and removes them from the list. */ private void cleanRestoredHandles() throws Exception { synchronized (pendingCheckpoints) { Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator(); while (pendingCheckpointIt.hasNext()) { PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next(); if (committer.isCheckpointCommitted(pendingCheckpoint.subtaskId, pendingCheckpoint.checkpointId)) { pendingCheckpoint.stateHandle.discardState(); pendingCheckpointIt.remove(); } } } }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( committer.commitCheckpoint(subtaskId, pastCheckpointId); streamHandle.discardState(); pendingCheckpointIt.remove();
/** * Internally used to set the job ID after instantiation. */ public void setJobId(String id) throws Exception { super.setJobId(id); table += id; }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( committer.commitCheckpoint(subtaskId, pastCheckpointId); streamHandle.discardState(); pendingCheckpointIt.remove();
public GenericWriteAheadSink( CheckpointCommitter committer, TypeSerializer<IN> serializer, String jobID) throws Exception { this.committer = Preconditions.checkNotNull(committer); this.serializer = Preconditions.checkNotNull(serializer); this.id = UUID.randomUUID().toString(); this.committer.setJobId(jobID); this.committer.createResource(); }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStreamFactory = getContainingTask() .createCheckpointStreamFactory(this); cleanRestoredHandles(); }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( committer.commitCheckpoint(subtaskId, pastCheckpointId); streamHandle.discardState(); pendingCheckpointIt.remove();
public void close() throws Exception { committer.close(); }
/** * Called at {@link #open()} to clean-up the pending handle list. * It iterates over all restored pending handles, checks which ones are already * committed to the outside storage system and removes them from the list. */ private void cleanRestoredHandles() throws Exception { synchronized (pendingCheckpoints) { Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator(); while (pendingCheckpointIt.hasNext()) { PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next(); if (committer.isCheckpointCommitted(pendingCheckpoint.subtaskId, pendingCheckpoint.checkpointId)) { pendingCheckpoint.stateHandle.discardState(); pendingCheckpointIt.remove(); } } } }
public GenericWriteAheadSink( CheckpointCommitter committer, TypeSerializer<IN> serializer, String jobID) throws Exception { this.committer = Preconditions.checkNotNull(committer); this.serializer = Preconditions.checkNotNull(serializer); this.id = UUID.randomUUID().toString(); this.committer.setJobId(jobID); this.committer.createResource(); }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( committer.commitCheckpoint(subtaskId, pastCheckpointId); streamHandle.discardState(); pendingCheckpointIt.remove();
public void close() throws Exception { committer.close(); }
/** * Called at {@link #open()} to clean-up the pending handle list. * It iterates over all restored pending handles, checks which ones are already * committed to the outside storage system and removes them from the list. */ private void cleanRestoredHandles() throws Exception { synchronized (pendingCheckpoints) { Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator(); while (pendingCheckpointIt.hasNext()) { PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next(); if (committer.isCheckpointCommitted(pendingCheckpoint.subtaskId, pendingCheckpoint.checkpointId)) { pendingCheckpoint.stateHandle.discardState(); pendingCheckpointIt.remove(); } } } }
public GenericWriteAheadSink( CheckpointCommitter committer, TypeSerializer<IN> serializer, String jobID) throws Exception { this.committer = Preconditions.checkNotNull(committer); this.serializer = Preconditions.checkNotNull(serializer); this.id = UUID.randomUUID().toString(); this.committer.setJobId(jobID); this.committer.createResource(); }
public void close() throws Exception { committer.close(); }