@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }