@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( new ReusingMutableToRegularIteratorWrapper<>( new InputViewIterator<>(
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
public void open() throws Exception { super.open(); if (!getRuntimeContext().isCheckpointingEnabled()) { throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled."); } cluster = builder.getCluster(); session = cluster.connect(); preparedStatement = session.prepare(insertQuery); fields = new Object[((TupleSerializer<IN>) serializer).getArity()]; }
@Override public void close() throws Exception { super.close(); try { if (session != null) { session.close(); } } catch (Exception e) { LOG.error("Error while closing session.", e); } try { if (cluster != null) { cluster.close(); } } catch (Exception e) { LOG.error("Error while closing cluster.", e); } }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
public void open() throws Exception { super.open(); if (!getRuntimeContext().isCheckpointingEnabled()) { throw new IllegalStateException("The write-ahead log requires checkpointing to be enabled."); } cluster = builder.getCluster(); session = cluster.connect(); preparedStatement = session.prepare(insertQuery); arity = ((RowSerializer) serializer).getArity(); fields = new Object[arity]; }
@Override public void close() throws Exception { super.close(); try { if (session != null) { session.close(); } } catch (Exception e) { LOG.error("Error while closing session.", e); } try { if (cluster != null) { cluster.close(); } } catch (Exception e) { LOG.error("Error while closing cluster.", e); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( new ReusingMutableToRegularIteratorWrapper<>( new InputViewIterator<>(
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStorage = getContainingTask().getCheckpointStorage(); cleanRestoredHandles(); }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); Preconditions.checkState(this.checkpointedState == null, "The reader state has already been initialized."); checkpointedState = context.getOperatorStateStore() .getSerializableListState("pending-checkpoints"); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); for (PendingCheckpoint pendingCheckpoint : checkpointedState.get()) { this.pendingCheckpoints.add(pendingCheckpoint); } if (LOG.isDebugEnabled()) { LOG.debug("GenericWriteAheadSink idx {} restored {}.", subtaskIdx, this.pendingCheckpoints); } } else { LOG.info("No state to restore for the GenericWriteAheadSink (taskIdx={}).", subtaskIdx); } }
if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) { try (FSDataInputStream in = streamHandle.openInputStream()) { boolean success = sendValues( new ReusingMutableToRegularIteratorWrapper<>( new InputViewIterator<>(
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); Preconditions.checkState(this.checkpointedState != null, "The operator state has not been properly initialized."); saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp()); this.checkpointedState.clear(); try { for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) { // create a new partition for each entry. this.checkpointedState.add(pendingCheckpoint); } } catch (Exception e) { checkpointedState.clear(); throw new Exception("Could not add panding checkpoints to operator state " + "backend of operator " + getOperatorName() + '.', e); } int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints); } }
@Override public void open() throws Exception { super.open(); committer.setOperatorId(id); committer.open(); checkpointStreamFactory = getContainingTask() .createCheckpointStreamFactory(this); cleanRestoredHandles(); }
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }