@Override public void notifyCheckpointComplete(long checkpointId) throws Exception { List<CheckpointMarkT> checkpointMarks = pendingCheckpoints.get(checkpointId); if (checkpointMarks != null) { // remove old checkpoints including the current one Iterator<Long> iterator = pendingCheckpoints.keySet().iterator(); long currentId; do { currentId = iterator.next(); iterator.remove(); } while (currentId != checkpointId); // confirm all marks for (CheckpointMarkT mark : checkpointMarks) { mark.finalizeCheckpoint(); } } } }
@Override public void notifyCheckpointComplete(long checkpointId) throws Exception { List<CheckpointMarkT> checkpointMarks = pendingCheckpoints.get(checkpointId); if (checkpointMarks != null) { // remove old checkpoints including the current one Iterator<Long> iterator = pendingCheckpoints.keySet().iterator(); long currentId; do { currentId = iterator.next(); iterator.remove(); } while (currentId != checkpointId); // confirm all marks for (CheckpointMarkT mark : checkpointMarks) { mark.finalizeCheckpoint(); } } } }
@Override public void notifyCheckpointComplete(long checkpointId) throws Exception { List<CheckpointMarkT> checkpointMarks = pendingCheckpoints.get(checkpointId); if (checkpointMarks != null) { // remove old checkpoints including the current one Iterator<Long> iterator = pendingCheckpoints.keySet().iterator(); long currentId; do { currentId = iterator.next(); iterator.remove(); } while (currentId != checkpointId); // confirm all marks for (CheckpointMarkT mark : checkpointMarks) { mark.finalizeCheckpoint(); } } } }
try { if (checkpoint != null) { checkpoint.finalizeCheckpoint();
final CheckpointMarkT mark = (CheckpointMarkT) reader.getCheckpointMark(); if (oldMark != null) { oldMark.finalizeCheckpoint(); () -> { try { mark.finalizeCheckpoint(); } catch (IOException e) { throw new RuntimeException(
@ProcessElement public void process( @Element Shard<T> shard, OutputReceiver<ValueWithRecordId<T>> out, PipelineOptions options) throws Exception { Instant endTime = shard.getMaxReadTime() == null ? null : Instant.now().plus(shard.getMaxReadTime()); if (shard.getMaxNumRecords() <= 0 || (shard.getMaxReadTime() != null && shard.getMaxReadTime().getMillis() == 0)) { return; } try (UnboundedSource.UnboundedReader<T> reader = SerializableUtils.clone(shard.getSource()).createReader(options, null)) { for (long i = 0L; i < shard.getMaxNumRecords(); ++i) { boolean available = (i == 0) ? reader.start() : reader.advance(); if (!available && !advanceWithBackoff(reader, endTime)) { break; } out.outputWithTimestamp( new ValueWithRecordId<T>(reader.getCurrent(), reader.getCurrentRecordId()), reader.getCurrentTimestamp()); } reader.getCheckpointMark().finalizeCheckpoint(); } }