/** * Get the original watermark that was attached to this record, * typically by a {@link org.apache.gobblin.source.extractor.StreamingExtractor} */ public CheckpointableWatermark getWatermark() { return _watermark.getCheckpointableWatermark(); }
public Map<String, CheckpointableWatermark> getUnacknowledgedWatermarks() { Map<String, CheckpointableWatermark> unackedWatermarks = new HashMap<>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark lowestUnacked = null; for (AcknowledgableWatermark watermark: watermarks) { if (!watermark.isAcked()) { lowestUnacked = watermark; break; } } if (lowestUnacked != null) { unackedWatermarks.put(source, lowestUnacked.getCheckpointableWatermark()); } } return unackedWatermarks; }
public Map<String, CheckpointableWatermark> getCommittableWatermarks() { Map<String, CheckpointableWatermark> commitableWatermarks = new HashMap<String, CheckpointableWatermark>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark highestWatermark = null; for (AcknowledgableWatermark watermark: watermarks) { if (watermark.isAcked()) { highestWatermark = watermark; } else { // hopefully we've already found the highest contiguous acked watermark break; } } if (highestWatermark != null) { commitableWatermarks.put(source, highestWatermark.getCheckpointableWatermark()); } } return commitableWatermarks; }
/** * Track a watermark. * Assumptions: Track is called sequentially from the same thread for watermarks that are * progressively increasing. */ public void track(AcknowledgableWatermark acknowledgableWatermark) { if (!_started.get() && _autoStart) { start(); } maybeAbort(); String source = acknowledgableWatermark.getCheckpointableWatermark().getSource(); Deque<AcknowledgableWatermark> sourceWatermarks = _watermarksMap.get(source); if (sourceWatermarks == null) { sourceWatermarks = new ConcurrentLinkedDeque<>(); _watermarksMap.put(source, sourceWatermarks); } sourceWatermarks.add(acknowledgableWatermark); _watermarksInserted.mark(); }
/** * Get the original watermark that was attached to this record, * typically by a {@link org.apache.gobblin.source.extractor.StreamingExtractor} */ public CheckpointableWatermark getWatermark() { return _watermark.getCheckpointableWatermark(); }
public Map<String, CheckpointableWatermark> getUnacknowledgedWatermarks() { Map<String, CheckpointableWatermark> unackedWatermarks = new HashMap<>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark lowestUnacked = null; for (AcknowledgableWatermark watermark: watermarks) { if (!watermark.isAcked()) { lowestUnacked = watermark; break; } } if (lowestUnacked != null) { unackedWatermarks.put(source, lowestUnacked.getCheckpointableWatermark()); } } return unackedWatermarks; }
public Map<String, CheckpointableWatermark> getCommittableWatermarks() { Map<String, CheckpointableWatermark> commitableWatermarks = new HashMap<String, CheckpointableWatermark>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark highestWatermark = null; for (AcknowledgableWatermark watermark: watermarks) { if (watermark.isAcked()) { highestWatermark = watermark; } else { // hopefully we've already found the highest contiguous acked watermark break; } } if (highestWatermark != null) { commitableWatermarks.put(source, highestWatermark.getCheckpointableWatermark()); } } return commitableWatermarks; }
/** * Track a watermark. * Assumptions: Track is called sequentially from the same thread for watermarks that are * progressively increasing. */ public void track(AcknowledgableWatermark acknowledgableWatermark) { if (!_started.get() && _autoStart) { start(); } maybeAbort(); String source = acknowledgableWatermark.getCheckpointableWatermark().getSource(); Deque<AcknowledgableWatermark> sourceWatermarks = _watermarksMap.get(source); if (sourceWatermarks == null) { sourceWatermarks = new ConcurrentLinkedDeque<>(); _watermarksMap.put(source, sourceWatermarks); } sourceWatermarks.add(acknowledgableWatermark); _watermarksInserted.mark(); }