@Override public void unacknowledgedWatermark(CheckpointableWatermark unacked) { if (_unackedWatermarkMap != null) { _unackedWatermarkMap.put(unacked.getSource(), unacked); } }
@Override public void committedWatermark(CheckpointableWatermark committed) { _committedWatermarkMap.put(committed.getSource(), committed); }
@Override public Map<String, CheckpointableWatermark> getAllCommitableWatermarks() { Map<String, CheckpointableWatermark> commitables = new HashMap<>(candidateCommittables.size()); for (String source: candidateCommittables.keySet()) { Optional<CheckpointableWatermark> commitable = getCommittableWatermark(source); if (commitable.isPresent()) { commitables.put(commitable.get().getSource(), commitable.get()); } } return commitables; }
/** * Create a CheckpointableWatermarkState object from a CheckpointableWatermark * @param watermark: the checkpointable watermark * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}. */ public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) { super.setProp(watermark.getSource(), gson.toJsonTree(watermark)); super.setId(watermark.getSource()); }
@Override public int compareTo(CheckpointableWatermark o) { if (!(this.source.equals(o.getSource()))) { throw new RuntimeException("Could not compare two checkpointable watermarks because they have different sources " + this.source + ":" + o.getSource()); } return this.comparable.compareTo(o.getWatermark()); }
@Override public Map<String, CheckpointableWatermark> getAllUnacknowledgedWatermarks() { Map<String, CheckpointableWatermark> unackedMap = new HashMap<>(unacknowledgedWatermarks.size()); for (String source: unacknowledgedWatermarks.keySet()) { Optional<CheckpointableWatermark> unacked = getUnacknowledgedWatermark(source); if (unacked.isPresent()) { unackedMap.put(unacked.get().getSource(), unacked.get()); } } return unackedMap; }
@Override public void committedWatermark(CheckpointableWatermark committed) { getOrCreate(candidateCommittables, committed.getSource()).add(committed); }
@Override public void unacknowledgedWatermark(CheckpointableWatermark unacked) { getOrCreate(unacknowledgedWatermarks, unacked.getSource()).add(unacked); }
@Override public void commitWatermarks(Iterable<CheckpointableWatermark> watermarks) throws IOException { for (CheckpointableWatermark watermark: watermarks) { this.watermarks.put(watermark.getSource(), watermark); } }
@Override public void commitWatermarks(Iterable<CheckpointableWatermark> watermarks) throws IOException { for (CheckpointableWatermark watermark: watermarks) { String tableName = watermark.getSource(); _stateStore.put(_storeName, tableName, new CheckpointableWatermarkState(watermark, GSON)); } }
/** * Track a watermark. * Assumptions: Track is called sequentially from the same thread for watermarks that are * progressively increasing. */ public void track(AcknowledgableWatermark acknowledgableWatermark) { if (!_started.get() && _autoStart) { start(); } maybeAbort(); String source = acknowledgableWatermark.getCheckpointableWatermark().getSource(); Deque<AcknowledgableWatermark> sourceWatermarks = _watermarksMap.get(source); if (sourceWatermarks == null) { sourceWatermarks = new ConcurrentLinkedDeque<>(); _watermarksMap.put(source, sourceWatermarks); } sourceWatermarks.add(acknowledgableWatermark); _watermarksInserted.mark(); }
@Override public void writeEnvelope(RecordEnvelope<Object> recordEnvelope) throws IOException { _recordCollector.add(recordEnvelope.getRecord()); String source = recordEnvelope.getWatermark().getSource(); if (this.source.get() != null) { if (!source.equals(this.source.get())) { throw new RuntimeException("This writer only supports a single source"); } } this.lastWatermark.set(recordEnvelope.getWatermark()); recordEnvelope.ack(); this.source.set(source); }
@Test public void testMultiSource() { MultiWriterWatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); commits(watermarkTracker, "default", 0, 4, 5, 6); commits(watermarkTracker, "other", 1, 3, 5, 7); Assert.assertEquals(watermarkTracker.getCommittableWatermark("default").get().getSource(), "default"); Assert.assertEquals(((LongWatermark) watermarkTracker.getCommittableWatermark("default") .get().getWatermark()).getValue(), 6L); Assert.assertEquals(watermarkTracker.getCommittableWatermark("other").get().getSource(), "other"); Assert.assertEquals(((LongWatermark) watermarkTracker.getCommittableWatermark("other") .get().getWatermark()).getValue(), 7L); }
@Test public void testSingleSource() { MultiWriterWatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); commits(watermarkTracker, "default", 0, 4, 5, 6); Assert.assertEquals(watermarkTracker.getCommittableWatermark("default").get().getSource(), "default"); Assert.assertEquals(((LongWatermark) watermarkTracker.getCommittableWatermark("default") .get().getWatermark()).getValue(), 6L); }
committedWatermarks.put(record.getWatermark().getSource(), record.getWatermark());
@Override public void unacknowledgedWatermark(CheckpointableWatermark unacked) { if (_unackedWatermarkMap != null) { _unackedWatermarkMap.put(unacked.getSource(), unacked); } }
@Override public int compareTo(CheckpointableWatermark o) { if (!(this.source.equals(o.getSource()))) { throw new RuntimeException("Could not compare two checkpointable watermarks because they have different sources " + this.source + ":" + o.getSource()); } return this.comparable.compareTo(o.getWatermark()); }
@Override public Map<String, CheckpointableWatermark> getAllUnacknowledgedWatermarks() { Map<String, CheckpointableWatermark> unackedMap = new HashMap<>(unacknowledgedWatermarks.size()); for (String source: unacknowledgedWatermarks.keySet()) { Optional<CheckpointableWatermark> unacked = getUnacknowledgedWatermark(source); if (unacked.isPresent()) { unackedMap.put(unacked.get().getSource(), unacked.get()); } } return unackedMap; }
/** * Create a CheckpointableWatermarkState object from a CheckpointableWatermark * @param watermark: the checkpointable watermark * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}. */ public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) { super.setProp(watermark.getSource(), gson.toJsonTree(watermark)); super.setId(watermark.getSource()); }
@Override public Map<String, CheckpointableWatermark> getAllCommitableWatermarks() { Map<String, CheckpointableWatermark> commitables = new HashMap<>(candidateCommittables.size()); for (String source: candidateCommittables.keySet()) { Optional<CheckpointableWatermark> commitable = getCommittableWatermark(source); if (commitable.isPresent()) { commitables.put(commitable.get().getSource(), commitable.get()); } } return commitables; }