@Override public void initPreviousRunState(@NonNull final IMetadataManager<StringValue> metadataManager) { final String topicName = this.conf.getTopicName(); final Map<Integer, Long> metadata = new HashMap<>(); final String topicSpecificName = getTopicSpecificMetadataKey(topicName); final List<String> toDelete = new LinkedList<>(); metadataManager.getAllKeys().forEach(key -> { if (key.startsWith(topicSpecificName)) { // this is my specific topic metadata.put(Integer.parseInt(key.substring(topicSpecificName.length())), Long.parseLong(metadataManager.get(key).get().getValue())); } else if (key.startsWith(KAFKA_METADATA_WITH_SEPARATOR)) { // this is a specific topic, but not mine. ignore. assert true; } else if (key.startsWith(KAFKA_METADATA_PREFIX)) { // this is unspecified topic metadata.put(Integer.parseInt(key.substring(KAFKA_METADATA_PREFIX.length())), Long.parseLong(metadataManager.get(key).get().getValue())); // delete the old, unspecified metadata toDelete.add(key); } }); toDelete.forEach(metadataManager::remove); this.previousRunState = Optional.of(new KafkaRunState(metadata)); }
/** * Set the metadata for this DAG, if not empty * @param key * @param value */ public void set(@NotEmpty final String key, @NonNull final Map<String, String> value) { try { if (!value.isEmpty()) { this.metadataManager.set(key, new StringValue(mapper.writeValueAsString(value))); } } catch (JsonProcessingException e) { throw new MetadataException("Unable to set the JobManager metadata for key :" + key); } }
metadataManager.set(MetadataConstants.CHECKPOINT_KEY, new StringValue(this.nextPartition.get())); } else { final Optional<StringValue> latestCheckpoint = metadataManager.get(MetadataConstants.CHECKPOINT_KEY); if (!checkpointGreaterThanNextPartition(latestCheckpoint)) { log.info("Save next partition {} in metadata manager", this.nextPartition); metadataManager.set(MetadataConstants.CHECKPOINT_KEY, new StringValue(this.nextPartition.get()));
@Test public void testSerDser() { final String tableName = "testTable"; final IMetadataManager metadataManager = new MemoryMetadataManager(); final SinkStatManager sinkStatManager1 = new SinkStatManager(tableName, metadataManager); // Initially nothing will be found; it should not crash. sinkStatManager1.init(); Assert.assertEquals(0, sinkStatManager1.getAvgRecordSize()); Assert.assertFalse(sinkStatManager1.isStatHistoryAvailable()); final int avgRecordSize1 = 30; sinkStatManager1.getCurrentStat().put(SinkStat.AVG_RECORD_SIZE, Integer.toString(avgRecordSize1)); // nothing is saved to metadata manager before persist. Assert.assertEquals(0, metadataManager.getAllKeys().size()); sinkStatManager1.persist(); Assert.assertEquals(1, metadataManager.getAllKeys().size()); final SinkStatManager sinkStatManager2 = new SinkStatManager(tableName, metadataManager); sinkStatManager2.init(); Assert.assertEquals(avgRecordSize1, sinkStatManager2.getAvgRecordSize()); final int avgRecordSize2 = 20; sinkStatManager2.getCurrentStat().put(SinkStat.AVG_RECORD_SIZE, Integer.toString(avgRecordSize2)); sinkStatManager2.persist(); final SinkStatManager sinkStatManager3 = new SinkStatManager(tableName, metadataManager); sinkStatManager3.init(); Assert.assertEquals((avgRecordSize1 + avgRecordSize2) / 2, sinkStatManager3.getAvgRecordSize()); }
/*** * Checks if metadata for a given DAG already exists * @param key * @return */ public boolean contains(@NotEmpty final String key) { return this.metadataManager.get(key).isPresent() ? true : false; }
/*** * Invokes the {@link IMetadataManager} to store the metadata information * */ public void writeJobManagerMetadata() { if (!this.shouldSaveChanges.get()) { throw new MetadataException("JobManager metadata can only be saved once."); } else { try { this.metadataManager.saveChanges(); this.shouldSaveChanges.compareAndSet(true, false); } catch (IOException e) { throw new MetadataException("Unable to save JobManager Metadata", e); } } } }
/*** * Returns the metadata for the given DAG * @param key * @return * @throws IOException */ public Optional<Map<String, String>> get(@NotEmpty final String key) throws IOException { final Optional<StringValue> metadataValues = this.metadataManager.get(key); if (metadataValues.isPresent()) { return Optional.of(mapper.readValue(metadataValues.get().getValue(), typeRef)); } return Optional.absent(); }
this.metadataManager.saveChanges(); } catch (IOException e) { final String msg = "Failed to save metadata changes" + e.getMessage();
@Override public void initPreviousRunState(@NonNull final IMetadataManager<StringValue> metadataManager) { try { final Optional<StringValue> latestCheckpoint = metadataManager.get(MetadataConstants.CHECKPOINT_KEY); log.info("Get latest change point: {}", latestCheckpoint); this.nextPartition = this.partitionManager.getNextPartition(latestCheckpoint); } catch (final IOException e) { throw new JobRuntimeException("Unable to get the next partition. Error message: " + this.nextPartition, e); } }
@Override public void saveNextRunState(@NonNull final IMetadataManager<StringValue> metadataManager, final KafkaRunState nextRunState) { final String topicName = this.conf.getTopicName(); final String topicSpecificName = getTopicSpecificMetadataKey(topicName); nextRunState.getPartitionOffsets().entrySet().forEach( entry -> { metadataManager.set(topicSpecificName + entry.getKey(), new StringValue(entry.getValue().toString())); }); }
public void init() { final Optional<StringValue> serialisedStats = this.metadataManager.get(getMetakey()); if (serialisedStats.isPresent()) { final Map<String, String> statHistory = MapUtil.deserializeMap(serialisedStats.get().getValue()); for (int i = 0; i < statHistory.size(); i++) { this.sinkStatQ.add(SinkStat.deserialize(statHistory.get(Integer.toString(i)))); } } }
public void persist() { final Map<String, String> stats = new HashMap<>(); if (!this.currentStat.isEmpty()) { this.sinkStatQ.add(this.currentStat); } while (this.sinkStatQ.size() > MAX_HISTORY_SIZE) { this.sinkStatQ.poll(); } for (int i = 0; !this.sinkStatQ.isEmpty(); i++) { stats.put(Integer.toString(i), SinkStat.serialize(this.sinkStatQ.poll())); } this.metadataManager.set(getMetakey(), new StringValue(MapUtil.serializeMap(stats))); }