/*** * Checks if metadata for a given DAG already exists * @param key * @return */ public boolean contains(@NotEmpty final String key) { return this.metadataManager.get(key).isPresent() ? true : false; }
@Override public void initPreviousRunState(@NonNull final IMetadataManager<StringValue> metadataManager) { final String topicName = this.conf.getTopicName(); final Map<Integer, Long> metadata = new HashMap<>(); final String topicSpecificName = getTopicSpecificMetadataKey(topicName); final List<String> toDelete = new LinkedList<>(); metadataManager.getAllKeys().forEach(key -> { if (key.startsWith(topicSpecificName)) { // this is my specific topic metadata.put(Integer.parseInt(key.substring(topicSpecificName.length())), Long.parseLong(metadataManager.get(key).get().getValue())); } else if (key.startsWith(KAFKA_METADATA_WITH_SEPARATOR)) { // this is a specific topic, but not mine. ignore. assert true; } else if (key.startsWith(KAFKA_METADATA_PREFIX)) { // this is unspecified topic metadata.put(Integer.parseInt(key.substring(KAFKA_METADATA_PREFIX.length())), Long.parseLong(metadataManager.get(key).get().getValue())); // delete the old, unspecified metadata toDelete.add(key); } }); toDelete.forEach(metadataManager::remove); this.previousRunState = Optional.of(new KafkaRunState(metadata)); }
/*** * Returns the metadata for the given DAG * @param key * @return * @throws IOException */ public Optional<Map<String, String>> get(@NotEmpty final String key) throws IOException { final Optional<StringValue> metadataValues = this.metadataManager.get(key); if (metadataValues.isPresent()) { return Optional.of(mapper.readValue(metadataValues.get().getValue(), typeRef)); } return Optional.absent(); }
@Override public void initPreviousRunState(@NonNull final IMetadataManager<StringValue> metadataManager) { try { final Optional<StringValue> latestCheckpoint = metadataManager.get(MetadataConstants.CHECKPOINT_KEY); log.info("Get latest change point: {}", latestCheckpoint); this.nextPartition = this.partitionManager.getNextPartition(latestCheckpoint); } catch (final IOException e) { throw new JobRuntimeException("Unable to get the next partition. Error message: " + this.nextPartition, e); } }
public void init() { final Optional<StringValue> serialisedStats = this.metadataManager.get(getMetakey()); if (serialisedStats.isPresent()) { final Map<String, String> statHistory = MapUtil.deserializeMap(serialisedStats.get().getValue()); for (int i = 0; i < statHistory.size(); i++) { this.sinkStatQ.add(SinkStat.deserialize(statHistory.get(Integer.toString(i)))); } } }
final Optional<StringValue> latestCheckpoint = metadataManager.get(MetadataConstants.CHECKPOINT_KEY); if (!checkpointGreaterThanNextPartition(latestCheckpoint)) { log.info("Save next partition {} in metadata manager", this.nextPartition);