@Override public List<SchemaVersion> load(Topic topic) throws Exception { logger.debug("Loading schema versions for topic {}", topic.getQualifiedName()); return rawSchemaClient.getVersions(topic.getName()); }
@JsonCreator public TopicWithSchema(@JsonProperty("schema") String schema, @JsonProperty("name") String qualifiedName, @JsonProperty("description") String description, @JsonProperty("owner") OwnerId owner, @JsonProperty("retentionTime") RetentionTime retentionTime, @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, @JsonProperty("ack") Ack ack, @JsonProperty("trackingEnabled") boolean trackingEnabled, @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, @JsonProperty("schemaVersionAwareSerializationEnabled") boolean schemaVersionAwareSerializationEnabled, @JsonProperty("contentType") ContentType contentType, @JsonProperty("maxMessageSize") Integer maxMessageSize, @JsonProperty("auth") PublishingAuth publishingAuth, @JsonProperty("subscribingRestricted") boolean subscribingRestricted, @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage) { super(qualifiedName, description, owner, retentionTime, jsonToAvroDryRunEnabled, ack, trackingEnabled, migratedFromJsonType, schemaVersionAwareSerializationEnabled, contentType, maxMessageSize, publishingAuth, subscribingRestricted, offlineStorage); this.topic = convertToTopic(); this.schema = schema; }
@Override public void onTopicRemoved(Topic topic) { if (topicCache.containsKey(topic.getName().qualifiedName())) { Topic cachedTopic = topicCache.get(topic.getName().qualifiedName()).getTopic(); if (cachedTopic.equals(topic)) { topicCache.remove(topic.getName().qualifiedName()); } else { logger.warn("Received event about removed topic but cache contains different topic under the same name." + "Cached topic {}, removed topic {}", cachedTopic, topic); } } }
public UnsupportedContentTypeException(Topic topic) { super(String.format( "Unsupported content type %s for topic %s", topic.getContentType(), topic.getQualifiedName() )); }
@Override public String toString() { return "Topic(" + getQualifiedName() + ")"; } }
private void migrateTopic(Topic topic, Group group, String sourceName, OwnerExistsStrategy strategy, EntityMigrationCounters topicCounters) { if (topic.getOwner() == null || strategy == OwnerExistsStrategy.OVERRIDE) { migrateEntity(topicCounters, "topic " + topic.getQualifiedName(), () -> topicService.updateTopic(topic.getName(), patchWithOwner(sourceName, group.getSupportTeam()), MIGRATION_USER) ); } else { topicCounters.markSkipped(OWNER_ALREADY_EXISTED_REASON); } }
private Message create(HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent) { long timestamp = clock.millis(); switch (topic.getContentType()) { case JSON: { if (topic.isJsonToAvroDryRunEnabled()) { try { createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); } catch (AvroConversionException exception) { logger.warn("Unsuccessful message conversion from JSON to AVRO on topic {} in dry run mode", topic.getQualifiedName(), exception); } catch (WrappingException exception) { logger.warn("Unsuccessful wrapping of AVRO message on topic {} in dry run mode", topic.getQualifiedName(), exception); } } return createJsonMessage(headerMap, messageId, messageContent, timestamp); } case AVRO: return createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); default: throw new UnsupportedContentTypeException(topic); } }
public void untilTopicCreated(Topic topic) { waitAtMost(adjust(Duration.ONE_MINUTE)).until(() -> endpoints.findTopics(topic, topic.isTrackingEnabled()).contains(topic.getQualifiedName()) ); }
public void registerSchema(Topic topic, String schema, boolean validate) { if (validate) { SchemaValidator validator = validatorProvider.provide(topic.getContentType()); validator.check(schema); } rawSchemaClient.registerSchema(topic.getName(), RawSchema.valueOf(schema)); }
private boolean contentTypeChanged(Topic updated, Topic previous) { return previous.getContentType() != updated.getContentType(); }
@Override public void updateTopic(Topic newTopic) { if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic) || this.topic.isSchemaVersionAwareSerializationEnabled() != newTopic.isSchemaVersionAwareSerializationEnabled()) { logger.info("Reinitializing message receiver, contentType, messageSize or schemaVersionAwareSerialization changed."); this.topic = newTopic; messageReceiver.stop(); initializeMessageReceiver(); } }
private boolean messageSizeChanged(Topic newTopic) { return this.topic.getMaxMessageSize() != newTopic.getMaxMessageSize() && configFactory.getBooleanProperty(Configs.CONSUMER_USE_TOPIC_MESSAGE_SIZE); }
@Test public void shouldDeserializeTopic() throws Exception { // given String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; // when Topic topic = objectMapper.readValue(json, Topic.class); // then assertThat(topic.getName().getName()).isEqualTo("bar"); assertThat(topic.getName().getGroupName()).isEqualTo("foo"); assertThat(topic.getDescription()).isEqualTo("description"); }
@Override public void check(Message message, Topic topic) { if (ContentType.AVRO != topic.getContentType() || (ContentType.JSON == topic.getContentType() && !topic.isJsonToAvroDryRunEnabled())) { return; } BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(message.getData(), null); try { new GenericDatumReader<>(message.getSchema()).read(null, binaryDecoder); } catch (Exception e) { String reason = e.getMessage() == null ? ExceptionUtils.getRootCauseMessage(e) : e.getMessage(); throw new InvalidMessageException("Could not deserialize avro message with provided schema", ImmutableList.of(reason)); } } }
@Test public void shouldNotResetPrimitiveFields() { // given Topic topic = topic("group.topic").withTrackingEnabled(true).build(); PatchData patch = patchData().set("schemaVersionAwareSerializationEnabled", true).build(); // when Topic patched = Patch.apply(topic, patch); // then assertThat(patched.isTrackingEnabled()).isTrue(); assertThat(patched.isSchemaVersionAwareSerializationEnabled()).isTrue(); } }
public void updateTopic(TopicName topicName, PatchData patch, String modifiedBy) { groupService.checkGroupExists(topicName.getGroupName()); Topic retrieved = getTopicDetails(topicName); Topic modified = Patch.apply(retrieved, patch); topicValidator.ensureUpdatedTopicIsValid(modified, retrieved); if (!retrieved.equals(modified)) { Instant beforeMigrationInstant = clock.instant(); if (retrieved.getRetentionTime() != modified.getRetentionTime()) { multiDCAwareService.manageTopic(brokerTopicManagement -> brokerTopicManagement.updateTopic(modified) ); } topicRepository.updateTopic(modified); if (!retrieved.wasMigratedFromJsonType() && modified.wasMigratedFromJsonType()) { logger.info("Waiting until all subscriptions have consumers assigned during topic {} content type migration...", topicName.qualifiedName()); topicContentTypeMigrationService.waitUntilAllSubscriptionsHasConsumersAssigned(modified, Duration.ofSeconds(topicProperties.getSubscriptionsAssignmentsCompletedTimeoutSeconds())); logger.info("Notifying subscriptions' consumers about changes in topic {} content type...", topicName.qualifiedName()); topicContentTypeMigrationService.notifySubscriptions(modified, beforeMigrationInstant); } auditor.objectUpdated(modifiedBy, retrieved, modified); topicOwnerCache.onUpdatedTopic(retrieved, modified); } }
private boolean isPayloadAwareOfSchemaVersion(byte[] data, Topic topic) { if (topic.isSchemaVersionAwareSerializationEnabled()) { if (SchemaAwareSerDe.startsWithMagicByte(data)) { return true; } deserializationWithMissedSchemaVersionInPayload.inc(); } return false; }
public void removeFromCache(Topic topic) { Set<TopicAndSchemaVersion> topicWithSchemas = cache.asMap().keySet().stream() .filter(topicWithSchema -> topicWithSchema.topic.equals(topic)) .collect(Collectors.toSet()); cache.invalidateAll(topicWithSchemas); }