@Test public void shouldReportConflictReplacement() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler.firstTimeOnly()); adjuster.adjust("some-invalid-fullname$"); adjuster.adjust("some-invalid%fullname_"); assertThat(counter.get()).isEqualTo(2); assertThat(conflicts.get()).isEqualTo(1); }
@Test public void shouldReportReplacementEveryTime() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler); for (int i = 0; i != 20; ++i) { adjuster.adjust("some-invalid-fullname$"); } assertThat(counter.get()).isEqualTo(20); assertThat(conflicts.get()).isEqualTo(0); }
@Test public void shouldReportReplacementOnlyOnce() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler.firstTimeOnly()); for (int i = 0; i != 20; ++i) { adjuster.adjust("some-invalid-fullname$"); } assertThat(counter.get()).isEqualTo(1); assertThat(conflicts.get()).isEqualTo(0); }
private Schema updateEnvelopeSchema(Schema oldEnvelopeSchema, String newTopicName) { Schema newEnvelopeSchema = envelopeSchemaUpdateCache.get(oldEnvelopeSchema); if (newEnvelopeSchema != null) { return newEnvelopeSchema; } final Schema oldValueSchema = oldEnvelopeSchema.field(Envelope.FieldName.BEFORE).schema(); final SchemaBuilder valueBuilder = copySchemaExcludingName(oldValueSchema, SchemaBuilder.struct()); valueBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Value")); final Schema newValueSchema = valueBuilder.build(); final SchemaBuilder envelopeBuilder = copySchemaExcludingName(oldEnvelopeSchema, SchemaBuilder.struct(), false); for (org.apache.kafka.connect.data.Field field : oldEnvelopeSchema.fields()) { final String fieldName = field.name(); Schema fieldSchema = field.schema(); if (Objects.equals(fieldName, Envelope.FieldName.BEFORE) || Objects.equals(fieldName, Envelope.FieldName.AFTER)) { fieldSchema = newValueSchema; } envelopeBuilder.field(fieldName, fieldSchema); } envelopeBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Envelope")); newEnvelopeSchema = envelopeBuilder.build(); envelopeSchemaUpdateCache.put(oldEnvelopeSchema, newEnvelopeSchema); return newEnvelopeSchema; }
this.restartOffset = restartOffset; this.schemaChangeKeySchema = SchemaBuilder.struct() .name(schemaNameAdjuster.adjust("io.debezium.connector.mysql.SchemaChangeKey")) .field(Fields.DATABASE_NAME, Schema.STRING_SCHEMA) .build(); this.schemaChangeValueSchema = SchemaBuilder.struct() .name(schemaNameAdjuster.adjust("io.debezium.connector.mysql.SchemaChangeValue")) .field(Fields.SOURCE, SourceInfo.SCHEMA) .field(Fields.DATABASE_NAME, Schema.STRING_SCHEMA)
private Schema updateKeySchema(Schema oldKeySchema, String newTopicName) { Schema newKeySchema = keySchemaUpdateCache.get(oldKeySchema); if (newKeySchema != null) { return newKeySchema; } final SchemaBuilder builder = copySchemaExcludingName(oldKeySchema, SchemaBuilder.struct()); builder.name(schemaNameAdjuster.adjust(newTopicName + ".Key")); // Now that multiple physical tables can share a topic, the event's key may need to be augmented to include // fields other than just those for the record's primary/unique key, since these are not guaranteed to be unique // across tables. We need some identifier added to the key that distinguishes the different physical tables. builder.field(keyFieldName, Schema.STRING_SCHEMA); newKeySchema = builder.build(); keySchemaUpdateCache.put(oldKeySchema, newKeySchema); return newKeySchema; }
final String schemaNamePrefix = schemaPrefix + tableIdStr; LOGGER.debug("Mapping table '{}' to schemas under '{}'", tableId, schemaNamePrefix); SchemaBuilder valSchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Value")); SchemaBuilder keySchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Key")); AtomicBoolean hasPrimaryKey = new AtomicBoolean(false); table.columns().forEach(column -> { .withName(schemaNameAdjuster.adjust(envelopSchemaName)) .withRecord(valSchema) .withSource(sourceInfoSchema)
protected RecordsForCollection(CollectionId collectionId, FieldFilter fieldFilter, SourceInfo source, String topicName, SchemaNameAdjuster adjuster, Function<Document, String> valueTransformer, BlockingConsumer<SourceRecord> recorder, boolean emitTombstonesOnDelete) { this.sourcePartition = source.partition(collectionId.replicaSetName()); this.collectionId = collectionId; this.replicaSetName = this.collectionId.replicaSetName(); this.fieldFilter = fieldFilter; this.source = source; this.topicName = topicName; this.keySchema = SchemaBuilder.struct() .name(adjuster.adjust(topicName + ".Key")) .field("id", Schema.STRING_SCHEMA) .build(); this.valueSchema = SchemaBuilder.struct() .name(adjuster.adjust(topicName + ".Envelope")) .field(FieldName.AFTER, Json.builder().optional().build()) .field("patch", Json.builder().optional().build()) .field(FieldName.SOURCE, source.schema()) .field(FieldName.OPERATION, Schema.OPTIONAL_STRING_SCHEMA) .field(FieldName.TIMESTAMP, Schema.OPTIONAL_INT64_SCHEMA) .build(); this.valueTransformer = valueTransformer; this.recorder = recorder; this.emitTombstonesOnDelete = emitTombstonesOnDelete; }
private Schema updateEnvelopeSchema(Schema oldEnvelopeSchema, String newTopicName) { Schema newEnvelopeSchema = envelopeSchemaUpdateCache.get(oldEnvelopeSchema); if (newEnvelopeSchema != null) { return newEnvelopeSchema; } final Schema oldValueSchema = oldEnvelopeSchema.field(Envelope.FieldName.BEFORE).schema(); final SchemaBuilder valueBuilder = copySchemaExcludingName(oldValueSchema, SchemaBuilder.struct()); valueBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Value")); final Schema newValueSchema = valueBuilder.build(); final SchemaBuilder envelopeBuilder = copySchemaExcludingName(oldEnvelopeSchema, SchemaBuilder.struct(), false); for (org.apache.kafka.connect.data.Field field : oldEnvelopeSchema.fields()) { final String fieldName = field.name(); Schema fieldSchema = field.schema(); if (Objects.equals(fieldName, Envelope.FieldName.BEFORE) || Objects.equals(fieldName, Envelope.FieldName.AFTER)) { fieldSchema = newValueSchema; } envelopeBuilder.field(fieldName, fieldSchema); } envelopeBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Envelope")); newEnvelopeSchema = envelopeBuilder.build(); envelopeSchemaUpdateCache.put(oldEnvelopeSchema, newEnvelopeSchema); return newEnvelopeSchema; }
private Schema updateKeySchema(Schema oldKeySchema, String newTopicName) { Schema newKeySchema = keySchemaUpdateCache.get(oldKeySchema); if (newKeySchema != null) { return newKeySchema; } final SchemaBuilder builder = copySchemaExcludingName(oldKeySchema, SchemaBuilder.struct()); builder.name(schemaNameAdjuster.adjust(newTopicName + ".Key")); // Now that multiple physical tables can share a topic, the event's key may need to be augmented to include // fields other than just those for the record's primary/unique key, since these are not guaranteed to be unique // across tables. We need some identifier added to the key that distinguishes the different physical tables. builder.field(keyFieldName, Schema.STRING_SCHEMA); newKeySchema = builder.build(); keySchemaUpdateCache.put(oldKeySchema, newKeySchema); return newKeySchema; }
final String schemaNamePrefix = schemaPrefix + tableIdStr; LOGGER.debug("Mapping table '{}' to schemas under '{}'", tableId, schemaNamePrefix); SchemaBuilder valSchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Value")); SchemaBuilder keySchemaBuilder = SchemaBuilder.struct().name(schemaNameAdjuster.adjust(schemaNamePrefix + ".Key")); AtomicBoolean hasPrimaryKey = new AtomicBoolean(false); table.columns().forEach(column -> { .withName(schemaNameAdjuster.adjust(envelopSchemaName)) .withRecord(valSchema) .withSource(sourceInfoSchema)