/** * Create a stateful Avro fullname adjuster that calls the supplied {@link ReplacementOccurred} function when an invalid * fullname is seen and replaced with a valid fullname. This method replaces all invalid characters with the underscore * character ('_'). * * @param uponReplacement the function called each time the original fullname is replaced; may be null * @return the adjuster; never null */ public static SchemaNameAdjuster create(ReplacementOccurred uponReplacement) { return create("_", uponReplacement); }
/** * Convert the proposed string to a valid Avro fullname, replacing all invalid characters with the underscore ('_') character. * * @param proposedName the proposed fullname; may not be null * @return the valid fullname for Avro; never null */ public static String validFullname(String proposedName) { return validFullname(proposedName, "_"); }
/** * Determine if the supplied string is a valid Avro namespace. * * @param fullname the name to be used as an Avro fullname; may not be null * @return {@code true} if the fullname satisfies Avro rules, or {@code false} otherwise */ public static boolean isValidFullname(String fullname) { if (fullname.length() == 0) return true; char c = fullname.charAt(0); if (!isValidFullnameFirstCharacter(c)) return false; for (int i = 1; i != fullname.length(); ++i) { c = fullname.charAt(i); if (!isValidFullnameNonFirstCharacter(c)) return false; } return true; }
@Test public void shouldReportConflictReplacement() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler.firstTimeOnly()); adjuster.adjust("some-invalid-fullname$"); adjuster.adjust("some-invalid%fullname_"); assertThat(counter.get()).isEqualTo(2); assertThat(conflicts.get()).isEqualTo(1); }
private Schema updateEnvelopeSchema(Schema oldEnvelopeSchema, String newTopicName) { Schema newEnvelopeSchema = envelopeSchemaUpdateCache.get(oldEnvelopeSchema); if (newEnvelopeSchema != null) { return newEnvelopeSchema; } final Schema oldValueSchema = oldEnvelopeSchema.field(Envelope.FieldName.BEFORE).schema(); final SchemaBuilder valueBuilder = copySchemaExcludingName(oldValueSchema, SchemaBuilder.struct()); valueBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Value")); final Schema newValueSchema = valueBuilder.build(); final SchemaBuilder envelopeBuilder = copySchemaExcludingName(oldEnvelopeSchema, SchemaBuilder.struct(), false); for (org.apache.kafka.connect.data.Field field : oldEnvelopeSchema.fields()) { final String fieldName = field.name(); Schema fieldSchema = field.schema(); if (Objects.equals(fieldName, Envelope.FieldName.BEFORE) || Objects.equals(fieldName, Envelope.FieldName.AFTER)) { fieldSchema = newValueSchema; } envelopeBuilder.field(fieldName, fieldSchema); } envelopeBuilder.name(schemaNameAdjuster.adjust(newTopicName + ".Envelope")); newEnvelopeSchema = envelopeBuilder.build(); envelopeSchemaUpdateCache.put(oldEnvelopeSchema, newEnvelopeSchema); return newEnvelopeSchema; }
/** * Determine if the supplied character is a valid non-first character for Avro fullnames. * * @param c the character * @return {@code true} if the character is a valid non-first character of an Avro fullname, or {@code false} otherwise * @see #isValidFullname(String) */ public static boolean isValidFullnameNonFirstCharacter(char c) { return c == '.' || isValidFullnameFirstCharacter(c) || (c >= '0' && c <= '9'); }
@Test public void shouldDetermineValidNonFirstCharacters() { String validChars = ".abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; for (int i = 0; i != validChars.length(); ++i) { assertThat(SchemaNameAdjuster.isValidFullnameNonFirstCharacter(validChars.charAt(i))).isTrue(); } }
protected void assertNotValidFullname(String fullname) { assertThat(SchemaNameAdjuster.isValidFullname(fullname)).isFalse(); } }
@Test public void shouldReportReplacementEveryTime() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler); for (int i = 0; i != 20; ++i) { adjuster.adjust("some-invalid-fullname$"); } assertThat(counter.get()).isEqualTo(20); assertThat(conflicts.get()).isEqualTo(0); }
this.restartOffset = restartOffset; this.schemaChangeKeySchema = SchemaBuilder.struct() .name(schemaNameAdjuster.adjust("io.debezium.connector.mysql.SchemaChangeKey")) .field(Fields.DATABASE_NAME, Schema.STRING_SCHEMA) .build(); this.schemaChangeValueSchema = SchemaBuilder.struct() .name(schemaNameAdjuster.adjust("io.debezium.connector.mysql.SchemaChangeValue")) .field(Fields.SOURCE, SourceInfo.SCHEMA) .field(Fields.DATABASE_NAME, Schema.STRING_SCHEMA)
@Test public void shouldDetermineValidFirstCharacters() { String validChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"; for (int i = 0; i != validChars.length(); ++i) { assertThat(SchemaNameAdjuster.isValidFullnameFirstCharacter(validChars.charAt(i))).isTrue(); } }
protected void assertValidFullname(String fullname) { assertThat(SchemaNameAdjuster.isValidFullname(fullname)).isTrue(); }
/** * Create a stateful Avro fullname adjuster that logs a warning the first time an invalid fullname is seen and replaced * with a valid fullname, and throws an error if the replacement conflicts with that of a different original. This method * replaces all invalid characters with the underscore character ('_'). * * @param logger the logger to use; may not be null * @return the validator; never null */ public static SchemaNameAdjuster create(Logger logger) { return create(logger, (original, replacement, conflict) -> { String msg = "The Kafka Connect schema name '" + original + "' is not a valid Avro schema name and its replacement '" + replacement + "' conflicts with another different schema '" + conflict + "'"; throw new ConnectException(msg); }); }
@Test public void shouldReportReplacementOnlyOnce() { AtomicInteger counter = new AtomicInteger(); AtomicInteger conflicts = new AtomicInteger(); ReplacementOccurred handler = (original, replacement, conflict) -> { if (conflict != null) conflicts.incrementAndGet(); counter.incrementAndGet(); }; SchemaNameAdjuster adjuster = SchemaNameAdjuster.create(handler.firstTimeOnly()); for (int i = 0; i != 20; ++i) { adjuster.adjust("some-invalid-fullname$"); } assertThat(counter.get()).isEqualTo(1); assertThat(conflicts.get()).isEqualTo(0); }
/** * Convert the proposed string to a valid Avro fullname, replacing all invalid characters with the supplied string. * * @param proposedName the proposed fullname; may not be null * @param replacement the character sequence that should be used to replace all invalid characters * @return the valid fullname for Avro; never null */ public static String validFullname(String proposedName, String replacement) { return validFullname(proposedName, c -> replacement); }
private Schema updateKeySchema(Schema oldKeySchema, String newTopicName) { Schema newKeySchema = keySchemaUpdateCache.get(oldKeySchema); if (newKeySchema != null) { return newKeySchema; } final SchemaBuilder builder = copySchemaExcludingName(oldKeySchema, SchemaBuilder.struct()); builder.name(schemaNameAdjuster.adjust(newTopicName + ".Key")); // Now that multiple physical tables can share a topic, the event's key may need to be augmented to include // fields other than just those for the record's primary/unique key, since these are not guaranteed to be unique // across tables. We need some identifier added to the key that distinguishes the different physical tables. builder.field(keyFieldName, Schema.STRING_SCHEMA); newKeySchema = builder.build(); keySchemaUpdateCache.put(oldKeySchema, newKeySchema); return newKeySchema; }
char c = proposedName.charAt(0); boolean changed = false; if (isValidFullnameFirstCharacter(c)) { sb.append(c); } else { if (isValidFullnameNonFirstCharacter(c)) { sb.append(c); } else {
/** * Determine if the supplied character is a valid non-first character for Avro fullnames. * * @param c the character * @return {@code true} if the character is a valid non-first character of an Avro fullname, or {@code false} otherwise * @see #isValidFullname(String) */ public static boolean isValidFullnameNonFirstCharacter(char c) { return c == '.' || isValidFullnameFirstCharacter(c) || (c >= '0' && c <= '9'); }
protected static void validateSchemaNames(Schema schema) { if (schema == null) return; String schemaName = schema.name(); if (schemaName != null && !SchemaNameAdjuster.isValidFullname(schemaName)) { fail("Kafka schema '" + schemaName + "' is not a valid Avro schema name"); } if (schema.type() == Type.STRUCT) { schema.fields().forEach(field -> { validateSubSchemaNames(schema, field); }); } }
/** * Create a stateful Avro fullname adjuster that logs a warning the first time an invalid fullname is seen and replaced * with a valid fullname. This method replaces all invalid characters with the underscore character ('_'). * * @param logger the logger to use; may not be null * @param uponConflict the function to be called when there is a conflict and after that conflict is logged; may be null * @return the validator; never null */ public static SchemaNameAdjuster create(Logger logger, ReplacementOccurred uponConflict) { ReplacementOccurred handler = (original, replacement, conflictsWith) -> { if (conflictsWith != null) { logger.error("The Kafka Connect schema name '{}' is not a valid Avro schema name and its replacement '{}' conflicts with another different schema '{}'", original, replacement, conflictsWith); if (uponConflict != null) { uponConflict.accept(original, replacement, conflictsWith); } } else { logger.warn("The Kafka Connect schema name '{}' is not a valid Avro schema name, so replacing with '{}'", original, replacement); } }; return create(handler.firstTimeOnly()); }