boolean valueSchemasEqual = true; boolean fieldsEqual = true; if (schema1.type() == Type.MAP && schema2.type() == Type.MAP) { keySchemasEqual = Objects.equals(schema1.keySchema(), schema2.keySchema()); valueSchemasEqual = Objects.equals(schema1.valueSchema(), schema2.valueSchema()); else if (schema1.type() == Type.ARRAY && schema2.type() == Type.ARRAY) { valueSchemasEqual = Objects.equals(schema1.valueSchema(), schema2.valueSchema()); else if (schema1.type() == Type.STRUCT && schema2.type() == Type.STRUCT) { fieldsEqual = areFieldListsEqual(schema1.fields(), schema2.fields()); boolean equal = Objects.equals(schema1.isOptional(), schema2.isOptional()) && Objects.equals(schema1.version(), schema2.version()) && Objects.equals(schema1.name(), schema2.name()) && Objects.equals(schema1.doc(), schema2.doc()) && Objects.equals(schema1.type(), schema2.type()) && Objects.deepEquals(schema1.defaultValue(), schema2.defaultValue()) && fieldsEqual && keySchemasEqual && valueSchemasEqual && Objects.equals(schema1.parameters(), schema2.parameters());
protected Field[] fieldsForColumns(Schema schema, List<Column> columns) { Field[] fields = new Field[columns.size()]; AtomicInteger i = new AtomicInteger(0); columns.forEach(column -> { Field field = schema.field(column.name()); // may be null if the field is unused ... fields[i.getAndIncrement()] = field; }); return fields; }
private boolean compareSchemas(final Schema schema1, final Schema schema2) { if (schema1.type() != schema2.type()) { return false; } switch (schema1.type()) { case STRUCT: return compareStructSchema(schema1, schema2); case ARRAY: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()); case MAP: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()) && compareSchemas(schema1.keySchema(), schema2.keySchema()); default: return true; } }
private List<FunctionParameter> mapToFunctionParameter(final List<Schema> params) { return params .stream() .map(schema -> schema == null ? new FunctionParameter(null, false) : new FunctionParameter(schema.type(), schema.isOptional())) .collect(Collectors.toList()); }
private static List<Pair<String, Type>> getStructItems(final Schema struct) { if (struct.type() != Schema.Type.STRUCT) { return null; } final List<Pair<String, Type>> itemList = new ArrayList<>(); for (final Field field: struct.schema().fields()) { itemList.add(new Pair<>(field.name(), getKsqlType(field.schema()))); } return itemList; }
protected void assertField(Field field, String fieldName, Schema expectedSchema, boolean optional) { assertThat(field.name()).isEqualTo(fieldName); Schema schema = field.schema(); assertThat(schema.name()).isEqualTo(expectedSchema.name()); assertThat(schema.doc()).isEqualTo(expectedSchema.doc()); assertThat(schema.parameters()).isEqualTo(expectedSchema.parameters()); assertThat(schema.version()).isEqualTo(expectedSchema.version()); assertThat(schema.isOptional()).isEqualTo(optional); switch (expectedSchema.type()) { case STRUCT: for (Field f : expectedSchema.fields()) { assertField(schema.field(f.name()),f.name(),f.schema(),f.schema().isOptional()); } break; default: } }
return null; switch (schema.type()) { case ARRAY: final List<Object> ksqlArray = new ArrayList<>(((List) object).size()); ((List) object).forEach( e -> ksqlArray.add(replaceSchema(schema.valueSchema(), e))); return ksqlArray; ((Map<Object, Object>) object).forEach( (key, value) -> ksqlMap.put( replaceSchema(schema.keySchema(), key), replaceSchema(schema.valueSchema(), value) )); return ksqlMap; schema.fields().forEach( f -> struct.put( f.name(),
private Schema buildAvroCompatibleSchema(final Schema schema, final TypeNameGenerator typeNameGenerator) { final SchemaBuilder schemaBuilder; switch (schema.type()) { default: return schema; case STRUCT: schemaBuilder = SchemaBuilder.struct(); if (schema.name() == null) { schemaBuilder.name(typeNameGenerator.name()); for (final Field f : schema.fields()) { schemaBuilder.field( avroCompatibleFieldName(f), case ARRAY: schemaBuilder = SchemaBuilder.array( buildAvroCompatibleSchema(schema.valueSchema(), typeNameGenerator)); break; case MAP: schemaBuilder = SchemaBuilder.map( buildAvroCompatibleSchema(schema.keySchema(), typeNameGenerator.with(TypeNameGenerator.MAP_KEY_NAME)), buildAvroCompatibleSchema(schema.valueSchema(), typeNameGenerator.with(TypeNameGenerator.MAP_VALUE_NAME))); break; if (schema.isOptional()) { schemaBuilder.optional();
@Test public void shouldTranslateStructInsideMap() { final Schema connectSchema = SchemaBuilder .struct() .field( "mapField", SchemaBuilder.map( Schema.STRING_SCHEMA, SchemaBuilder.struct() .field("innerIntField", Schema.INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.isOptional(), is(true)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(mapSchema.valueSchema().fields().size(), equalTo(1)); assertThat(mapSchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(mapSchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void unsignedSmallIntTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_SMALLINT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(0); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(10); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(0); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(0); assertEmptyFieldValue(record, "G"); }
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateStructInsideArray() { final Schema connectSchema = SchemaBuilder .struct() .field( "arrayField", SchemaBuilder.array( SchemaBuilder.struct() .field("innerIntField", Schema.OPTIONAL_INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(arraySchema.valueSchema().fields().size(), equalTo(1)); assertThat(arraySchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(arraySchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
protected static void validateSchemaNames(Schema schema) { if (schema == null) return; String schemaName = schema.name(); if (schemaName != null && !SchemaNameAdjuster.isValidFullname(schemaName)) { fail("Kafka schema '" + schemaName + "' is not a valid Avro schema name"); } if (schema.type() == Type.STRUCT) { schema.fields().forEach(field -> { validateSubSchemaNames(schema, field); }); } }
@Test public void booleanTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("BOOLEAN_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); assertThat(schemaA.defaultValue()).isEqualTo((short) 0); assertThat(schemaB.defaultValue()).isEqualTo((short) 1); assertThat(schemaC.defaultValue()).isEqualTo((short) 1); assertThat(schemaD.defaultValue()).isEqualTo((short) 1); assertThat(schemaE.defaultValue()).isEqualTo(null); }
@Test public void shouldTranslateNested() { final Schema connectInnerSchema = SchemaBuilder .struct() .field("intField", Schema.INT32_SCHEMA) .build(); final Schema connectSchema = SchemaBuilder .struct() .field("structField", connectInnerSchema) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("STRUCTFIELD"), notNullValue()); final Schema innerSchema = ksqlSchema.field("STRUCTFIELD").schema(); assertThat(innerSchema.fields().size(), equalTo(connectInnerSchema.fields().size())); for (int i = 0; i < connectInnerSchema.fields().size(); i++) { assertThat( innerSchema.fields().get(i).name().toUpperCase(), equalTo(connectInnerSchema.fields().get(i).name().toUpperCase())); assertThat( innerSchema.fields().get(i).schema().type(), equalTo(connectInnerSchema.fields().get(i).schema().type())); assertThat(innerSchema.fields().get(i).schema().isOptional(), is(true)); } }
public static Set<Integer> getRowTimeRowKeyIndexes(final Schema schema) { final Set<Integer> indexSet = new HashSet<>(); for (int i = 0; i < schema.fields().size(); i++) { final Field field = schema.fields().get(i); if (field.name().equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) || field.name().equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { indexSet.add(i); } } return indexSet; }
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
private SchemaBuilder copySchemaExcludingName(Schema source, SchemaBuilder builder, boolean copyFields) { builder.version(source.version()); builder.doc(source.doc()); Map<String, String> params = source.parameters(); if (params != null) { builder.parameters(params); } if (source.isOptional()) { builder.optional(); } else { builder.required(); } if (copyFields) { for (org.apache.kafka.connect.data.Field field : source.fields()) { builder.field(field.name(), field.schema()); } } return builder; } }
@Test public void shouldTranslatePrimitives() { final Schema connectSchema = SchemaBuilder .struct() .field("intField", Schema.INT32_SCHEMA) .field("longField", Schema.INT64_SCHEMA) .field("doubleField", Schema.FLOAT64_SCHEMA) .field("stringField", Schema.STRING_SCHEMA) .field("booleanField", Schema.BOOLEAN_SCHEMA) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.schema().type(), equalTo(Schema.Type.STRUCT)); assertThat(ksqlSchema.fields().size(), equalTo(connectSchema.fields().size())); for (int i = 0; i < ksqlSchema.fields().size(); i++) { assertThat( ksqlSchema.fields().get(i).name(), equalTo(connectSchema.fields().get(i).name().toUpperCase())); assertThat( ksqlSchema.fields().get(i).schema().type(), equalTo(connectSchema.fields().get(i).schema().type())); assertThat(ksqlSchema.fields().get(i).schema().isOptional(), is(true)); } }
@Test public void shouldGetArraySchemaFromListClass() throws NoSuchMethodException { final Type type = getClass().getDeclaredMethod("listType", List.class) .getGenericParameterTypes()[0]; final Schema schema = SchemaUtil.getSchemaFromType(type); assertThat(schema.type(), equalTo(Schema.Type.ARRAY)); assertThat(schema.valueSchema(), equalTo(Schema.OPTIONAL_FLOAT64_SCHEMA)); }