protected Field[] fieldsForColumns(Schema schema, List<Column> columns) { Field[] fields = new Field[columns.size()]; AtomicInteger i = new AtomicInteger(0); columns.forEach(column -> { Field field = schema.field(column.name()); // may be null if the field is unused ... fields[i.getAndIncrement()] = field; }); return fields; }
private Struct updateEnvelope(Schema newEnvelopeSchema, Struct oldEnvelope) { final Struct newEnvelope = new Struct(newEnvelopeSchema); final Schema newValueSchema = newEnvelopeSchema.field(Envelope.FieldName.BEFORE).schema(); for (org.apache.kafka.connect.data.Field field : oldEnvelope.schema().fields()) { final String fieldName = field.name(); Object fieldValue = oldEnvelope.get(field); if ((Objects.equals(fieldName, Envelope.FieldName.BEFORE) || Objects.equals(fieldName, Envelope.FieldName.AFTER)) && fieldValue != null) { fieldValue = updateValue(newValueSchema, requireStruct(fieldValue, "Updating schema")); } newEnvelope.put(fieldName, fieldValue); } return newEnvelope; }
protected void assertColumnsExcluded(String...columnNames) { Arrays.stream(columnNames).forEach(fqColumnName -> { int lastDotIdx = fqColumnName.lastIndexOf("."); String fullyQualifiedTableName = fqColumnName.substring(0, lastDotIdx); String columnName = lastDotIdx > 0 ? fqColumnName.substring(lastDotIdx + 1) : fqColumnName; TableSchema tableSchema = schemaFor(fullyQualifiedTableName); assertNotNull(fullyQualifiedTableName + " not included", tableSchema); Schema valueSchema = tableSchema.valueSchema(); assertNotNull(fullyQualifiedTableName + ".Value schema not included", valueSchema); assertNull(columnName + " not excluded;", valueSchema.field(columnName)); }); }
private void assertSchemaContent(String[] fields, Schema[] types, Schema keySchema) { IntStream.range(0, fields.length).forEach(i -> { String fieldName = fields[i].trim(); Field field = keySchema.field(Strings.unquoteIdentifierPart(fieldName)); assertNotNull(fieldName + " not found in schema", field); VerifyRecord.assertConnectSchemasAreEqual(fieldName, types[i], field.schema()); }); }
protected void assertRequiredField(Envelope env, String fieldName, Schema expectedSchema) { assertField(env.schema().field(fieldName),fieldName,expectedSchema, false); }
protected void assertOptionalField(Envelope env, String fieldName, Schema expectedSchema) { assertField(env.schema().field(fieldName),fieldName,expectedSchema, true); }
/** * Obtain the operation for the given source record. * * @param record the source record; may not be null * @return the operation, or null if no valid operation was found in the record */ public static Operation operationFor(SourceRecord record) { Struct value = (Struct) record.value(); Field opField = value.schema().field(FieldName.OPERATION); if (opField != null) { return Operation.forCode(value.getString(opField.name())); } return null; } }
private void assertSchema(Struct valueA, Schema expected) { Assertions.assertThat(valueA.schema().field("after").schema().field("cola").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("colb").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("colc").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("cold").schema()).isEqualTo(expected); } }
public SourceRecordAssert valueAfterFieldSchemaIsEqualTo(Schema expectedSchema) { Schema valueSchema = record.valueSchema(); Schema afterFieldSchema = valueSchema.field("after").schema(); VerifyRecord.assertConnectSchemasAreEqual(null, afterFieldSchema, expectedSchema); return this; } }
protected String getAffectedDatabase(SourceRecord record) { Struct value = (Struct) record.value(); if (value != null) { Field dbField = value.schema().field(HistoryRecord.Fields.DATABASE_NAME); if (dbField != null) { return value.getString(dbField.name()); } } return null; }
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMapWithNonStringKey() { final Schema connectSchema = SchemaBuilder .struct() .field("mapfield", SchemaBuilder.map(Schema.INT32_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
protected String getAffectedDatabase(SourceRecord record) { Struct envelope = (Struct) record.value(); Field dbField = envelope.schema().field(HistoryRecord.Fields.DATABASE_NAME); if (dbField != null) { return envelope.getString(dbField.name()); } return null; } }
protected static Struct valueFor(SourceRecord record) { Struct envelope = (Struct) record.value(); Field afterField = envelope.schema().field(Envelope.FieldName.AFTER); if (afterField != null) { return envelope.getStruct(afterField.name()); } return null; }
protected static Struct sourceFor(SourceRecord record) { Struct envelope = (Struct) record.value(); Field field = envelope.schema().field(Envelope.FieldName.SOURCE); if (field != null) { return envelope.getStruct(field.name()); } return null; } }
@Test public void shouldHaveSchemaForSource() { Schema schema = source.schema(); assertThat(schema.name()).isNotEmpty(); assertThat(schema.version()).isNotNull(); assertThat(schema.field(SourceInfo.SERVER_NAME).schema()).isEqualTo(Schema.STRING_SCHEMA); assertThat(schema.field(SourceInfo.REPLICA_SET_NAME).schema()).isEqualTo(Schema.STRING_SCHEMA); assertThat(schema.field(SourceInfo.NAMESPACE).schema()).isEqualTo(Schema.STRING_SCHEMA); assertThat(schema.field(SourceInfo.TIMESTAMP).schema()).isEqualTo(Schema.INT32_SCHEMA); assertThat(schema.field(SourceInfo.ORDER).schema()).isEqualTo(Schema.INT32_SCHEMA); assertThat(schema.field(SourceInfo.OPERATION_ID).schema()).isEqualTo(Schema.OPTIONAL_INT64_SCHEMA); assertThat(schema.field(SourceInfo.INITIAL_SYNC).schema()).isEqualTo(SchemaBuilder.bool().optional().defaultValue(false).build()); }
private void assertSchema(Struct content) { if (schema == null) { return; } Schema schema = content.schema(); Field field = schema.field(fieldName); assertNotNull(fieldName + " not found in schema " + SchemaUtil.asString(schema), field); VerifyRecord.assertConnectSchemasAreEqual(field.name(), field.schema(), this.schema); } }
private void assertSchema(Struct content) { if (schema == null) { return; } Schema schema = content.schema(); Field field = schema.field(fieldName); Assertions.assertThat(field).as(fieldName + " not found in schema " + schema).isNotNull(); VerifyRecord.assertConnectSchemasAreEqual(field.name(), field.schema(), this.schema); } }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/addToSet/#up._S_addToSet">MongoDB operator array update $addToSet</a> */ @Test public void shouldTransformOperationAddToSet() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$addToSet': {dataArrayOfStr: 'b'}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("a", "c", "e", "b")); }