public SourceRecordAssert valueAfterFieldSchemaIsEqualTo(Schema expectedSchema) { Schema valueSchema = record.valueSchema(); Schema afterFieldSchema = valueSchema.field("after").schema(); VerifyRecord.assertConnectSchemasAreEqual(null, afterFieldSchema, expectedSchema); return this; } }
/** * Utility method to replace the offset in the given record with the latest. This is used on the last record produced * during the snapshot. * * @param record the record * @return the updated record */ protected SourceRecord replaceOffset(SourceRecord record) { if (record == null) return null; Map<String, ?> newOffset = context.source().offset(); return new SourceRecord(record.sourcePartition(), newOffset, record.topic(), record.kafkaPartition(), record.keySchema(), record.key(), record.valueSchema(), record.value()); }
@Test public void numericAndDecimalToDecimalTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .with(MySqlConnectorConfig.DECIMAL_HANDLING_MODE, JdbcValueConverters.DecimalMode.PRECISE) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("NUMERIC_DECIMAL_TABLE")).get(0); // TODO can't validate due to https://github.com/confluentinc/schema-registry/issues/833 // enable once that's resolved upstream // validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); assertThat(schemaA.defaultValue()).isEqualTo(BigDecimal.valueOf(1.23)); assertThat(schemaB.defaultValue()).isEqualTo(BigDecimal.valueOf(2.321)); assertEmptyFieldValue(record, "D"); }
@Test public void floatAndDoubleTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("FlOAT_DOUBLE_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); assertThat(schemaA.defaultValue()).isEqualTo(0d); assertThat(schemaB.defaultValue()).isEqualTo(1.0d); assertEmptyFieldValue(record, "H"); }
@Test public void realTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("REAL_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); assertThat(schemaA.defaultValue()).isEqualTo(1d); assertThat(schemaB.defaultValue()).isEqualTo(null); assertEmptyFieldValue(record, "C"); }
@Test public void numericAndDecimalToDoubleTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .with(MySqlConnectorConfig.DECIMAL_HANDLING_MODE, JdbcValueConverters.DecimalMode.DOUBLE) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("NUMERIC_DECIMAL_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); assertThat(schemaA.defaultValue()).isEqualTo(1.23d); assertThat(schemaB.defaultValue()).isEqualTo(2.321d); assertThat(schemaC.defaultValue()).isEqualTo(12.678d); assertEmptyFieldValue(record, "D"); }
/** * Verify that the given {@link SourceRecord} is a valid tombstone, meaning it has a non-null key and key schema but null * value and value schema. * * @param record the source record; may not be null */ public static void isValidTombstone(SourceRecord record) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); assertThat(record.value()).isNull(); assertThat(record.valueSchema()).isNull(); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/pop/#up._S_pop">MongoDB operator array update $pop</a> */ @Test public void shouldTransformOperationPop() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$pop': {dataArrayOfStr: -1}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("c", "e")); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/pull/#pull">MongoDB operator array update $pull</a> */ @Test public void shouldTransformOperationPull() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$pull': {dataArrayOfStr: {$in: ['c']}}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("a", "e")); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/addToSet/#up._S_addToSet">MongoDB operator array update $addToSet</a> */ @Test public void shouldTransformOperationAddToSet() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$addToSet': {dataArrayOfStr: 'b'}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("a", "c", "e", "b")); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/pullAll/#up._S_pullAll">MongoDB operator array update $pullAll</a> */ @Test public void shouldTransformOperationPullAll() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$pullAll': {dataArrayOfStr: ['c']}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("a", "e")); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/bit/#bitwise-and">MongoDB operator update $bit AND</a> */ @Test public void shouldTransformOperationBitAnd() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$bit': {dataInt: {and: NumberInt(1010)}}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(114); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/bit/#bitwise-or">MongoDB operator update $bit OR</a> */ @Test public void shouldTransformOperationBitOr() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$bit': {dataInt: {or: NumberInt(1001)}}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(1019); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/bit/#bitwise-xor">MongoDB operator update $bit XOR</a> */ @Test public void shouldTransformOperationBitXor() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$bit': {dataInt: {xor: NumberInt(111)}}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(20); } }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/push/#push">MongoDB operator array update $push</a> */ @Test public void shouldTransformOperationPush() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$push': {dataArrayOfStr: 'g'}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); // Operations which include items to arrays result in a new field where the structure looks like "FIELD_NAME.ARRAY_INDEX" VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr.3", valueSchema.field("dataArrayOfStr.3").schema(), Schema.OPTIONAL_STRING_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr.3")).isEqualTo("g"); } }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/max/#up._S_max">MongoDB operator update $max</a> */ @Test public void shouldTransformOperationMax() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$max': {'dataInt': 122, 'nested.dataInt': 124}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("nested.dataInt", valueSchema.field("nested.dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); // Since 122 < 123 we should expect "dataInt" to not be present assertThat(valueSchema.field("dataInt")).isNull(); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("nested.dataInt")).isEqualTo(124); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/min/#up._S_min">MongoDB operator update $min</a> */ @Test public void shouldTransformOperationMin() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$min': {'dataInt': 122, 'nested.dataInt': 124}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); // Since 124 > 123 we should expect "nested.dataInt" to not be present assertThat(valueSchema.field("nested.dataInt")).isNull(); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(122); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#READ READ} record. * * @param record the source record; may not be null */ public static void isValidRead(SourceRecord record) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.READ.code()); assertThat(value.get(FieldName.AFTER)).isNotNull(); assertThat(value.get(FieldName.BEFORE)).isNull(); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/inc/#up._S_inc">MongoDB operator update $inc</a> */ @Test public void shouldTransformOperationInc() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$inc': {'dataInt': 123, 'nested.dataInt': -23}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("nested.dataInt", valueSchema.field("nested.dataInt").schema(), Schema.OPTIONAL_INT32_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(246); assertThat(transformedUpdateValue.get("nested.dataInt")).isEqualTo(100); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/rename/#up._S_rename">MongoDB operator update $rename</a> */ @Test public void shouldTransformOperationRename() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$rename': {'dataInt': 'dataIntNewName', 'nonExistentField': 'nonExistentFieldRenamed'}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataIntNewName", valueSchema.field("dataIntNewName").schema(), Schema.OPTIONAL_INT32_SCHEMA); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataIntNewName")).isEqualTo(123); // Ensure the rename causes the old field value to be set to null VerifyRecord.assertConnectSchemasAreEqual("dataInt", valueSchema.field("dataInt").schema(), Schema.OPTIONAL_STRING_SCHEMA); assertThat(transformedUpdateValue.get("dataInt")).isEqualTo(null); }