public static Operation forCode(String code) { for (Operation op : Operation.values()) { if (op.code().equalsIgnoreCase(code)) return op; } return null; }
SourceRecord getRecordByOperation(Envelope.Operation operation) throws InterruptedException { final SourceRecord candidateRecord = getNextRecord(); if (!((Struct) candidateRecord.value()).get("op").equals(operation.code())) { // MongoDB is not providing really consistent snapshot, so the initial insert // can arrive both in initial sync snapshot and in oplog return getRecordByOperation(operation); } return candidateRecord; }
value.put(FieldName.OPERATION, operation.code()); value.put(FieldName.TIMESTAMP, timestamp); SourceRecord record = new SourceRecord(sourcePartition, offset, topicName, partition, keySchema, key, valueSchema, value);
assertThat(operationHeader.next().value().toString()).isEqualTo(Envelope.Operation.DELETE.code());
+ "}" ); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE);
assertThat(value.getString(FieldName.AFTER)).isNull(); assertThat(value.getString("patch")).isNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE);
@Test @FixFor("DBZ-582") public void shouldGenerateRecordForDeleteEventWithoutTombstone() throws InterruptedException { RecordMakers recordMakers = new RecordMakers(filters, source, topicSelector, produced::add, false); BsonTimestamp ts = new BsonTimestamp(1000, 1); CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); ObjectId objId = new ObjectId(); Document obj = new Document("_id", objId); Document event = new Document().append("o", obj) .append("ns", "dbA.c1") .append("ts", ts) .append("h", new Long(12345678)) .append("op", "d"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo(JSONSerializers.getStrict().serialize(objId)); assertThat(value.schema()).isSameAs(record.valueSchema()); assertThat(value.getString(FieldName.AFTER)).isNull(); assertThat(value.getString("patch")).isNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
@Test public void shouldGenerateRecordForUpdateEvent() throws InterruptedException { BsonTimestamp ts = new BsonTimestamp(1000, 1); CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); ObjectId objId = new ObjectId(); Document obj = new Document().append("$set", new Document("name", "Sally")); Document event = new Document().append("o", obj) .append("o2", objId) .append("ns", "dbA.c1") .append("ts", ts) .append("h", Long.valueOf(12345678)) .append("op", "u"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo(JSONSerializers.getStrict().serialize(objId)); assertThat(value.schema()).isSameAs(record.valueSchema()); // assertThat(value.getString(FieldName.BEFORE)).isNull(); assertThat(value.getString(FieldName.AFTER)).isNull(); assertThat(value.getString("patch")).isEqualTo(obj.toJson(WRITER_SETTINGS)); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.UPDATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
@Test public void shouldGenerateRecordForInsertEvent() throws InterruptedException { CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); BsonTimestamp ts = new BsonTimestamp(1000, 1); ObjectId objId = new ObjectId(); Document obj = new Document().append("_id", objId).append("name", "Sally"); Document event = new Document().append("o", obj) .append("ns", "dbA.c1") .append("ts", ts) .append("h", Long.valueOf(12345678)) .append("op", "i"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo("{ \"$oid\" : \"" + objId + "\"}"); assertThat(value.schema()).isSameAs(record.valueSchema()); // assertThat(value.getString(FieldName.BEFORE)).isNull(); assertThat(value.getString(FieldName.AFTER)).isEqualTo(obj.toJson(WRITER_SETTINGS)); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#DELETE DELETE} record. * * @param record the source record; may not be null */ public static void isValidDelete(SourceRecord record, boolean keyExpected) { if (keyExpected) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); } else { assertThat(record.key()).isNull(); assertThat(record.keySchema()).isNull(); } assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code()); assertThat(value.get(FieldName.BEFORE)).isNotNull(); assertThat(value.get(FieldName.AFTER)).isNull(); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#CREATE INSERT/CREATE} record. * * @param record the source record; may not be null */ public static void isValidInsert(SourceRecord record, boolean keyExpected) { if (keyExpected) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); } else { assertThat(record.key()).isNull(); assertThat(record.keySchema()).isNull(); } assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.get(FieldName.AFTER)).isNotNull(); assertThat(value.get(FieldName.BEFORE)).isNull(); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#UPDATE UPDATE} record. * * @param record the source record; may not be null */ public static void isValidUpdate(SourceRecord record, boolean keyExpected) { if (keyExpected) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); } else { assertThat(record.key()).isNull(); assertThat(record.keySchema()).isNull(); } assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.UPDATE.code()); assertThat(value.get(FieldName.AFTER)).isNotNull(); // assertThat(value.get(FieldName.BEFORE)).isNull(); // may be null }
/** * Generate an {@link Operation#UPDATE update} message with the given information. * * @param before the state of the record before the update; may be null * @param after the state of the record after the update; may not be null * @param source the information about the source where the update occurred; may be null * @param timestamp the timestamp for this message; may be null * @return the update message; never null */ public Struct update(Object before, Struct after, Struct source, Long timestamp) { Struct struct = new Struct(schema); struct.put(FieldName.OPERATION, Operation.UPDATE.code()); if (before != null) struct.put(FieldName.BEFORE, before); struct.put(FieldName.AFTER, after); if (source != null) struct.put(FieldName.SOURCE, source); if (timestamp != null) struct.put(FieldName.TIMESTAMP, timestamp); return struct; }
/** * Generate a {@link Operation#READ read} message with the given information. * * @param record the state of the record as read; may not be null * @param source the information about the source that was read; may be null * @param timestamp the timestamp for this message; may be null * @return the read message; never null */ public Struct read(Object record, Struct source, Long timestamp) { Struct struct = new Struct(schema); struct.put(FieldName.OPERATION, Operation.READ.code()); struct.put(FieldName.AFTER, record); if (source != null) struct.put(FieldName.SOURCE, source); if (timestamp != null) struct.put(FieldName.TIMESTAMP, timestamp); return struct; }
/** * Generate an {@link Operation#DELETE delete} message with the given information. * * @param before the state of the record before the delete; may be null * @param source the information about the source where the deletion occurred; may be null * @param timestamp the timestamp for this message; may be null * @return the delete message; never null */ public Struct delete(Object before, Struct source, Long timestamp) { Struct struct = new Struct(schema); struct.put(FieldName.OPERATION, Operation.DELETE.code()); if (before != null) struct.put(FieldName.BEFORE, before); if (source != null) struct.put(FieldName.SOURCE, source); if (timestamp != null) struct.put(FieldName.TIMESTAMP, timestamp); return struct; }
@Test public void testHandleCreateRewrite() { try (final UnwrapFromEnvelope<SourceRecord> transform = new UnwrapFromEnvelope<>()) { final Map<String, String> props = new HashMap<>(); props.put(HANDLE_DELETES, "rewrite"); props.put(OPERATION_HEADER, "true"); transform.configure(props); final SourceRecord createRecord = createCreateRecord(); final SourceRecord unwrapped = transform.apply(createRecord); assertThat(((Struct)unwrapped.value()).getString("__deleted")).isEqualTo("false"); assertThat(unwrapped.headers()).hasSize(1); String headerValue = getSourceRecordHeaderByKey(unwrapped, transform.DEBEZIUM_OPERATION_HEADER_KEY); assertThat(headerValue).isEqualTo(Envelope.Operation.CREATE.code()); } }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#READ READ} record. * * @param record the source record; may not be null */ public static void isValidRead(SourceRecord record) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.READ.code()); assertThat(value.get(FieldName.AFTER)).isNotNull(); assertThat(value.get(FieldName.BEFORE)).isNull(); }
@Test public void testDeleteForwardConfigured() { try (final UnwrapFromEnvelope<SourceRecord> transform = new UnwrapFromEnvelope<>()) { final Map<String, String> props = new HashMap<>(); props.put(DROP_DELETES, "false"); props.put(OPERATION_HEADER, "true"); transform.configure(props); final SourceRecord deleteRecord = createDeleteRecord(); final SourceRecord tombstone = transform.apply(deleteRecord); assertThat(tombstone.value()).isNull(); assertThat(tombstone.headers()).hasSize(1); String headerValue = getSourceRecordHeaderByKey(tombstone, transform.DEBEZIUM_OPERATION_HEADER_KEY); assertThat(headerValue).isEqualTo(Envelope.Operation.DELETE.code()); } }
/** * Generate a {@link Operation#CREATE create} message with the given information. * * @param record the state of the record after creation; may not be null * @param source the information about the source where the creation occurred; may be null * @param timestamp the timestamp for this message; may be null * @return the create message; never null */ public Struct create(Object record, Struct source, Long timestamp) { Struct struct = new Struct(schema); struct.put(FieldName.OPERATION, Operation.CREATE.code()); struct.put(FieldName.AFTER, record); if (source != null) struct.put(FieldName.SOURCE, source); if (timestamp != null) struct.put(FieldName.TIMESTAMP, timestamp); return struct; }
protected void verifyOperation(SourceRecord record, Operation expected) { Struct value = (Struct) record.value(); assertThat(value.getString(Envelope.FieldName.OPERATION)).isEqualTo(expected.code()); }