private void changeSourceToLastSnapshotRecord(SourceRecord currentRecord) { final Struct envelope = (Struct)currentRecord.value(); final Struct source = (Struct)envelope.get("source"); if (source.getBoolean(SourceInfo.LAST_SNAPSHOT_RECORD_KEY) != null) { source.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, true); } }
/** * Verify that the given {@link SourceRecord} is a valid tombstone, meaning it has a non-null key and key schema but null * value and value schema. * * @param record the source record; may not be null */ public static void isValidTombstone(SourceRecord record) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); assertThat(record.value()).isNull(); assertThat(record.valueSchema()).isNull(); }
@Override public void accept(SourceRecord record) { if (record.value() == null) { tombstones.incrementAndGet(); } else { Operation op = Envelope.operationFor(record); if (op != null) { statsByOperation.computeIfAbsent(op, key -> new AtomicLong()).incrementAndGet(); } } }
SourceRecord getRecordByOperation(Envelope.Operation operation) throws InterruptedException { final SourceRecord candidateRecord = getNextRecord(); if (!((Struct) candidateRecord.value()).get("op").equals(operation.code())) { // MongoDB is not providing really consistent snapshot, so the initial insert // can arrive both in initial sync snapshot and in oplog return getRecordByOperation(operation); } return candidateRecord; }
public SourceRecordAssert valueAfterFieldIsEqualTo(Struct expectedValue) { Struct value = (Struct) record.value(); Struct afterValue = (Struct) value.get("after"); Assertions.assertThat(afterValue).isEqualTo(expectedValue); return this; }
protected void consumeLines(int numberOfLines) throws InterruptedException { consumeRecords(numberOfLines, record -> { String line = record.value().toString(); assertThat(line).isEqualTo(generateLine(nextConsumedLineNumber)); ++nextConsumedLineNumber; }); } }
@Test public void testHandleDeleteNone() { try (final UnwrapFromEnvelope<SourceRecord> transform = new UnwrapFromEnvelope<>()) { final Map<String, String> props = new HashMap<>(); props.put(HANDLE_DELETES, "none"); transform.configure(props); final SourceRecord deleteRecord = createDeleteRecord(); final SourceRecord tombstone = transform.apply(deleteRecord); assertThat(tombstone.value()).isNull(); } }
protected static Struct valueFor(SourceRecord record) { Struct envelope = (Struct) record.value(); Field afterField = envelope.schema().field(Envelope.FieldName.AFTER); if (afterField != null) { return envelope.getStruct(afterField.name()); } return null; }
protected static Struct sourceFor(SourceRecord record) { Struct envelope = (Struct) record.value(); Field field = envelope.schema().field(Envelope.FieldName.SOURCE); if (field != null) { return envelope.getStruct(field.name()); } return null; } }
private void assertEmptyFieldValue(SourceRecord record, String fieldName) { final Struct envelope = (Struct)record.value(); final Struct after = (Struct)envelope.get("after"); assertThat(after.getWithoutDefault(fieldName)).isNull(); }
protected void assertSourceInfo(SourceRecord record, String db, String schema, String table) { assertTrue(record.value() instanceof Struct); Struct source = ((Struct) record.value()).getStruct("source"); assertEquals(db, source.getString("db")); assertEquals(schema, source.getString("schema")); assertEquals(table, source.getString("table")); }
protected String getAffectedDatabase(SourceRecord record) { Struct value = (Struct) record.value(); if (value != null) { Field dbField = value.schema().field(HistoryRecord.Fields.DATABASE_NAME); if (dbField != null) { return value.getString(dbField.name()); } } return null; }
@Test public void testHandleDeleteRewrite() { try (final UnwrapFromEnvelope<SourceRecord> transform = new UnwrapFromEnvelope<>()) { final Map<String, String> props = new HashMap<>(); props.put(HANDLE_DELETES, "rewrite"); transform.configure(props); final SourceRecord deleteRecord = createDeleteRecord(); final SourceRecord unwrapped = transform.apply(deleteRecord); assertThat(((Struct)unwrapped.value()).getString("__deleted")).isEqualTo("true"); } }
protected void verifyFromInitialSync(SourceRecord record, AtomicBoolean foundLast) { if (record.sourceOffset().containsKey(SourceInfo.INITIAL_SYNC)) { assertThat(record.sourceOffset().containsKey(SourceInfo.INITIAL_SYNC)).isTrue(); Struct value = (Struct) record.value(); assertThat(value.getStruct(Envelope.FieldName.SOURCE).getBoolean(SourceInfo.INITIAL_SYNC)).isTrue(); } else { // Only the last record in the initial sync should be marked as not being part of the initial sync ... assertThat(foundLast.getAndSet(true)).isFalse(); } }
@Test public void intTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_int"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_INT); }
@Test public void fpTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_fp"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_FP); }
@Test public void dateTimeTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_time"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_DATE_TIME); }
protected void verifyNotFromInitialSync(SourceRecord record) { assertThat(record.sourceOffset().containsKey(SourceInfo.INITIAL_SYNC)).isFalse(); Struct value = (Struct) record.value(); assertThat(value.getStruct(Envelope.FieldName.SOURCE).getBoolean(SourceInfo.INITIAL_SYNC)).isNull(); }
@Test public void otherTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_xml"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_XML); }