protected void assertRequiredField(Envelope env, String fieldName, Schema expectedSchema) { assertField(env.schema().field(fieldName),fieldName,expectedSchema, false); }
protected void assertOptionalField(Envelope env, String fieldName, Schema expectedSchema) { assertField(env.schema().field(fieldName),fieldName,expectedSchema, true); }
protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
keySchema, oldKey, envelope.schema(), envelope.delete(valueBefore, origin, ts)); consumer.accept(record); ++count; keySchema, key, envelope.schema(), envelope.create(valueAfter, origin, ts)); consumer.accept(record); ++count; keySchema, key, envelope.schema(), envelope.update(valueBefore, valueAfter, origin, ts)); consumer.accept(record); ++count;
private SourceRecord createCreateRecord() { final Schema recordSchema = SchemaBuilder.struct().field("id", SchemaBuilder.int8()).build(); Envelope envelope = Envelope.defineSchema() .withName("dummy.Envelope") .withRecord(recordSchema) .withSource(SchemaBuilder.struct().build()) .build(); final Struct before = new Struct(recordSchema); before.put("id", (byte)1); final Struct payload = envelope.create(before, null, System.nanoTime()); return new SourceRecord(new HashMap<>(), new HashMap<>(), "dummy", envelope.schema(), payload); }
private SourceRecord createDeleteRecord() { final Schema recordSchema = SchemaBuilder.struct().field("id", SchemaBuilder.int8()).build(); Envelope envelope = Envelope.defineSchema() .withName("dummy.Envelope") .withRecord(recordSchema) .withSource(SchemaBuilder.struct().build()) .build(); final Struct before = new Struct(recordSchema); before.put("id", (byte)1); final Struct payload = envelope.delete(before, null, System.nanoTime()); return new SourceRecord(new HashMap<>(), new HashMap<>(), "dummy", envelope.schema(), payload); }
@Override public int read(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.read(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
@Override public int insert(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.create(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
new SourceRecord( partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.delete(value, sourceInfo.source(), clock().currentTimeInMillis())), lastCompletelyProcessedLsn);
@Test public void shouldBuildWithSimpleOptionalTypesForBeforeAndAfter() { Envelope env = Envelope.defineSchema() .withName("someName") .withRecord(Schema.OPTIONAL_STRING_SCHEMA) .withSource(Schema.OPTIONAL_INT64_SCHEMA) .build(); assertThat(env.schema()).isNotNull(); assertThat(env.schema().name()).isEqualTo("someName"); assertThat(env.schema().doc()).isNull(); assertThat(env.schema().version()).isNull(); assertOptionalField(env, Envelope.FieldName.AFTER, Schema.OPTIONAL_STRING_SCHEMA); assertOptionalField(env, Envelope.FieldName.BEFORE, Schema.OPTIONAL_STRING_SCHEMA); assertOptionalField(env, Envelope.FieldName.SOURCE, Schema.OPTIONAL_INT64_SCHEMA); assertRequiredField(env, Envelope.FieldName.OPERATION, Schema.STRING_SCHEMA); }
partition, offset, topicName, null, oldKeySchema, oldKey, envelope.schema(), envelope.delete(oldValue, source, clock().currentTimeInMillis())), lastCompletelyProcessedLsn); partition, offset, topicName, null, newKeySchema, newKey, envelope.schema(), envelope.create(newValue, source, clock().currentTimeInMillis())), lastCompletelyProcessedLsn); } else { SourceRecord record = new SourceRecord(partition, offset, topicName, null, newKeySchema, newKey, envelope.schema(), envelope.update(oldValue, newValue, source, clock().currentTimeInMillis())); recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn));
protected void generateReadRecord(TableId tableId, Object[] rowData) { if (rowData.length == 0) { return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (key == null || value == null) { return; } Schema keySchema = tableSchema.keySchema(); sourceInfo.update(clock().currentTimeInMicros(), tableId); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.read(value, sourceInfo.source(), clock().currentTimeInMillis()))); }
@Override public int delete(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { int count = 0; Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); // Send a delete message ... SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.delete(value, origin, ts)); consumer.accept(record); ++count; // And send a tombstone ... if (emitTombstoneOnDelete) { record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, null, null); consumer.accept(record); ++count; } } return count; }
protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
new SourceRecord( partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.delete(value, sourceInfo.source(), clock().currentTimeInMillis())), lastCompletelyProcessedLsn);
protected void generateReadRecord(TableId tableId, Object[] rowData) { // Clear the existing record to prevent reprocessing stale data. currentRecord.set(null); if (rowData.length == 0) { return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (key == null || value == null) { return; } Schema keySchema = tableSchema.keySchema(); sourceInfo.update(clock().currentTimeInMicros(), tableId); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.read(value, sourceInfo.source(), clock().currentTimeInMillis()))); }
partition, offset, topicName, null, oldKeySchema, oldKey, envelope.schema(), envelope.delete(oldValue, source, clock().currentTimeInMillis())), lastCompletelyProcessedLsn); partition, offset, topicName, null, newKeySchema, newKey, envelope.schema(), envelope.create(newValue, source, clock().currentTimeInMillis())), lastCompletelyProcessedLsn); } else { SourceRecord record = new SourceRecord(partition, offset, topicName, null, newKeySchema, newKey, envelope.schema(), envelope.update(oldValue, newValue, source, clock().currentTimeInMillis())); recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn));