private void emitDeleteRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); }
private void emitReadRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().read(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.READ, newKey, envelope, offsetContext); }
private void emitCreateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); }
protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
private void emitUpdateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object[] newColumnValues = getNewColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); // regular update if (Objects.equals(oldKey, newKey)) { Struct envelope = tableSchema.getEnvelopeSchema().update(oldValue, newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.UPDATE, newKey, envelope, offsetContext); } // PK update -> emit as delete and re-insert with new key else { Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); } }
@Override public int insert(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.create(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
@Override public int read(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.read(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
assert tableSchema != null; Object key = tableSchema.keyFromColumnData(oldRowData); Struct value = tableSchema.valueFromColumnData(oldRowData); if (value == null) { logger.warn("ignoring delete message for table '{}' because it does not have a primary key defined and replica identity for the table is not FULL", tableId);
int count = 0; Object key = tableSchema.keyFromColumnData(after); Struct valueAfter = tableSchema.valueFromColumnData(after); if (valueAfter != null || key != null) { Object oldKey = tableSchema.keyFromColumnData(before); Struct valueBefore = tableSchema.valueFromColumnData(before); Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition();
oldKey = tableSchema.keyFromColumnData(oldRowData); oldKeySchema = tableSchema.keySchema(); oldValue = tableSchema.valueFromColumnData(oldRowData); Struct newValue = tableSchema.valueFromColumnData(newRowData);
protected void generateReadRecord(TableId tableId, Object[] rowData) { if (rowData.length == 0) { return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (key == null || value == null) { return; } Schema keySchema = tableSchema.keySchema(); sourceInfo.update(clock().currentTimeInMicros(), tableId); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.read(value, sourceInfo.source(), clock().currentTimeInMillis()))); }
@Override public int delete(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { int count = 0; Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); // Send a delete message ... SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.delete(value, origin, ts)); consumer.accept(record); ++count; // And send a tombstone ... if (emitTombstoneOnDelete) { record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, null, null); consumer.accept(record); ++count; } } return count; }
assertThat(values.field("C6").schema()).isEqualTo(SchemaBuilder.int16().build()); Struct value = schema.valueFromColumnData(data); assertThat(value).isNotNull(); assertThat(value.get("C1")).isEqualTo("c1value");
private void emitCreateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); }
private void emitReadRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().read(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.READ, newKey, envelope, offsetContext); }
private void emitDeleteRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); }
protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
private void emitUpdateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object[] newColumnValues = getNewColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); // regular update if (Objects.equals(oldKey, newKey)) { Struct envelope = tableSchema.getEnvelopeSchema().update(oldValue, newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.UPDATE, newKey, envelope, offsetContext); } // PK update -> emit as delete and re-insert with new key else { Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); } }
assert tableSchema != null; Object key = tableSchema.keyFromColumnData(oldRowData); Struct value = tableSchema.valueFromColumnData(oldRowData); if (value == null) { logger.warn("ignoring delete message for table '{}' because it does not have a primary key defined and replica identity for the table is not FULL", tableId);
protected void generateReadRecord(TableId tableId, Object[] rowData) { // Clear the existing record to prevent reprocessing stale data. currentRecord.set(null); if (rowData.length == 0) { return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (key == null || value == null) { return; } Schema keySchema = tableSchema.keySchema(); sourceInfo.update(clock().currentTimeInMicros(), tableId); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.read(value, sourceInfo.source(), clock().currentTimeInMillis()))); }