protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
@Override public boolean equals(Object obj) { if ( obj == this ) return true; if ( obj instanceof TableSchema ) { TableSchema that = (TableSchema)obj; return Objects.equals(this.keySchema(),that.keySchema()) && Objects.equals(this.valueSchema(),that.valueSchema()); } return false; }
private void emitDeleteRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); }
@Override public int insert(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.create(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
assertThat(schema).isNotNull(); assertThat(schema.keySchema()).isNull(); assertThat(schema.keyFromColumnData(data)).isNull(); Schema values = schema.valueSchema(); assertThat(values).isNotNull(); assertThat(values.field("C1").name()).isEqualTo("C1"); assertThat(values.field("C6").schema()).isEqualTo(SchemaBuilder.int16().build()); Struct value = schema.valueFromColumnData(data); assertThat(value).isNotNull(); assertThat(value.get("C1")).isEqualTo("c1value");
@Override public int hashCode() { return valueSchema().hashCode(); }
protected void assertKeySchema(String fullyQualifiedTableName, String fields, Schema... types) { TableSchema tableSchema = schemaFor(fullyQualifiedTableName); Schema keySchema = tableSchema.keySchema(); assertSchemaContent(fields.split(","), types, keySchema); }
Envelope envelope = tableSchema.getEnvelopeSchema();
return new TableSchema(tableId, keySchema, keyGenerator, envelope, valSchema, valueGenerator);
private void emitReadRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().read(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.READ, newKey, envelope, offsetContext); }
@Override public int read(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.read(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
protected void assertTableSchema(String fullyQualifiedTableName, String fields, Schema... types) { TableSchema tableSchema = schemaFor(fullyQualifiedTableName); Schema keySchema = tableSchema.valueSchema(); assertSchemaContent(fields.split(","), types, keySchema); }
protected void assertKeySchema(String fullyQualifiedTableName, String fields, Schema... types) { TableSchema tableSchema = schemaFor(fullyQualifiedTableName); Schema keySchema = tableSchema.keySchema(); assertSchemaContent(fields.split(","), types, keySchema); }
return new TableSchema(tableId, keySchema, keyGenerator, envelope, valSchema, valueGenerator);
protected void generateReadRecord(TableId tableId, Object[] rowData) { if (rowData.length == 0) { return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (key == null || value == null) { return; } Schema keySchema = tableSchema.keySchema(); sourceInfo.update(clock().currentTimeInMicros(), tableId); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); currentRecord.set(new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.read(value, sourceInfo.source(), clock().currentTimeInMillis()))); }
private void emitCreateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); }
@Override public String toString() { return "{ key : " + SchemaUtil.asString(keySchema()) + ", value : " + SchemaUtil.asString(valueSchema()) + " }"; } }
@Override public int delete(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { int count = 0; Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); // Send a delete message ... SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.delete(value, origin, ts)); consumer.accept(record); ++count; // And send a tombstone ... if (emitTombstoneOnDelete) { record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, null, null); consumer.accept(record); ++count; } } return count; }
protected void assertColumnsExcluded(String...columnNames) { Arrays.stream(columnNames).forEach(fqColumnName -> { int lastDotIdx = fqColumnName.lastIndexOf("."); String fullyQualifiedTableName = fqColumnName.substring(0, lastDotIdx); String columnName = lastDotIdx > 0 ? fqColumnName.substring(lastDotIdx + 1) : fqColumnName; TableSchema tableSchema = schemaFor(fullyQualifiedTableName); assertNotNull(fullyQualifiedTableName + " not included", tableSchema); Schema valueSchema = tableSchema.valueSchema(); assertNotNull(fullyQualifiedTableName + ".Value schema not included", valueSchema); assertNull(columnName + " not excluded;", valueSchema.field(columnName)); }); }
Object key = tableSchema.keyFromColumnData(oldRowData); Struct value = tableSchema.valueFromColumnData(oldRowData); if (value == null) { logger.warn("ignoring delete message for table '{}' because it does not have a primary key defined and replica identity for the table is not FULL", tableId); return; Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema();