private void assertBigintUnsignedLong(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting int-64 since we have forced Long mode for BIGINT UNSIGNED assertThat(after.schema().field("c1").schema()).isEqualTo(Schema.INT64_SCHEMA); assertThat(after.schema().field("c2").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate the schema first, we are expecting int-64 since we are dealing with signed-bigint. //So Signed BIGINT would be an INT64 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate candidates values, note the loss in precision which is expected since BIGINT UNSIGNED cannot always be represented by //a long datatype. switch (i) { case 1: assertThat(after.getInt64("c1")).isEqualTo(-1L); assertThat(after.getInt64("c2")).isEqualTo(-1L); assertThat(after.getInt64("c3")).isEqualTo(9223372036854775807L); break; case 2: assertThat(after.getInt64("c1")).isEqualTo(-4000000000000000001L); assertThat(after.getInt64("c2")).isEqualTo(-4000000000000000001L); assertThat(after.getInt64("c3")).isEqualTo(-1223372036854775807L); break; case 3: assertThat(after.getInt64("c1")).isEqualTo(0L); assertThat(after.getInt64("c2")).isEqualTo(0L); assertThat(after.getInt64("c3")).isEqualTo(-9223372036854775808L); } }
assertThat(after.getInt64("c1")).isEqualTo(4294967295L); assertThat(after.getInt64("c2")).isEqualTo(4294967295L); assertThat(after.getInt32("c3")).isEqualTo(2147483647); assertThat(after.getInt64("c4")).isEqualTo(4294967295L); assertThat(after.getInt64("c5")).isEqualTo(4294967295L); assertThat(after.getInt32("c6")).isEqualTo(2147483647); break; case 2: assertThat(after.getInt64("c1")).isEqualTo(3294967295L); assertThat(after.getInt64("c2")).isEqualTo(3294967295L); assertThat(after.getInt32("c3")).isEqualTo(-1147483647); assertThat(after.getInt64("c4")).isEqualTo(3294967295L); assertThat(after.getInt64("c5")).isEqualTo(3294967295L); assertThat(after.getInt32("c6")).isEqualTo(-1147483647); break; case 3: assertThat(after.getInt64("c1")).isEqualTo(0L); assertThat(after.getInt64("c2")).isEqualTo(0L); assertThat(after.getInt32("c3")).isEqualTo(-2147483648); assertThat(after.getInt64("c4")).isEqualTo(0L); assertThat(after.getInt64("c5")).isEqualTo(0L); assertThat(after.getInt32("c6")).isEqualTo(-2147483648);
private void assertBigintUnsignedPrecise(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting org.apache.kafka.connect.data.Decimal:Byte since we are dealing with unsignd-bigint //So Unsigned BIGINY would be an int32 type assertThat(after.schema().field("c1").schema()).isEqualTo(Decimal.builder(0).schema()); assertThat(after.schema().field("c2").schema()).isEqualTo(Decimal.builder(0).schema()); //Validate the schema first, we are expecting int-64 since we are dealing with signed-bigint. //So Signed BIGINT would be an INT64 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate candidates values switch (i) { case 1: assertThat(after.get("c1")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(9223372036854775807L); break; case 2: assertThat(after.get("c1")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(-1223372036854775807L); break; case 3: assertThat(after.get("c1")).isEqualTo(new BigDecimal("0")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("0")); assertThat(after.getInt64("c3")).isEqualTo(-9223372036854775808L); } }
equalTo(123)); assertThat( ((Struct)struct.getMap("MAP").get("bar")).getInt64("MAP_INNER"), equalTo(456L)); assertThat(
assertThat(recordSource.getInt64(SourceInfo.BINLOG_POSITION_OFFSET_KEY)).isEqualTo(positionOfEvent); assertThat(recordSource.getInt32(SourceInfo.BINLOG_ROW_IN_EVENT_OFFSET_KEY)).isEqualTo(row); assertThat(recordSource.getString(SourceInfo.BINLOG_FILENAME_OFFSET_KEY)).isEqualTo(FILENAME);
@Test public void shouldGenerateRecordForInsertEvent() throws InterruptedException { CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); BsonTimestamp ts = new BsonTimestamp(1000, 1); ObjectId objId = new ObjectId(); Document obj = new Document().append("_id", objId).append("name", "Sally"); Document event = new Document().append("o", obj) .append("ns", "dbA.c1") .append("ts", ts) .append("h", Long.valueOf(12345678)) .append("op", "i"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo("{ \"$oid\" : \"" + objId + "\"}"); assertThat(value.schema()).isSameAs(record.valueSchema()); // assertThat(value.getString(FieldName.BEFORE)).isNull(); assertThat(value.getString(FieldName.AFTER)).isEqualTo(obj.toJson(WRITER_SETTINGS)); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
@Test public void shouldGenerateRecordForUpdateEvent() throws InterruptedException { BsonTimestamp ts = new BsonTimestamp(1000, 1); CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); ObjectId objId = new ObjectId(); Document obj = new Document().append("$set", new Document("name", "Sally")); Document event = new Document().append("o", obj) .append("o2", objId) .append("ns", "dbA.c1") .append("ts", ts) .append("h", Long.valueOf(12345678)) .append("op", "u"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo(JSONSerializers.getStrict().serialize(objId)); assertThat(value.schema()).isSameAs(record.valueSchema()); // assertThat(value.getString(FieldName.BEFORE)).isNull(); assertThat(value.getString(FieldName.AFTER)).isNull(); assertThat(value.getString("patch")).isEqualTo(obj.toJson(WRITER_SETTINGS)); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.UPDATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
assertThat(struct.getInt32(SourceInfo.TIMESTAMP)).isEqualTo(100); assertThat(struct.getInt32(SourceInfo.ORDER)).isEqualTo(2); assertThat(struct.getInt64(SourceInfo.OPERATION_ID)).isEqualTo(1987654321L); assertThat(struct.getString(SourceInfo.NAMESPACE)).isEqualTo("dbA.collectA"); assertThat(struct.getString(SourceInfo.REPLICA_SET_NAME)).isEqualTo(REPLICA_SET_NAME);
@Test public void shouldReturnRecordedOffsetForUsedReplicaName() { Document event = new Document().append("ts", new BsonTimestamp(100, 2)) .append("h", Long.valueOf(1987654321)) .append("ns", "dbA.collectA"); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); source.offsetStructForEvent(REPLICA_SET_NAME, event); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(true); Map<String, ?> offset = source.lastOffset(REPLICA_SET_NAME); assertThat(offset.get(SourceInfo.TIMESTAMP)).isEqualTo(100); assertThat(offset.get(SourceInfo.ORDER)).isEqualTo(2); assertThat(offset.get(SourceInfo.OPERATION_ID)).isEqualTo(1987654321L); BsonTimestamp ts = source.lastOffsetTimestamp(REPLICA_SET_NAME); assertThat(ts.getTime()).isEqualTo(100); assertThat(ts.getInc()).isEqualTo(2); Struct struct = source.lastOffsetStruct(REPLICA_SET_NAME,new CollectionId(REPLICA_SET_NAME,"dbA","collectA")); assertThat(struct.getInt32(SourceInfo.TIMESTAMP)).isEqualTo(100); assertThat(struct.getInt32(SourceInfo.ORDER)).isEqualTo(2); assertThat(struct.getInt64(SourceInfo.OPERATION_ID)).isEqualTo(1987654321L); assertThat(struct.getString(SourceInfo.NAMESPACE)).isEqualTo("dbA.collectA"); assertThat(struct.getString(SourceInfo.REPLICA_SET_NAME)).isEqualTo(REPLICA_SET_NAME); assertThat(struct.getString(SourceInfo.SERVER_NAME)).isEqualTo("serverX"); assertThat(struct.getBoolean(SourceInfo.INITIAL_SYNC)).isNull(); }
@Test public void shouldReturnRecordedOffsetForUsedReplicaNameDuringInitialSync() { source.startInitialSync(REPLICA_SET_NAME); Document event = new Document().append("ts", new BsonTimestamp(100, 2)) .append("h", Long.valueOf(1987654321)) .append("ns", "dbA.collectA"); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); source.offsetStructForEvent(REPLICA_SET_NAME, event); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(true); Map<String, ?> offset = source.lastOffset(REPLICA_SET_NAME); assertThat(offset.get(SourceInfo.TIMESTAMP)).isEqualTo(100); assertThat(offset.get(SourceInfo.ORDER)).isEqualTo(2); assertThat(offset.get(SourceInfo.OPERATION_ID)).isEqualTo(1987654321L); BsonTimestamp ts = source.lastOffsetTimestamp(REPLICA_SET_NAME); assertThat(ts.getTime()).isEqualTo(100); assertThat(ts.getInc()).isEqualTo(2); Struct struct = source.lastOffsetStruct(REPLICA_SET_NAME,new CollectionId(REPLICA_SET_NAME,"dbA","collectA")); assertThat(struct.getInt32(SourceInfo.TIMESTAMP)).isEqualTo(100); assertThat(struct.getInt32(SourceInfo.ORDER)).isEqualTo(2); assertThat(struct.getInt64(SourceInfo.OPERATION_ID)).isEqualTo(1987654321L); assertThat(struct.getString(SourceInfo.NAMESPACE)).isEqualTo("dbA.collectA"); assertThat(struct.getString(SourceInfo.REPLICA_SET_NAME)).isEqualTo(REPLICA_SET_NAME); assertThat(struct.getString(SourceInfo.SERVER_NAME)).isEqualTo("serverX"); assertThat(struct.getBoolean(SourceInfo.INITIAL_SYNC)).isEqualTo(true); }
@Test @FixFor("DBZ-582") public void shouldGenerateRecordForDeleteEventWithoutTombstone() throws InterruptedException { RecordMakers recordMakers = new RecordMakers(filters, source, topicSelector, produced::add, false); BsonTimestamp ts = new BsonTimestamp(1000, 1); CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); ObjectId objId = new ObjectId(); Document obj = new Document("_id", objId); Document event = new Document().append("o", obj) .append("ns", "dbA.c1") .append("ts", ts) .append("h", new Long(12345678)) .append("op", "d"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo(JSONSerializers.getStrict().serialize(objId)); assertThat(value.schema()).isSameAs(record.valueSchema()); assertThat(value.getString(FieldName.AFTER)).isNull(); assertThat(value.getString("patch")).isNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId);
@Test public void shouldReturnOffsetForUnusedReplicaName() { assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); Map<String, ?> offset = source.lastOffset(REPLICA_SET_NAME); assertThat(offset.get(SourceInfo.TIMESTAMP)).isEqualTo(0); assertThat(offset.get(SourceInfo.ORDER)).isEqualTo(0); assertThat(offset.get(SourceInfo.OPERATION_ID)).isNull(); BsonTimestamp ts = source.lastOffsetTimestamp(REPLICA_SET_NAME); assertThat(ts.getTime()).isEqualTo(0); assertThat(ts.getInc()).isEqualTo(0); Struct struct = source.lastOffsetStruct(REPLICA_SET_NAME,new CollectionId(REPLICA_SET_NAME,"dbA","collectA")); assertThat(struct.getInt32(SourceInfo.TIMESTAMP)).isEqualTo(0); assertThat(struct.getInt32(SourceInfo.ORDER)).isEqualTo(0); assertThat(struct.getInt64(SourceInfo.OPERATION_ID)).isNull(); assertThat(struct.getString(SourceInfo.NAMESPACE)).isEqualTo("dbA.collectA"); assertThat(struct.getString(SourceInfo.REPLICA_SET_NAME)).isEqualTo(REPLICA_SET_NAME); assertThat(struct.getString(SourceInfo.SERVER_NAME)).isEqualTo("serverX"); assertThat(struct.getBoolean(SourceInfo.INITIAL_SYNC)).isNull(); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); }
@Test public void shouldReturnOffsetForUnusedReplicaNameDuringInitialSync() { source.startInitialSync(REPLICA_SET_NAME); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); Map<String, ?> offset = source.lastOffset(REPLICA_SET_NAME); assertThat(offset.get(SourceInfo.TIMESTAMP)).isEqualTo(0); assertThat(offset.get(SourceInfo.ORDER)).isEqualTo(0); assertThat(offset.get(SourceInfo.OPERATION_ID)).isNull(); BsonTimestamp ts = source.lastOffsetTimestamp(REPLICA_SET_NAME); assertThat(ts.getTime()).isEqualTo(0); assertThat(ts.getInc()).isEqualTo(0); Struct struct = source.lastOffsetStruct(REPLICA_SET_NAME,new CollectionId(REPLICA_SET_NAME,"dbA","collectA")); assertThat(struct.getInt32(SourceInfo.TIMESTAMP)).isEqualTo(0); assertThat(struct.getInt32(SourceInfo.ORDER)).isEqualTo(0); assertThat(struct.getInt64(SourceInfo.OPERATION_ID)).isNull(); assertThat(struct.getString(SourceInfo.NAMESPACE)).isEqualTo("dbA.collectA"); assertThat(struct.getString(SourceInfo.REPLICA_SET_NAME)).isEqualTo(REPLICA_SET_NAME); assertThat(struct.getString(SourceInfo.SERVER_NAME)).isEqualTo("serverX"); assertThat(struct.getBoolean(SourceInfo.INITIAL_SYNC)).isEqualTo(true); assertThat(source.hasOffset(REPLICA_SET_NAME)).isEqualTo(false); }
assertThat(value.getString("patch")).isNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.DELETE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId);
@Override public Instant getEventTimestamp(DataCollectionId source, OffsetContext offset, Object key, Struct value) { if (value == null) { return null; } final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE); if (source == null) { return null; } final Long timestamp = sourceInfo.getInt64(SourceInfo.TIMESTAMP_KEY); return timestamp == null ? null : Instant.ofEpochMilli(timestamp); }
@Override public Map<String, String> getEventSourcePosition(DataCollectionId source, OffsetContext offset, Object key, Struct value) { if (value == null) { return null; } final Struct sourceInfo = value.getStruct(Envelope.FieldName.SOURCE); if (source == null) { return null; } final Long scn = sourceInfo.getInt64(SourceInfo.SCN_KEY); return Collect.hashMapOf( SourceInfo.SCN_KEY, scn == null ? "null" : Long.toString(scn) ); }
jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName)); } else { jsonMap.put(fieldName, struct.getInt64(fieldName));
private void assertBigintUnsignedLong(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting int-64 since we have forced Long mode for BIGINT UNSIGNED assertThat(after.schema().field("c1").schema()).isEqualTo(Schema.INT64_SCHEMA); assertThat(after.schema().field("c2").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate the schema first, we are expecting int-64 since we are dealing with signed-bigint. //So Signed BIGINT would be an INT64 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate candidates values, note the loss in precision which is expected since BIGINT UNSIGNED cannot always be represented by //a long datatype. switch (i) { case 1: assertThat(after.getInt64("c1")).isEqualTo(-1L); assertThat(after.getInt64("c2")).isEqualTo(-1L); assertThat(after.getInt64("c3")).isEqualTo(9223372036854775807L); break; case 2: assertThat(after.getInt64("c1")).isEqualTo(-4000000000000000001L); assertThat(after.getInt64("c2")).isEqualTo(-4000000000000000001L); assertThat(after.getInt64("c3")).isEqualTo(-1223372036854775807L); break; case 3: assertThat(after.getInt64("c1")).isEqualTo(0L); assertThat(after.getInt64("c2")).isEqualTo(0L); assertThat(after.getInt64("c3")).isEqualTo(-9223372036854775808L); } }
private void assertBigintUnsignedPrecise(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting org.apache.kafka.connect.data.Decimal:Byte since we are dealing with unsignd-bigint //So Unsigned BIGINY would be an int32 type assertThat(after.schema().field("c1").schema()).isEqualTo(Decimal.builder(0).schema()); assertThat(after.schema().field("c2").schema()).isEqualTo(Decimal.builder(0).schema()); //Validate the schema first, we are expecting int-64 since we are dealing with signed-bigint. //So Signed BIGINT would be an INT64 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate candidates values switch (i) { case 1: assertThat(after.get("c1")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(9223372036854775807L); break; case 2: assertThat(after.get("c1")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(-1223372036854775807L); break; case 3: assertThat(after.get("c1")).isEqualTo(new BigDecimal("0")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("0")); assertThat(after.getInt64("c3")).isEqualTo(-9223372036854775808L); } }