/** * Obtain the operation for the given source record. * * @param record the source record; may not be null * @return the operation, or null if no valid operation was found in the record */ public static Operation operationFor(SourceRecord record) { Struct value = (Struct) record.value(); Field opField = value.schema().field(FieldName.OPERATION); if (opField != null) { return Operation.forCode(value.getString(opField.name())); } return null; } }
private void changeSourceToLastSnapshotRecord(SourceRecord currentRecord) { final Struct envelope = (Struct)currentRecord.value(); final Struct source = (Struct)envelope.get("source"); if (source.getBoolean(SourceInfo.LAST_SNAPSHOT_RECORD_KEY) != null) { source.put(SourceInfo.LAST_SNAPSHOT_RECORD_KEY, true); } }
@Test public void shouldGenerateRecordForInsertEvent() throws InterruptedException { CollectionId collectionId = new CollectionId("rs0", "dbA", "c1"); BsonTimestamp ts = new BsonTimestamp(1000, 1); ObjectId objId = new ObjectId(); Document obj = new Document().append("_id", objId).append("name", "Sally"); Document event = new Document().append("o", obj) .append("ns", "dbA.c1") .append("ts", ts) .append("h", Long.valueOf(12345678)) .append("op", "i"); RecordsForCollection records = recordMakers.forCollection(collectionId); records.recordEvent(event, 1002); assertThat(produced.size()).isEqualTo(1); SourceRecord record = produced.get(0); Struct key = (Struct) record.key(); Struct value = (Struct) record.value(); assertThat(key.schema()).isSameAs(record.keySchema()); assertThat(key.get("id")).isEqualTo("{ \"$oid\" : \"" + objId + "\"}"); assertThat(value.schema()).isSameAs(record.valueSchema()); // assertThat(value.getString(FieldName.BEFORE)).isNull(); assertThat(value.getString(FieldName.AFTER)).isEqualTo(obj.toJson(WRITER_SETTINGS)); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.CREATE.code()); assertThat(value.getInt64(FieldName.TIMESTAMP)).isEqualTo(1002L); Struct actualSource = value.getStruct(FieldName.SOURCE); Struct expectedSource = source.lastOffsetStruct("rs0", collectionId); assertThat(actualSource).isEqualTo(expectedSource); }
private void assertBigintUnsignedPrecise(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting org.apache.kafka.connect.data.Decimal:Byte since we are dealing with unsignd-bigint //So Unsigned BIGINY would be an int32 type assertThat(after.schema().field("c1").schema()).isEqualTo(Decimal.builder(0).schema()); assertThat(after.schema().field("c2").schema()).isEqualTo(Decimal.builder(0).schema()); //Validate the schema first, we are expecting int-64 since we are dealing with signed-bigint. //So Signed BIGINT would be an INT64 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT64_SCHEMA); //Validate candidates values switch (i) { case 1: assertThat(after.get("c1")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("18446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(9223372036854775807L); break; case 2: assertThat(after.get("c1")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("14446744073709551615")); assertThat(after.getInt64("c3")).isEqualTo(-1223372036854775807L); break; case 3: assertThat(after.get("c1")).isEqualTo(new BigDecimal("0")); assertThat(after.get("c2")).isEqualTo(new BigDecimal("0")); assertThat(after.getInt64("c3")).isEqualTo(-9223372036854775808L); } }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#READ READ} record. * * @param record the source record; may not be null */ public static void isValidRead(SourceRecord record) { assertThat(record.key()).isNotNull(); assertThat(record.keySchema()).isNotNull(); assertThat(record.valueSchema()).isNotNull(); Struct value = (Struct) record.value(); assertThat(value).isNotNull(); assertThat(value.getString(FieldName.OPERATION)).isEqualTo(Operation.READ.code()); assertThat(value.get(FieldName.AFTER)).isNotNull(); assertThat(value.get(FieldName.BEFORE)).isNull(); }
private void assertSmallUnsigned(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting int-32 since we are dealing with unsignd-smallint //So Unsigned SMALLINT would be an int32 type assertThat(after.schema().field("c1").schema()).isEqualTo(Schema.INT32_SCHEMA); assertThat(after.schema().field("c2").schema()).isEqualTo(Schema.INT32_SCHEMA); //Validate the schema first, we are expecting int-16 since we are dealing with signed-smallint. //So Signed SMALLINT would be an INT16 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT16_SCHEMA); //Validate candidates values switch (i) { case 1: assertThat(after.getInt32("c1")).isEqualTo(65535); assertThat(after.getInt32("c2")).isEqualTo(65535); assertThat(after.getInt16("c3")).isEqualTo((short)32767); break; case 2: assertThat(after.getInt32("c1")).isEqualTo(45535); assertThat(after.getInt32("c2")).isEqualTo(45535); assertThat(after.getInt16("c3")).isEqualTo((short)-12767); break; case 3: assertThat(after.getInt32("c1")).isEqualTo(0); assertThat(after.getInt32("c2")).isEqualTo(0); assertThat(after.getInt16("c3")).isEqualTo((short)-32768); } }
@Test public void shouldSetNullRecordToNull() { // When: final SchemaAndValue msg = ProcessingLogMessageFactory.deserializationErrorMsg( error, Optional.empty() ).get(); // Then: final Struct struct = (Struct) msg.value(); final Struct deserializationError = struct.getStruct(DESERIALIZATION_ERROR); assertThat(deserializationError.get(DESERIALIZATION_ERROR_FIELD_RECORD), is(nullValue())); }
private void assertPoint(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); Testing.debug(after); assertThat(i).isNotNull(); Double expectedX = after.getFloat64("expected_x"); Double expectedY = after.getFloat64("expected_y"); Integer expectedSrid = after.getInt32("expected_srid"); if (after.getStruct("point") != null) { Double actualX = after.getStruct("point").getFloat64("x"); Double actualY = after.getStruct("point").getFloat64("y"); Integer actualSrid = after.getStruct("point").getInt32("srid"); //Validate the values databaseDifferences.geometryAssertPoints(expectedX, expectedY, actualX, actualY); assertThat(actualSrid).isEqualTo(expectedSrid); //Test WKB Point point = (Point) WkbGeometryReader.readGeometry(new ByteReader((byte[]) after.getStruct("point") .get("wkb"))); databaseDifferences.geometryAssertPoints(expectedX, expectedY, point.getX(), point.getY()); } else if (expectedX != null) { Assert.fail("Got a null geometry but didn't expect to"); } }
@Override public byte[] serialize(final String topic, final GenericRow genericRow) { if (genericRow == null) { return null; } final Struct struct = translator.toConnectRow(genericRow); try { return converter.fromConnectData(topic, struct.schema(), struct); } catch (final Exception e) { throw new SerializationException( "Error serializing row to topic " + topic + " using Converter API", e); } }
private Struct offsetStructFor(String replicaSetName, String namespace, Position position, boolean isInitialSync) { if (position == null) position = INITIAL_POSITION; Struct result = super.struct(); result.put(SERVER_NAME, serverName); result.put(REPLICA_SET_NAME, replicaSetName); result.put(NAMESPACE, namespace); result.put(TIMESTAMP, position.getTime()); result.put(ORDER, position.getInc()); result.put(OPERATION_ID, position.getOperationId()); if (isInitialSync) { result.put(INITIAL_SYNC, true); } return result; }
private void assertMediumUnsigned(Struct value) { Struct after = value.getStruct(Envelope.FieldName.AFTER); Integer i = after.getInt32("id"); assertThat(i).isNotNull(); //Validate the schema first, we are expecting int-32 since we are dealing with unsignd-mediumint //So Unsigned MEDIUMINT would be an int32 type assertThat(after.schema().field("c1").schema()).isEqualTo(Schema.INT32_SCHEMA); assertThat(after.schema().field("c2").schema()).isEqualTo(Schema.INT32_SCHEMA); //Validate the schema first, we are expecting int-32 since we are dealing with signed-mediumint. //So Signed MEDIUMINT would be an INT32 type assertThat(after.schema().field("c3").schema()).isEqualTo(Schema.INT32_SCHEMA); //Validate candidates values switch (i) { case 1: assertThat(after.getInt32("c1")).isEqualTo(16777215); assertThat(after.getInt32("c2")).isEqualTo(16777215); assertThat(after.getInt32("c3")).isEqualTo(8388607); break; case 2: assertThat(after.getInt32("c1")).isEqualTo(10777215); assertThat(after.getInt32("c2")).isEqualTo(10777215); assertThat(after.getInt32("c3")).isEqualTo(-6388607); break; case 3: assertThat(after.getInt32("c1")).isEqualTo(0); assertThat(after.getInt32("c2")).isEqualTo(0); assertThat(after.getInt32("c3")).isEqualTo(-8388608); } }
@Test public void shouldTranslateNullValueCorrectly() { final Schema rowSchema = SchemaBuilder.struct() .field("INT", SchemaBuilder.OPTIONAL_INT32_SCHEMA) .optional() .build(); final Struct connectStruct = new Struct(rowSchema); final ConnectDataTranslator connectToKsqlTranslator = new ConnectDataTranslator(rowSchema); final GenericRow row = connectToKsqlTranslator.toKsqlRow(rowSchema, connectStruct); assertThat(row.getColumns().size(), equalTo(1)); assertThat(row.getColumnValue(0), is(nullValue())); }
protected void verifyOperation(SourceRecord record, Operation expected) { Struct value = (Struct) record.value(); assertThat(value.getString(Envelope.FieldName.OPERATION)).isEqualTo(expected.code()); }