public KafkaAvroSerde() { ser = new KafkaAvroSerializer(); deser = new KafkaAvroDeserializer(); }
@Override public byte[] serialize(String topic, Object data) { return avroSnapshotSerializer.serialize(data, createSchemaMetadata(topic)); }
private void _testKafkaSerDes(String topic, boolean isKey, Object payload) { KafkaAvroSerializer avroSerializer = new KafkaAvroSerializer(); Map<String,Object> schemaRegistryClientConf = SCHEMA_REGISTRY_TEST_SERVER_CLIENT_WRAPPER.exportClientConf(true); avroSerializer.configure(schemaRegistryClientConf, isKey); KafkaAvroDeserializer avroDeserializer = new KafkaAvroDeserializer(); avroDeserializer.configure(schemaRegistryClientConf, isKey); byte[] serializedData = avroSerializer.serialize(topic, payload); Object deserializedObj = avroDeserializer.deserialize(topic, serializedData); if (payload instanceof byte[]) { Assert.assertArrayEquals((byte[]) payload, (byte[]) deserializedObj); } else { AvroSchemaRegistryClientUtil.assertAvroObjs(payload, deserializedObj); } }
LOG.info(confluentResult.toString()); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(); kafkaAvroDeserializer.configure(registryConfig, false); GenericRecord registryResult = (GenericRecord) kafkaAvroDeserializer.deserialize("topic", bytes);
@Override public byte[] serialize(String topic, Headers headers, Object data) { if (useRecordHeader) { final MessageAndMetadata context = messageAndMetadataAvroSerializer.serialize(data, createSchemaMetadata(topic)); headers.add(isKey ? keySchemaVersionIdHeaderName : valueSchemaVersionIdHeaderName, context.metadata()); return context.payload(); } else { return serialize(topic, data); } }
@Test public void testToggleStoringSchemaInHeader() { TestRecord record = new TestRecord(); record.setField1("Hello"); record.setField2("World"); String keySchemaHeaderName = KafkaAvroSerde.DEFAULT_KEY_SCHEMA_VERSION_ID; for (Boolean storeScheamIdInHeader : Arrays.asList(true, false)) { Map<String, Object> configs = new HashMap<>(); configs.put(KafkaAvroSerializer.STORE_SCHEMA_VERSION_ID_IN_HEADER, storeScheamIdInHeader.toString()); configs.put(AbstractAvroSnapshotDeserializer.SPECIFIC_AVRO_READER, true); KafkaAvroSerde serde = new KafkaAvroSerde(schemaRegistryClient); final ExtendedSerializer<Object> serializer = serde.extendedSerializer(); serializer.configure(configs, true); Headers headers = new RecordHeaders(); final byte[] bytes = serializer.serialize(topic, headers, record); Assert.assertEquals(storeScheamIdInHeader, headers.lastHeader(keySchemaHeaderName) != null); final ExtendedDeserializer<Object> deserializer = serde.extendedDeserializer(); deserializer.configure(configs, true); final TestRecord actual = (TestRecord) deserializer.deserialize(topic, headers, bytes); Assert.assertEquals(record, actual); } } }
@Override public Object deserialize(String topic, Headers headers, byte[] data) { if (headers != null) { final Header header = headers.lastHeader(isKey ? keySchemaVersionIdHeaderName : valueSchemaVersionIdHeaderName); if (header != null) { return messageAndMetadataAvroDeserializer.deserialize(new MessageAndMetadata(header.value(), data), readerVersions.get(topic)); } } return deserialize(topic, data); }
@Test public void testDefaultSchemaHeaderNames() { testSchemaHeaderNames(KafkaAvroSerde.DEFAULT_KEY_SCHEMA_VERSION_ID, KafkaAvroSerde.DEFAULT_VALUE_SCHEMA_VERSION_ID); }
private SchemaMetadata createSchemaMetadata(String topic) { SchemaMetadata schemaMetadata = getSchemaKey(topic, isKey); String description = "Schema registered by KafkaAvroSerializer for topic: [" + topic + "] iskey: [" + isKey + "]"; return new SchemaMetadata.Builder(schemaMetadata).description(description).compatibility(compatibility).build(); }
@Test public void testSpecificSerializedGenericDeserialized() { Map<String, Object> config = new HashMap<>(); config.put(AvroSnapshotDeserializer.SPECIFIC_AVRO_READER, false); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient); kafkaAvroDeserializer.configure(config, false); KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient); kafkaAvroSerializer.configure(config, false); TestRecord record = new TestRecord(); record.setField1("some value"); record.setField1("some other value"); byte[] bytes = kafkaAvroSerializer.serialize(topic, record); Object o = kafkaAvroDeserializer.deserialize(topic, bytes); checkSpecificSerializedGenericDeserializedEquals(record, o); Headers headers = new RecordHeaders(); bytes = kafkaAvroSerializer.serialize(topic, headers, record); o = kafkaAvroDeserializer.deserialize(topic, headers, bytes); checkSpecificSerializedGenericDeserializedEquals(record, o); }
public KafkaAvroSerde(ISchemaRegistryClient client) { ser = new KafkaAvroSerializer(client); deser = new KafkaAvroDeserializer(client); }
private void testSchemaHeaderNames(String customKeySchemaHeaderName, String customValueSchemaHeaderName) { TestRecord record = new TestRecord(); record.setField1("Hello"); record.setField2("World"); Map<String, Object> configs = new HashMap<>(); configs.put(KafkaAvroSerde.KEY_SCHEMA_VERSION_ID_HEADER_NAME, customKeySchemaHeaderName); configs.put(KafkaAvroSerde.VALUE_SCHEMA_VERSION_ID_HEADER_NAME, customValueSchemaHeaderName); configs.put(KafkaAvroSerializer.STORE_SCHEMA_VERSION_ID_IN_HEADER, "true"); configs.put(AbstractAvroSnapshotDeserializer.SPECIFIC_AVRO_READER, true); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); AvroSerDesHandler handler = new DefaultAvroSerDesHandler(); handler.handlePayloadSerialization(outputStream, record); for (Boolean isKey : Arrays.asList(true, false)) { KafkaAvroSerde serde = new KafkaAvroSerde(schemaRegistryClient); final ExtendedSerializer<Object> serializer = serde.extendedSerializer(); serializer.configure(configs, isKey); Headers headers = new RecordHeaders(); final byte[] bytes = serializer.serialize(topic, headers, record); Assert.assertArrayEquals(outputStream.toByteArray(), bytes); Assert.assertEquals(isKey, headers.lastHeader(customKeySchemaHeaderName) != null); Assert.assertEquals(!isKey, headers.lastHeader(customValueSchemaHeaderName) != null); final ExtendedDeserializer<Object> deserializer = serde.extendedDeserializer(); deserializer.configure(configs, isKey); final TestRecord actual = (TestRecord) deserializer.deserialize(topic, headers, bytes); Assert.assertEquals(record, actual); } }
@Test public void testCustomSchemaHeaderNames() { testSchemaHeaderNames("ksid", "vsid"); }
@Test public void testSpecificSerializedSpecificDeserialized() { Map<String, Object> config = new HashMap<>(); config.put(AvroSnapshotDeserializer.SPECIFIC_AVRO_READER, true); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient); kafkaAvroDeserializer.configure(config, false); KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient); kafkaAvroSerializer.configure(Collections.emptyMap(), false); TestRecord record = new TestRecord(); record.setField1("some value"); record.setField1("some other value"); byte[] bytes = kafkaAvroSerializer.serialize(topic, record); Object o = kafkaAvroDeserializer.deserialize(topic, bytes); checkSpecificSerializedSpecificDeserializedEquals(record, o); Headers headers = new RecordHeaders(); bytes = kafkaAvroSerializer.serialize(topic, headers, record); o = kafkaAvroDeserializer.deserialize(topic, headers, bytes); checkSpecificSerializedSpecificDeserializedEquals(record, o); }
@Test public void testGenericSerializedSpecificDeserialized() { Map<String, Object> config = new HashMap<>(); config.put(AvroSnapshotDeserializer.SPECIFIC_AVRO_READER, true); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient); kafkaAvroDeserializer.configure(config, false); KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient); kafkaAvroSerializer.configure(Collections.emptyMap(), false); GenericRecord record = new GenericRecordBuilder(schema).set("field1", "some value").set("field2", "some other value").build(); byte[] payload = kafkaAvroSerializer.serialize(topic, record); Object o = kafkaAvroDeserializer.deserialize(topic, payload); checkGenericSerializedSpecificDeserializedEquals(record, o); Headers headers = new RecordHeaders(); payload = kafkaAvroSerializer.serialize(topic, headers, record); o = kafkaAvroDeserializer.deserialize(topic, headers, payload); checkGenericSerializedSpecificDeserializedEquals(record, o); }
@Test public void testGenericSerializedGenericDeserialized() { String topic = "topic"; Map<String, Object> config = new HashMap<>(); config.put(AvroSnapshotDeserializer.SPECIFIC_AVRO_READER, false); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient); kafkaAvroDeserializer.configure(config, false); KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient); kafkaAvroSerializer.configure(config, false); GenericRecord record = new GenericRecordBuilder(schema).set("field1", "some value").set("field2", "some other value").build(); byte[] bytes = kafkaAvroSerializer.serialize(topic , record); Object o = kafkaAvroDeserializer.deserialize(topic, bytes); checkGenericSerializedGenericDeserializedEquals(record, o); Headers headers = new RecordHeaders(); bytes = kafkaAvroSerializer.serialize(topic, headers, record); o = kafkaAvroDeserializer.deserialize(topic, headers, bytes); checkGenericSerializedGenericDeserializedEquals(record, o); }
KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(); kafkaAvroSerializer.configure(registryConfig, false); byte[] bytes = kafkaAvroSerializer.serialize("topic", record); KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(); kafkaAvroDeserializer.configure(registryConfig, false); GenericRecord registryResult = (GenericRecord) kafkaAvroDeserializer.deserialize("topic", bytes); LOG.info(registryResult.toString());