protected Schema fetchLatestPayloadSchema() throws Exception { Schema latestPayloadSchema = (Schema)registry.getLatestSchemaByTopic(payloadSchemaTopic); latestPayloadReader = new GenericDatumReader<>(latestPayloadSchema); return latestPayloadSchema; }
/** * Get the schema (metadata) of the extracted data records. * * @return the Kafka topic being extracted * @throws IOException if there is problem getting the schema */ @Override public String getSchema() throws IOException { try { return this.kafkaSchemaRegistry.getLatestSchemaByTopic(this.topicName); } catch (SchemaRegistryException e) { throw new RuntimeException(e); } } }
protected Schema getLatestSchemaByTopic(String topic) { Preconditions.checkState(this.schemaRegistry.isPresent()); try { return this.schemaRegistry.get().getLatestSchemaByTopic(topic); } catch (SchemaRegistryException e) { log.error(String.format("Cannot find latest schema for topic %s. This topic will be skipped", topic), e); return null; } }
@Override public Object getSchema() { try { LOG.info("Getting schema for {}. Gap: {} HighWaterMark: {}", this.topicName, this.lowWatermark.getGap(this.highWatermark)); //If HighWatermark equals LowWatermark that might mean the workunit is an empty workunit if (this.lowWatermark.getGap(this.highWatermark) == 0) { LOG.info("Not getting schema for {} as the gap between high and low watermark is 0", this.topicName); return null; } return this.kafkaSchemaRegistry.getLatestSchemaByTopic(this.topicName); } catch (SchemaRegistryException e) { throw new RuntimeException(e); } }
/** * Get the schema (metadata) of the extracted data records. * * @return the schema of Kafka topic being extracted * @throws IOException if there is problem getting the schema */ @Override public S getSchema() throws IOException { try { if (_schemaRegistry.isPresent()) { return _schemaRegistry.get().getLatestSchemaByTopic(this._partition.topic()); } } catch (SchemaRegistryException e) { throw new RuntimeException(e); } return ((S) this._partition.topic()); }
@Test public void testConfluentShouldNotQuerySchemaRegistryWhenTheGapIsZero() throws IOException, RestClientException, SchemaRegistryException { WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L, 0L); mockWorkUnitState.setProp("schema.registry.url", TEST_URL); SchemaRegistryClient mockSchemaRegistryClient = mock(SchemaRegistryClient.class); Deserializer<Object> kafkaDecoder = new KafkaAvroDeserializer(mockSchemaRegistryClient); KafkaSchemaRegistry<Integer, Schema> mockKafkaSchemaRegistry = mock(KafkaSchemaRegistry.class); KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState, Optional.fromNullable(Deserializers.CONFLUENT_AVRO), kafkaDecoder, mockKafkaSchemaRegistry); verify(mockKafkaSchemaRegistry, never()).getLatestSchemaByTopic(any()); kafkaDecoderExtractor.getSchema(); }
when(mockRegistry.getLatestSchemaByTopic(any())).thenReturn(latestPayloadSchema); when(mockRegistry.getSchemaByKey(any())).thenReturn(inputSchema.getField("nestedRecord").schema());
when(mockRegistry.getLatestSchemaByTopic(any())).thenReturn(latestPayloadSchema); when(mockRegistry.getSchemaByKey(any())).thenReturn(inputSchema.getField("nestedRecord").schema());
@Test public void testConfluentAvroDeserializerForSchemaEvolution() throws IOException, RestClientException, SchemaRegistryException { WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L, 10L); mockWorkUnitState.setProp("schema.registry.url", TEST_URL); Schema schemaV1 = SchemaBuilder.record(TEST_RECORD_NAME) .namespace(TEST_NAMESPACE).fields() .name(TEST_FIELD_NAME).type().stringType().noDefault() .endRecord(); Schema schemaV2 = SchemaBuilder.record(TEST_RECORD_NAME) .namespace(TEST_NAMESPACE).fields() .name(TEST_FIELD_NAME).type().stringType().noDefault() .optionalString(TEST_FIELD_NAME2).endRecord(); GenericRecord testGenericRecord = new GenericRecordBuilder(schemaV1).set(TEST_FIELD_NAME, "testValue").build(); SchemaRegistryClient mockSchemaRegistryClient = mock(SchemaRegistryClient.class); when(mockSchemaRegistryClient.getByID(any(Integer.class))).thenReturn(schemaV1); Serializer<Object> kafkaEncoder = new KafkaAvroSerializer(mockSchemaRegistryClient); Deserializer<Object> kafkaDecoder = new KafkaAvroDeserializer(mockSchemaRegistryClient); ByteBuffer testGenericRecordByteBuffer = ByteBuffer.wrap(kafkaEncoder.serialize(TEST_TOPIC_NAME, testGenericRecord)); KafkaSchemaRegistry<Integer, Schema> mockKafkaSchemaRegistry = mock(KafkaSchemaRegistry.class); when(mockKafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)).thenReturn(schemaV2); KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState, Optional.fromNullable(Deserializers.CONFLUENT_AVRO), kafkaDecoder, mockKafkaSchemaRegistry); when(kafkaDecoderExtractor.getSchema()).thenReturn(schemaV2); ByteArrayBasedKafkaRecord mockMessageAndOffset = getMockMessageAndOffset(testGenericRecordByteBuffer); GenericRecord received = (GenericRecord) kafkaDecoderExtractor.decodeRecord(mockMessageAndOffset); Assert.assertEquals(received.toString(), "{\"testField\": \"testValue\", \"testField2\": null}"); }
private void doTestRegisterAndGetLatest(Properties properties) throws SchemaRegistryException { SchemaRegistryClient schemaRegistryClient = new MockSchemaRegistryClient(); KafkaSchemaRegistry<Integer, Schema> kafkaSchemaRegistry = new ConfluentKafkaSchemaRegistry(properties, schemaRegistryClient); Schema schema1 = SchemaBuilder.record(TEST_RECORD_NAME + "1").namespace(TEST_NAMESPACE).fields().name(TEST_FIELD_NAME).type() .stringType().noDefault().endRecord(); Schema schema2 = SchemaBuilder.record(TEST_RECORD_NAME + "2").namespace(TEST_NAMESPACE).fields().name(TEST_FIELD_NAME).type() .stringType().noDefault().endRecord(); kafkaSchemaRegistry.register(schema1, TEST_TOPIC_NAME); kafkaSchemaRegistry.register(schema2, TEST_TOPIC_NAME); Assert.assertNotEquals(schema1, kafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)); Assert.assertEquals(schema2, kafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)); } }
protected Schema fetchLatestPayloadSchema() throws Exception { Schema latestPayloadSchema = (Schema)registry.getLatestSchemaByTopic(payloadSchemaTopic); latestPayloadReader = new GenericDatumReader<>(latestPayloadSchema); return latestPayloadSchema; }
/** * Get the schema (metadata) of the extracted data records. * * @return the Kafka topic being extracted * @throws IOException if there is problem getting the schema */ @Override public String getSchema() throws IOException { try { return this.kafkaSchemaRegistry.getLatestSchemaByTopic(this.topicName); } catch (SchemaRegistryException e) { throw new RuntimeException(e); } } }
protected Schema getLatestSchemaByTopic(String topic) { Preconditions.checkState(this.schemaRegistry.isPresent()); try { return this.schemaRegistry.get().getLatestSchemaByTopic(topic); } catch (SchemaRegistryException e) { log.error(String.format("Cannot find latest schema for topic %s. This topic will be skipped", topic), e); return null; } }
@Override public Object getSchema() { try { LOG.info("Getting schema for {}. Gap: {} HighWaterMark: {}", this.topicName, this.lowWatermark.getGap(this.highWatermark)); //If HighWatermark equals LowWatermark that might mean the workunit is an empty workunit if (this.lowWatermark.getGap(this.highWatermark) == 0) { LOG.info("Not getting schema for {} as the gap between high and low watermark is 0", this.topicName); return null; } return this.kafkaSchemaRegistry.getLatestSchemaByTopic(this.topicName); } catch (SchemaRegistryException e) { throw new RuntimeException(e); } }