@Override public Schema getLatestSchemaByTopic(String topic) throws SchemaRegistryException { String schemaName = topic + this.schemaNameSuffix; try { return new Schema.Parser().parse(this.schemaRegistryClient.getLatestSchemaMetadata(schemaName).getSchema()); } catch (IOException | RestClientException e) { log.error("Failed to get schema for topic " + topic + "; subject " + schemaName); throw new SchemaRegistryException(e); } }
@Override public String getFingerprint(Schema schema) { final String subject = schema.getName(); final int guid; try { guid = theClient.register(subject, schema); } catch (IOException | RestClientException e) { throw new RuntimeException(e); } return Integer.toString(guid); }
@Override public Schema getSchema(String fingerPrint) { final Schema theSchema; try { theSchema = theClient.getByID(Integer.parseInt(fingerPrint)); } catch (IOException | RestClientException e) { throw new RuntimeException(e); } return theSchema; } }
@Override public SchemaReference registerSchema(String subject, String schema) throws SchemaManagerException { SchemaReference schemaReference; try { org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(schema); int id = schemaRegistryClient.register(subject, avroSchema); int version = schemaRegistryClient.getLatestSchemaMetadata(subject).getVersion(); schemaReference = new SchemaReference(subject, id, version); } catch (IOException | RestClientException | RuntimeException e) { log.error("caught an exception while registering a new schema for subject '{}'", subject); throw new SchemaManagerException(e); } return schemaReference; }
@Test public void shouldMatchAvroFormatter() throws Exception { // Given: final Schema schema = parseAvroSchema( "{\n" + " \"fields\": [\n" + " { \"name\": \"str1\", \"type\": \"string\" }\n" + " ],\n" + " \"name\": \"myrecord\",\n" + " \"type\": \"record\"\n" + "}"); final GenericData.Record avroRecord = new GenericData.Record(schema); avroRecord.put("str1", "My first string"); expect(schemaRegistryClient.register(anyString(), anyObject())).andReturn(1); expect(schemaRegistryClient.getById(anyInt())).andReturn(schema).times(2); replay(schemaRegistryClient); final byte[] avroData = serializeAvroRecord(avroRecord); // When: final Result result = getFormatter(avroData); // Then: assertThat(result.format, is(Format.AVRO)); assertThat(result.formatted, endsWith(", key, {\"str1\": \"My first string\"}\n")); }
Schema schema = schemaRegistry.getById(id); if (schema == null) throw new IllegalStateException("Unknown schema id: " + id); SchemaMetadata metadata = schemaRegistry.getLatestSchemaMetadata(subject); latestSchema = new Schema.Parser().parse(metadata.getSchema()); latestVersion = metadata.getVersion(); int version = schemaRegistry.getVersion(subject, schema); if (version > latestVersion) { String errorMsg = String.format(
@Override public Schema readSchema(InputStream in) throws IOException { DataInputStream dataInputStream = new DataInputStream(in); if (dataInputStream.readByte() != 0) { throw new IOException("Unknown data format. Magic number does not match"); } else { int schemaId = dataInputStream.readInt(); try { return schemaRegistryClient.getById(schemaId); } catch (RestClientException e) { throw new IOException(format("Could not find schema with id %s in registry", schemaId), e); } } }
@Override public boolean checkCompatibility(String subject, String schema) throws SchemaException { try { org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(schema); return !schemaRegistryClient.getAllSubjects().contains(subject) || schemaRegistryClient.testCompatibility(subject, avroSchema); } catch (IOException | RestClientException e) { String message = String.format("caught an exception while checking compatibility for subject '%s'", subject); log.error(message); throw new SchemaException(message, e); } }
Schema writeSchema; try { writeSchema = client.getById(id); } catch (IOException e) { throw new StreamRuntimeException("Cannot retrieve write schema id: " + id + " on " + messageClass, e);
@Override public Integer register(Schema schema, String name) throws SchemaRegistryException { try { String schemaName = name + this.schemaNameSuffix; return this.schemaRegistryClient.register(schemaName, schema); } catch (IOException | RestClientException e) { throw new SchemaRegistryException(e); } } }
@Override public Schema getLatestSchemaByTopic(String topic) throws SchemaRegistryException { String schemaName = topic + this.schemaNameSuffix; try { return new Schema.Parser().parse(this.schemaRegistryClient.getLatestSchemaMetadata(schemaName).getSchema()); } catch (IOException | RestClientException e) { log.error("Failed to get schema for topic " + topic + "; subject " + schemaName); throw new SchemaRegistryException(e); } }
@Override public GenericRecord parse(ByteBuffer bytes) { try { bytes.get(); // ignore first \0 byte int id = bytes.getInt(); // extract schema registry id int length = bytes.limit() - 1 - 4; int offset = bytes.position() + bytes.arrayOffset(); Schema schema = registry.getByID(id); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); return reader.read(null, DecoderFactory.get().binaryDecoder(bytes.array(), offset, length, null)); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } }
private static void registerSchema(final SchemaRegistryClient schemaRegistryClient) throws IOException, RestClientException { final String ordersAveroSchemaStr = "{" + "\"namespace\": \"kql\"," + " \"name\": \"orders\"," + " \"type\": \"record\"," + " \"fields\": [" + " {\"name\": \"ordertime\", \"type\": \"long\"}," + " {\"name\": \"orderid\", \"type\": \"long\"}," + " {\"name\": \"itemid\", \"type\": \"string\"}," + " {\"name\": \"orderunits\", \"type\": \"double\"}," + " {\"name\": \"arraycol\", \"type\": {\"type\": \"array\", \"items\": \"double\"}}," + " {\"name\": \"mapcol\", \"type\": {\"type\": \"map\", \"values\": \"double\"}}" + " ]" + "}"; final org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser(); final org.apache.avro.Schema avroSchema = parser.parse(ordersAveroSchemaStr); schemaRegistryClient.register("orders-topic" + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX, avroSchema); }
@Override public Schema retrieveSchema(TableId table, String topic) { String subject = getSubject(topic); try { logger.debug("Retrieving schema information for topic {} with subject {}", topic, subject); SchemaMetadata latestSchemaMetadata = schemaRegistryClient.getLatestSchemaMetadata(subject); org.apache.avro.Schema avroSchema = new Parser().parse(latestSchemaMetadata.getSchema()); return avroData.toConnectSchema(avroSchema); } catch (IOException | RestClientException exception) { throw new ConnectException(String.format( "Exception while fetching latest schema metadata for topic=%s, subject=%s", topic, subject), exception ); } }
@Override protected Schema fetchSchemaByKey(Integer key) throws SchemaRegistryException { try { return this.schemaRegistryClient.getByID(key); } catch (IOException | RestClientException e) { throw new SchemaRegistryException(e); } }
@SuppressWarnings("SameParameterValue") private void givenAvroSchemaNotEvolveable(final String topicName) { final org.apache.avro.Schema schema = org.apache.avro.Schema.create(Type.INT); try { schemaRegistryClient.register(topicName + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX, schema); } catch (final Exception e) { fail(e.getMessage()); } }
private void downloadSchema(final String url, final String subject) { try { final SchemaMetadata metadata = RegistryClientSingleton .getInstance().client(url).getLatestSchemaMetadata(subject); writeSchemas(subject, metadata.getSchema()); } catch (IOException | RestClientException e) { e.printStackTrace(); } }
@Test public void testConfluentAvroDeserializer() throws IOException, RestClientException { WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L,10L); mockWorkUnitState.setProp("schema.registry.url", TEST_URL); Schema schema = SchemaBuilder.record(TEST_RECORD_NAME) .namespace(TEST_NAMESPACE).fields() .name(TEST_FIELD_NAME).type().stringType().noDefault() .endRecord(); GenericRecord testGenericRecord = new GenericRecordBuilder(schema).set(TEST_FIELD_NAME, "testValue").build(); SchemaRegistryClient mockSchemaRegistryClient = mock(SchemaRegistryClient.class); when(mockSchemaRegistryClient.getByID(any(Integer.class))).thenReturn(schema); Serializer<Object> kafkaEncoder = new KafkaAvroSerializer(mockSchemaRegistryClient); Deserializer<Object> kafkaDecoder = new KafkaAvroDeserializer(mockSchemaRegistryClient); ByteBuffer testGenericRecordByteBuffer = ByteBuffer.wrap(kafkaEncoder.serialize(TEST_TOPIC_NAME, testGenericRecord)); KafkaSchemaRegistry<Integer, Schema> mockKafkaSchemaRegistry = mock(KafkaSchemaRegistry.class); KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState, Optional.fromNullable(Deserializers.CONFLUENT_AVRO), kafkaDecoder, mockKafkaSchemaRegistry); ByteArrayBasedKafkaRecord mockMessageAndOffset = getMockMessageAndOffset(testGenericRecordByteBuffer); Assert.assertEquals(kafkaDecoderExtractor.decodeRecord(mockMessageAndOffset), testGenericRecord); }
@Override public Integer register(Schema schema, String name) throws SchemaRegistryException { try { String schemaName = name + this.schemaNameSuffix; return this.schemaRegistryClient.register(schemaName, schema); } catch (IOException | RestClientException e) { throw new SchemaRegistryException(e); } } }
@Test public void testRetrieveSchema() throws Exception { final TableId table = TableId.of("test", "kafka_topic"); final String testTopic = "kafka-topic"; final String testSubject = "kafka-topic-value"; final String testAvroSchemaString = "{\"type\": \"record\", " + "\"name\": \"testrecord\", " + "\"fields\": [{\"name\": \"f1\", \"type\": \"string\"}]}"; final SchemaMetadata testSchemaMetadata = new SchemaMetadata(1, 1, testAvroSchemaString); SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class); when(schemaRegistryClient.getLatestSchemaMetadata(testSubject)).thenReturn(testSchemaMetadata); SchemaRegistrySchemaRetriever testSchemaRetriever = new SchemaRegistrySchemaRetriever( schemaRegistryClient, new AvroData(0) ); Schema expectedKafkaConnectSchema = SchemaBuilder.struct().field("f1", Schema.STRING_SCHEMA).name("testrecord").build(); assertEquals(expectedKafkaConnectSchema, testSchemaRetriever.retrieveSchema(table, testTopic)); } }