/** * A constructor for use by test cases ONLY, thus the default scope. * @param url The complete URL reference of a confluent schema registry, e.g. "http://HOST:PORT" */ ConfluentAvroSerializer(String url) { this.url = url; this.theClient = new CachedSchemaRegistryClient(this.url, 10000); }
@Override public Schema getSchema(String fingerPrint) { final Schema theSchema; try { theSchema = theClient.getByID(Integer.parseInt(fingerPrint)); } catch (IOException | RestClientException e) { throw new RuntimeException(e); } return theSchema; } }
@Override public Schema getLatestSchemaByTopic(String topic) throws SchemaRegistryException { String schemaName = topic + this.schemaNameSuffix; try { return new Schema.Parser().parse(this.schemaRegistryClient.getLatestSchemaMetadata(schemaName).getSchema()); } catch (IOException | RestClientException e) { log.error("Failed to get schema for topic " + topic + "; subject " + schemaName); throw new SchemaRegistryException(e); } }
@Override public Integer register(Schema schema, String name) throws SchemaRegistryException { try { String schemaName = name + this.schemaNameSuffix; return this.schemaRegistryClient.register(schemaName, schema); } catch (IOException | RestClientException e) { throw new SchemaRegistryException(e); } } }
@Test public void shouldMatchAvroFormatter() throws Exception { // Given: final Schema schema = parseAvroSchema( "{\n" + " \"fields\": [\n" + " { \"name\": \"str1\", \"type\": \"string\" }\n" + " ],\n" + " \"name\": \"myrecord\",\n" + " \"type\": \"record\"\n" + "}"); final GenericData.Record avroRecord = new GenericData.Record(schema); avroRecord.put("str1", "My first string"); expect(schemaRegistryClient.register(anyString(), anyObject())).andReturn(1); expect(schemaRegistryClient.getById(anyInt())).andReturn(schema).times(2); replay(schemaRegistryClient); final byte[] avroData = serializeAvroRecord(avroRecord); // When: final Result result = getFormatter(avroData); // Then: assertThat(result.format, is(Format.AVRO)); assertThat(result.formatted, endsWith(", key, {\"str1\": \"My first string\"}\n")); }
private void initKafka() { schemaRegistryClient = new MockSchemaRegistryClient(); kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient); Properties defaultConfig = new Properties(); defaultConfig.put(KafkaAvroDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "bogus"); avroSerializer = new KafkaAvroSerializer(schemaRegistryClient); }
@Override public Schema readSchema(InputStream in) throws IOException { DataInputStream dataInputStream = new DataInputStream(in); if (dataInputStream.readByte() != 0) { throw new IOException("Unknown data format. Magic number does not match"); } else { int schemaId = dataInputStream.readInt(); try { return schemaRegistryClient.getById(schemaId); } catch (RestClientException e) { throw new IOException(format("Could not find schema with id %s in registry", schemaId), e); } } }
/** * A constructor with a signature that Storm can locate and use with kryo registration. * See Storm's SerializationFactory class for details * * @param k Unused but needs to be present for Serialization Factory to find this constructor * @param topoConf The global storm configuration. Must define "avro.schemaregistry.confluent" to locate the * confluent schema registry. Should in the form of "http://HOST:PORT" */ public ConfluentAvroSerializer(Kryo k, Map<String, Object> topoConf) { url = (String) topoConf.get("avro.schemaregistry.confluent"); this.theClient = new CachedSchemaRegistryClient(this.url, 10000); }
@Override public String getFingerprint(Schema schema) { final String subject = schema.getName(); final int guid; try { guid = theClient.register(subject, schema); } catch (IOException | RestClientException e) { throw new RuntimeException(e); } return Integer.toString(guid); }
@Override protected Schema fetchSchemaByKey(Integer key) throws SchemaRegistryException { try { return this.schemaRegistryClient.getByID(key); } catch (IOException | RestClientException e) { throw new SchemaRegistryException(e); } }
@Test public void testRegisterAndGetByKey() throws SchemaRegistryException { Properties properties = new Properties(); properties.setProperty(KafkaSchemaRegistry.KAFKA_SCHEMA_REGISTRY_URL, TEST_URL); SchemaRegistryClient schemaRegistryClient = new MockSchemaRegistryClient(); KafkaSchemaRegistry<Integer, Schema> kafkaSchemaRegistry = new ConfluentKafkaSchemaRegistry(properties, schemaRegistryClient); Schema schema = SchemaBuilder.record(TEST_RECORD_NAME).namespace(TEST_NAMESPACE).fields().name(TEST_FIELD_NAME).type() .stringType().noDefault().endRecord(); Integer id = kafkaSchemaRegistry.register(schema); Assert.assertEquals(schema, kafkaSchemaRegistry.getSchemaByKey(id)); }
@Override public SchemaCoder get() { return new ConfluentSchemaRegistryCoder(new CachedSchemaRegistryClient( url, identityMapCapacity)); } }
private static void registerSchema(final SchemaRegistryClient schemaRegistryClient) throws IOException, RestClientException { final String ordersAveroSchemaStr = "{" + "\"namespace\": \"kql\"," + " \"name\": \"orders\"," + " \"type\": \"record\"," + " \"fields\": [" + " {\"name\": \"ordertime\", \"type\": \"long\"}," + " {\"name\": \"orderid\", \"type\": \"long\"}," + " {\"name\": \"itemid\", \"type\": \"string\"}," + " {\"name\": \"orderunits\", \"type\": \"double\"}," + " {\"name\": \"arraycol\", \"type\": {\"type\": \"array\", \"items\": \"double\"}}," + " {\"name\": \"mapcol\", \"type\": {\"type\": \"map\", \"values\": \"double\"}}" + " ]" + "}"; final org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser(); final org.apache.avro.Schema avroSchema = parser.parse(ordersAveroSchemaStr); schemaRegistryClient.register("orders-topic" + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX, avroSchema); }
@Override public GenericRecord parse(ByteBuffer bytes) { try { bytes.get(); // ignore first \0 byte int id = bytes.getInt(); // extract schema registry id int length = bytes.limit() - 1 - 4; int offset = bytes.position() + bytes.arrayOffset(); Schema schema = registry.getByID(id); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); return reader.read(null, DecoderFactory.get().binaryDecoder(bytes.array(), offset, length, null)); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } }
private void doTestRegisterAndGetLatest(Properties properties) throws SchemaRegistryException { SchemaRegistryClient schemaRegistryClient = new MockSchemaRegistryClient(); KafkaSchemaRegistry<Integer, Schema> kafkaSchemaRegistry = new ConfluentKafkaSchemaRegistry(properties, schemaRegistryClient); Schema schema1 = SchemaBuilder.record(TEST_RECORD_NAME + "1").namespace(TEST_NAMESPACE).fields().name(TEST_FIELD_NAME).type() .stringType().noDefault().endRecord(); Schema schema2 = SchemaBuilder.record(TEST_RECORD_NAME + "2").namespace(TEST_NAMESPACE).fields().name(TEST_FIELD_NAME).type() .stringType().noDefault().endRecord(); kafkaSchemaRegistry.register(schema1, TEST_TOPIC_NAME); kafkaSchemaRegistry.register(schema2, TEST_TOPIC_NAME); Assert.assertNotEquals(schema1, kafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)); Assert.assertEquals(schema2, kafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)); } }
@JsonCreator public SchemaRegistryBasedAvroBytesDecoder( @JsonProperty("url") String url, @JsonProperty("capacity") Integer capacity ) { int identityMapCapacity = capacity == null ? Integer.MAX_VALUE : capacity; this.registry = new CachedSchemaRegistryClient(url, identityMapCapacity); }
@SuppressWarnings("SameParameterValue") private void givenAvroSchemaNotEvolveable(final String topicName) { final org.apache.avro.Schema schema = org.apache.avro.Schema.create(Type.INT); try { schemaRegistryClient.register(topicName + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX, schema); } catch (final Exception e) { fail(e.getMessage()); } }
public ConfluentKafkaSchemaRegistry(Properties props) { this(props, new CachedSchemaRegistryClient(props.getProperty(KAFKA_SCHEMA_REGISTRY_URL), Integer.parseInt(props.getProperty(CONFLUENT_MAX_SCHEMAS_PER_SUBJECT, String.valueOf(Integer.MAX_VALUE))))); }
public SecorSchemaRegistryClient(SecorConfig config) { try { Properties props = new Properties(); props.put("schema.registry.url", config.getSchemaRegistryUrl()); schemaRegistryClient = new CachedSchemaRegistryClient(config.getSchemaRegistryUrl(), 30); init(config); } catch (Exception e){ LOG.error("Error initalizing schema registry", e); throw new RuntimeException(e); } }
public AvroProducer(final KsqlConfig ksqlConfig) { if (ksqlConfig.getString(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY) == null) { throw new KsqlException("Schema registry url is not set."); } this.ksqlConfig = ksqlConfig; this.schemaRegistryClient = new CachedSchemaRegistryClient( ksqlConfig.getString(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY), 100 ); }