private synchronized S fetchFromSchemaRegistry(String topic, String id) { try { S schema = registry.getSchemaByID(topic, id); return schema; } catch (SchemaNotFoundException e) { addFetchToFailureHistory(id); throw e; } }
public String register(String topic, S schema) { return registry.register(topic, schema); }
public S getSchemaByID(String topic, String id) { CachedSchemaTuple cacheKey = new CachedSchemaTuple(topic, id); S schema = cachedById.get(cacheKey); if (schema == null) { synchronized (this) { if (shouldFetchFromSchemaRegistry(id)) { schema = fetchFromSchemaRegistry(topic, id); cachedById.putIfAbsent(new CachedSchemaTuple(topic, id), schema); } else { throw new SchemaNotFoundException(); } } } return schema; }
@Test public void testMaxRetries() { EasyMock.expect(registry.getSchemaByID(EasyMock.anyString(), EasyMock.anyString())).andThrow( new SchemaNotFoundException()); EasyMock.expectLastCall().times(20); EasyMock.replay(registry); props.setProperty(GET_SCHEMA_BY_ID_MAX_RETIRES, "10"); props.setProperty(GET_SCHEMA_BY_ID_MIN_INTERVAL_SECONDS, "0"); cachedRegistry = new CachedSchemaRegistry<Schema>(registry, props); for (int i = 0; i < 100; i++) { try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID"); } catch (SchemaNotFoundException e) { } try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID2"); } catch (SchemaNotFoundException e) { } } EasyMock.verify(registry); }
@Override public SchemaDetails<S> getLatestSchemaByTopic(String topicName) { File topicDir = getTopicPath(topicName); if (topicDir.exists()) { for (File file : topicDir.listFiles()) { if (file.getName().endsWith(".latest")) { String id = file.getName().replace(".schema.latest", ""); return new SchemaDetails<S>(topicName, id, serde.fromBytes(readBytes(file))); } } } throw new SchemaNotFoundException("Unable to find a latest schema for topic " + topicName + "."); }
public void init(Properties props, String topicName) { super.init(props, topicName); this.props = props; this.topicName = topicName; this.schemaIDField = props.getProperty(CAMUS_SCHEMA_ID_FIELD, DEFAULT_SCHEMA_ID_FIELD); try { SchemaRegistry<Schema> registry = (SchemaRegistry<Schema>) Class.forName( props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)).newInstance(); log.info("Prop " + KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS + " is: " + props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)); log.info("Underlying schema registry for topic: " + topicName + " is: " + registry); registry.init(props); this.registry = new CachedSchemaRegistry<Schema>(registry, props); this.latestSchema = ((Schema) registry.getLatestSchemaByTopic(topicName).getSchema()); } catch (Exception e) { throw new MessageDecoderException(e); } this.decoderFactory = DecoderFactory.get(); }
@Override public S getSchemaByID(String topic, String id) { File file = getSchemaPath(topic, id, false); if (!file.exists()) { file = getSchemaPath(topic, id, true); } if (!file.exists()) { throw new SchemaNotFoundException("No matching schema found for topic " + topic + " and id " + id + "."); } return serde.fromBytes(readBytes(file)); }
@Override public SchemaDetails<S> getLatestSchemaByTopic(String topicName) { MemorySchemaRegistryTuple tuple = latest.get(topicName); if (tuple == null) { throw new SchemaNotFoundException(); } S schema = schemasById.get(tuple); if (schema == null) { throw new SchemaNotFoundException(); } return new SchemaDetails<S>(topicName, Long.toString(tuple.getId()), schema); }
public MessageDecoderHelper invoke(String id) { this.schema = (this.registry.getSchemaByID(this.topicName, id)); if (this.schema == null) throw new IllegalStateException("Unknown schema id: " + id); this.targetSchema = JSONToAvroMessageDecoder.this.latestSchema; return this; } }
public static MessageDecoder createMockDecoder30PercentSchemaNotFound() { MessageDecoder mockDecoder = EasyMock.createNiceMock(MessageDecoder.class); EasyMock.expect(mockDecoder.decode(EasyMock.anyObject())).andThrow(new SchemaNotFoundException()).times(3); EasyMock.expect(mockDecoder.decode(EasyMock.anyObject())).andReturn(new CamusWrapper<String>("dummy")).times(7); EasyMock.replay(mockDecoder); return mockDecoder; }
@Override public CamusWrapper<Record> decode(Message message) { try { GenericDatumReader<Record> reader = new GenericDatumReader<Record>(); Schema schema = super.registry.getLatestSchemaByTopic(super.topicName).getSchema(); reader.setSchema(schema); return new CamusWrapper<Record>(reader.read(null, decoderFactory.jsonDecoder(schema, new String(message.getPayload(), //Message.payloadOffset(message.magic()), kafka.message.Message.MagicOffset(), message.getPayload().length - kafka.message.Message.MagicOffset())))); } catch (Exception e) { throw new RuntimeException(e); } } }
public SchemaDetails<S> getLatestSchemaByTopic(String topicName) { SchemaDetails<S> schema = cachedLatest.get(topicName); if (schema == null) { schema = registry.getLatestSchemaByTopic(topicName); cachedLatest.putIfAbsent(topicName, schema); } return schema; }
public CachedSchemaRegistry(SchemaRegistry<S> registry, Properties props) { this.registry = registry; this.cachedById = new ConcurrentHashMap<CachedSchemaTuple, S>(); this.cachedLatest = new ConcurrentHashMap<String, SchemaDetails<S>>(); this.failedFetchHistories = new HashMap<String, FailedFetchHistory>(); this.init(props); }
private void addFetchToFailureHistory(String id) { if (!failedFetchHistories.containsKey(id)) { failedFetchHistories.put(id, new FailedFetchHistory(1, System.nanoTime())); } else { failedFetchHistories.get(id).numOfAttempts++; failedFetchHistories.get(id).previousAttemptTime = System.nanoTime(); } }
@Test public void testMinInterval() throws InterruptedException { EasyMock.expect(registry.getSchemaByID(EasyMock.anyString(), EasyMock.anyString())).andThrow( new SchemaNotFoundException()); EasyMock.expectLastCall().times(4); EasyMock.replay(registry); props.setProperty(GET_SCHEMA_BY_ID_MAX_RETIRES, String.valueOf(Integer.MAX_VALUE)); props.setProperty(GET_SCHEMA_BY_ID_MIN_INTERVAL_SECONDS, "2"); cachedRegistry = new CachedSchemaRegistry<Schema>(registry, props); for (int i = 0; i < 2; i++) { for (int j = 0; j < 5; j++) { try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID"); } catch (SchemaNotFoundException e) { } try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID2"); } catch (SchemaNotFoundException e) { } } Thread.sleep(2500); } EasyMock.verify(registry); } }
@Override public void init(Properties props, String topicName) { super.init(props, topicName); try { SchemaRegistry<Schema> registry = (SchemaRegistry<Schema>) Class.forName( props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)).newInstance(); log.info("Prop " + KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS + " is: " + props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)); log.info("Underlying schema registry for topic: " + topicName + " is: " + registry); registry.init(props); this.registry = new CachedSchemaRegistry<Schema>(registry, props); this.latestSchema = registry.getLatestSchemaByTopic(topicName).getSchema(); } catch (Exception e) { throw new MessageDecoderException(e); } decoderFactory = DecoderFactory.get(); }
@Override public S getSchemaByID(String topic, String id) { File file = getSchemaPath(topic, id, false); if (!file.exists()) { file = getSchemaPath(topic, id, true); } if (!file.exists()) { throw new SchemaNotFoundException("No matching schema found for topic " + topic + " and id " + id + "."); } return serde.fromBytes(readBytes(file)); }
private synchronized S fetchFromSchemaRegistry(String topic, String id) { try { S schema = registry.getSchemaByID(topic, id); return schema; } catch (SchemaNotFoundException e) { addFetchToFailureHistory(id); throw e; } }
public MessageDecoderHelper invoke() { buffer = getByteBuffer(payload); String id = Integer.toString(buffer.getInt()); schema = registry.getSchemaByID(topicName, id); if (schema == null) throw new IllegalStateException("Unknown schema id: " + id); start = buffer.position() + buffer.arrayOffset(); length = buffer.limit() - 5; // try to get a target schema, if any targetSchema = latestSchema; return this; } }