@OnEnabled public void enable(final ConfigurationContext context) throws InitializationException { schemaRegistryConfig = new HashMap<>(); versionInfoCacheNanos = context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.NANOSECONDS); // The below properties may or may not need to be exposed to the end // user. We just need to watch usage patterns to see if sensible default // can satisfy NiFi requirements String urlValue = context.getProperty(URL).evaluateAttributeExpressions().getValue(); if (urlValue == null || urlValue.trim().isEmpty()) { throw new IllegalArgumentException("'Schema Registry URL' must not be null or empty."); } schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), urlValue); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_SIZE.name(), 10L); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_EXPIRY_INTERVAL_SECS.name(), context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.SECONDS)); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_SIZE.name(), context.getProperty(CACHE_SIZE).asInteger()); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_EXPIRY_INTERVAL_SECS.name(), context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.SECONDS)); }
private Map<String, ?> buildConfig(Map<String, ?> config) { Map<String, Object> result = new HashMap<>(); for (Map.Entry<String, ?> entry : config.entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); ConfigEntry configEntry = options.get(key); if (configEntry != null) { if (value != null) { configEntry.validator().validate((value)); } else { value = configEntry.defaultValue(); } } result.put(key, value); } return result; }
public static <T> ConfigEntry<T> mandatory(String name, Class<? extends T> type, String description, T defaultValue, Validator<? extends T> validator) { return new ConfigEntry<T>(name, type, description, true, defaultValue, validator); }
public <T> T getValue(String propertyKey) { return (T) (config.containsKey(propertyKey) ? config.get(propertyKey) : options.get(propertyKey).defaultValue()); }
public static <T> ConfigEntry<T> optional(String name, Class<? extends T> type, String description, T defaultValue, Validator<? extends T> validator) { return new ConfigEntry<T>(name, type, description, false, defaultValue, validator); }
public static Map<String, Object> createConfig(String schemaRegistryUrl) { Map<String, Object> config = new HashMap<>(); config.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), schemaRegistryUrl); config.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_SIZE.name(), 10L); config.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_EXPIRY_INTERVAL_SECS.name(), 5000L); config.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_SIZE.name(), 1000L); config.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_EXPIRY_INTERVAL_SECS.name(), 60 * 60 * 1000L); return config; }
public static <T> ConfigEntry<T> optional(String name, Class<? extends T> type, String description, T defaultValue, Validator<? extends T> validator) { return new ConfigEntry<T>(name, type, description, false, defaultValue, validator); }
public static Map<String, Object> createConfig(String schemaRegistryUrl) { Map<String, Object> config = new HashMap<>(); config.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), schemaRegistryUrl); config.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_SIZE.name(), 10L); config.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_EXPIRY_INTERVAL_SECS.name(), 5000L); config.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_SIZE.name(), 1000L); config.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_EXPIRY_INTERVAL_SECS.name(), 60 * 60 * 1000L); return config; } }
public static <T> ConfigEntry<T> mandatory(String name, Class<? extends T> type, String description, T defaultValue, Validator<? extends T> validator) { return new ConfigEntry<T>(name, type, description, true, defaultValue, validator); }
private SchemaRegistryClient createSchemaRegistryClient() { Map<String, ?> conf = Collections.singletonMap(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), config.get("schemaRegistryUrl")); return new SchemaRegistryClient(conf); }
private Map<String, ConfigEntry<?>> buildOptions(Field[] fields) { Map<String, ConfigEntry<?>> options = new HashMap<>(); for (Field field : fields) { Class<?> type = field.getType(); if (type.isAssignableFrom(ConfigEntry.class)) { field.setAccessible(true); try { ConfigEntry configEntry = (ConfigEntry) field.get(this); options.put(configEntry.name(), configEntry); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } } return options; }
private AvroStreamsSnapshotDeserializer deserializer () { //initializing deserializer here should be synchronized (using DCL pattern?) when single threaded nature of kafka spout does not hold true anymore if (avroStreamsSnapshotDeserializer == null) { AvroStreamsSnapshotDeserializer deserializer = new AvroStreamsSnapshotDeserializer(); Map<String, Object> config = new HashMap<>(); config.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), schemaRegistryUrl); deserializer.init(config); avroStreamsSnapshotDeserializer = deserializer; } return avroStreamsSnapshotDeserializer; }
private void consumeUserActivity() { Properties props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); props.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), SCHEMA_REGISTRY_URL); props.put(ConsumerConfig.GROUP_ID_CONFIG, "user-activity-reader"); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, LongDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(AbstractAvroSnapshotDeserializer.SPECIFIC_AVRO_READER, true); try (KafkaConsumer<Integer, UserActivity> consumer = new KafkaConsumer<>(props)) { consumer.subscribe(Collections.singleton(USER_ACTIVITY_TOPIC)); while (true) { final ConsumerRecords<Integer, UserActivity> consumerRecords = consumer.poll(Duration.ofSeconds(1)); consumerRecords.forEach(System.out::println); } } }
private UrlSelector createUrlSelector() { UrlSelector urlSelector = null; String rootCatalogURL = configuration.getValue(SCHEMA_REGISTRY_URL.name()); String urlSelectorClass = configuration.getValue(Configuration.URL_SELECTOR_CLASS.name()); if (urlSelectorClass == null) { urlSelector = new LoadBalancedFailoverUrlSelector(rootCatalogURL); } else { try { urlSelector = (UrlSelector) Class.forName(urlSelectorClass) .getConstructor(String.class) .newInstance(rootCatalogURL); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) { throw new RuntimeException(e); } } urlSelector.init(configuration.getConfig()); return urlSelector; }
public ClassLoaderCache(final SchemaRegistryClient schemaRegistryClient) { this.schemaRegistryClient = schemaRegistryClient; CacheLoader<String, ClassLoader> cacheLoader = new CacheLoader<String, ClassLoader>() { @Override public ClassLoader load(String fileId) throws Exception { File file = getFile(fileId); return new URLClassLoader(new URL[]{file.toURI().toURL()}); } }; SchemaRegistryClient.Configuration configuration = schemaRegistryClient.getConfiguration(); loadingCache = CacheBuilder.newBuilder() .maximumSize(((Number) configuration.getValue(CACHE_SIZE_KEY)).longValue()) .expireAfterAccess(((Number) configuration.getValue(CACHE_EXPIRY_INTERVAL_KEY)).longValue(), TimeUnit.SECONDS) .build(cacheLoader); localJarsDir = new File((String) this.schemaRegistryClient.getConfiguration().getValue(SchemaRegistryClient.Configuration.LOCAL_JAR_PATH.name())); ensureLocalDirsExist(); }
public static void main(String[] args) throws Exception { String schemaRegistryUrl = System.getProperty(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), DEFAULT_SCHEMA_REG_URL); Map<String, Object> config = createConfig(schemaRegistryUrl); SampleSchemaRegistryClientApp sampleSchemaRegistryClientApp = new SampleSchemaRegistryClientApp(config); sampleSchemaRegistryClientApp.runSchemaApis(); sampleSchemaRegistryClientApp.runCustomSerDesApi(); sampleSchemaRegistryClientApp.runAvroSerDesApis(); }
props.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), SCHEMA_REGISTRY_URL); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class);
((Number) configuration.getValue(Configuration.SCHEMA_VERSION_CACHE_SIZE.name())).intValue(), ((Number) configuration.getValue(Configuration.SCHEMA_VERSION_CACHE_EXPIRY_INTERVAL_SECS.name())).longValue() * 1000L ); .name())).longValue(), ((Number) configuration.getValue(Configuration.SCHEMA_METADATA_CACHE_EXPIRY_INTERVAL_SECS .name())).longValue(), schemaMetadataFetcher); .name())).longValue()) .expireAfterAccess(((Number) configuration.getValue(Configuration.SCHEMA_TEXT_CACHE_EXPIRY_INTERVAL_SECS .name())).longValue(), TimeUnit.SECONDS) .build();
private Map<String, Object> createClientConf() { String registryURL = localSchemaRegistryServer.getLocalURL() + V1_API_PATH; if (schemaRegistryTestConfiguration.getClientYAMLPath() == null) { Map<String, Object> ret = new HashMap<>(); ret.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), registryURL); return ret; } try (FileInputStream fis = new FileInputStream(schemaRegistryTestConfiguration.getClientYAMLPath())) { Map<String, Object> ret = (Map<String, Object>) new Yaml().load(IOUtils.toString(fis, "UTF-8")); ret.put("schema.registry.url", registryURL); return ret; } catch(Exception e) { throw new RuntimeException("Failed to export schema client configuration for yaml : " + schemaRegistryTestConfiguration.getClientYAMLPath(), e); } } }
registryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), registryUrl); registryConfig.put(SERDES_PROTOCOL_VERSION, SerDesProtocolHandlerRegistry.CONFLUENT_VERSION_PROTOCOL);