/** * Pulled from global config. * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. * * @return batch size for writing to kafka * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_SIZE_CONF */ public int getBatchSize() { return getAs(BATCH_SIZE_CONF, getGlobalConfig(true), DEFAULT_KAFKA_BATCH_SIZE, Integer.class); }
/** * Pulled from global config * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. * * @return batch timeout for writing to kafka * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_TIMEOUT_CONF */ public int getBatchTimeout() { return getAs(BATCH_TIMEOUT_CONF, getGlobalConfig(true), 0, Integer.class); }
@Override public Map<String, Object> getGlobalConfig() { return config.orElse(new EnrichmentConfigurations()).getGlobalConfig(); }
@Override public Map<String, Object> get() throws RestException { Map<String, Object> globalConfig; try { EnrichmentConfigurations configs = cache.get( EnrichmentConfigurations.class); globalConfig = configs.getGlobalConfig(false); } catch (Exception e) { throw new RestException(e.getMessage(), e); } return globalConfig; }
@Override public void prepare(Map map, TopologyContext topologyContext) { super.prepare(map, topologyContext); GeoLiteCityDatabase.INSTANCE.update((String)getConfigurations().getGlobalConfig().get( GeoLiteCityDatabase.GEO_HDFS_FILE)); GeoLiteAsnDatabase.INSTANCE.update((String)getConfigurations().getGlobalConfig().get( GeoLiteAsnDatabase.ASN_HDFS_FILE)); initializeStellar(); }
@Override public final void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { super.prepare(map, topologyContext, outputCollector); perfLog = new PerformanceLogger(() -> getConfigurations().getGlobalConfig(), Perf.class.getName()); collector = outputCollector; prepare(map, topologyContext); }
@Override public void reloadCallback(String name, ConfigurationType type) { if(invalidateCacheOnReload) { if (cache != null) { cache.invalidateAll(); } } if(type == ConfigurationType.GLOBAL) { adapter.updateAdapter(getConfigurations().getGlobalConfig()); } }
@Override public void reloadCallback(String name, ConfigurationType type) { super.reloadCallback(name, type); if(type == ConfigurationType.GLOBAL) { GeoLiteCityDatabase.INSTANCE.updateIfNecessary(getConfigurations().getGlobalConfig()); } }
protected void initializeStellar() { stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig()) .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig()) .build(); StellarFunctions.initialize(stellarContext); }
protected void initializeStellar() { stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig()) .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig()) .build(); StellarFunctions.initialize(stellarContext); }
@Override public void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector) { super.prepare(conf, topologyContext, collector); this.collector = collector; if (this.maxCacheSize == null) throw new IllegalStateException("MAX_CACHE_SIZE_OBJECTS_NUM must be specified"); if (this.maxTimeRetain == null) throw new IllegalStateException("MAX_TIME_RETAIN_MINUTES must be specified"); if (this.adapter == null) throw new IllegalStateException("Adapter must be specified"); loader = key -> adapter.enrich(key); cache = Caffeine.newBuilder().maximumSize(maxCacheSize) .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES) .build(loader); boolean success = adapter.initializeAdapter(getConfigurations().getGlobalConfig()); if (!success) { LOG.error("[Metron] GenericEnrichmentBolt could not initialize adapter"); throw new IllegalStateException("Could not initialize adapter..."); } perfLog = new PerformanceLogger(() -> getConfigurations().getGlobalConfig(), GenericEnrichmentBolt.Perf.class.getName()); initializeStellar(); }
boolean success = adapterKv.getValue().initializeAdapter(getConfigurations().getGlobalConfig()); if (!success) { LOG.error("[Metron] Could not initialize adapter: " + adapterKv.getKey()); perfLog = new PerformanceLogger(() -> getConfigurations().getGlobalConfig(), Perf.class.getName()); GeoLiteCityDatabase.INSTANCE.update((String)getConfigurations().getGlobalConfig().get( GeoLiteCityDatabase.GEO_HDFS_FILE)); GeoLiteAsnDatabase.INSTANCE.update((String)getConfigurations().getGlobalConfig().get( GeoLiteAsnDatabase.ASN_HDFS_FILE)); initializeStellar();
protected void initializeStellar() { this.stellarContext = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig()) .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig()) .build(); StellarFunctions.initialize(stellarContext); this.functionResolver = StellarFunctions.FUNCTION_RESOLVER(); }
@Override public void reloadCallback(String name, ConfigurationType type) { if(invalidateCacheOnReload) { if(strategy != null && ConcurrencyContext.get(strategy).getCache() != null) { ConcurrencyContext.get(strategy).getCache().invalidateAll(); } } if(type == ConfigurationType.GLOBAL && enrichmentsByType != null) { for(EnrichmentAdapter adapter : enrichmentsByType.values()) { adapter.updateAdapter(getConfigurations().getGlobalConfig()); } } }
@Override public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { super.prepare(map, topologyContext, outputCollector); perfLog = new PerformanceLogger(() -> getConfigurations().getGlobalConfig(), Perf.class.getName()); keyGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("key"); subgroupGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("subgroup"); messageGetStrategy = MessageGetters.OBJECT_FROM_FIELD.get("message"); this.collector = outputCollector; if (this.maxCacheSize == null) { throw new IllegalStateException("maxCacheSize must be specified"); } if (this.maxTimeRetain == null) { throw new IllegalStateException("maxTimeRetain must be specified"); } loader = s -> new HashMap<>(); cache = Caffeine.newBuilder().maximumSize(maxCacheSize) .expireAfterWrite(maxTimeRetain, TimeUnit.MINUTES) .removalListener(new JoinRemoveListener()) .build(loader); prepare(map, topologyContext); }
Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig());
EnrichmentConfigurations config = cache.get( EnrichmentConfigurations.class); assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test"))); assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
assertEventually(()-> Assert.assertNull(config.getGlobalConfig(false)));
EnrichmentConfigurations config = cache.get( EnrichmentConfigurations.class); assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test"))); assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig())); assertEventually(() -> Assert.assertNull(config.getSensorEnrichmentConfig("notthere")));