/** * Get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name)))
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name)))
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name)))
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name)))
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name)))
@Test public void testStorageConfig() { Map<String, String> map = new HashMap<String, String>(); map.put("stores.test.factory", "testFactory"); map.put("stores.test.changelog", "testSystem.testChangelog"); map.put("stores.test.key.serde", "string"); map.put("stores.test.msg.serde", "integer"); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(map)); assertEquals("testFactory", config.getStorageFactoryClassName("test")); assertEquals("testSystem.testChangelog", config.getChangelogStream("test")); assertEquals("string", config.getStorageKeySerde("test")); assertEquals("integer", config.getStorageMsgSerde("test")); assertEquals("test", config.getStoreNames().get(0)); }