/** * Get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
@Test public void testEmptyStringOrNullChangelogStream() { Map<String, String> configMap = new HashMap<>(); configMap.put("stores.store1.changelog", ""); configMap.put("stores.store2.changelog", " "); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(configMap)); assertNull(config.getChangelogStream("store1")); assertNull(config.getChangelogStream("store2")); assertNull(config.getChangelogStream("store-changelog-none")); } }
@Test public void testIsChangelogSystemSetting() { Map<String, String> configMap = new HashMap<>(); configMap.put("stores.store1.changelog", "system1.stream1"); configMap.put("job.changelog.system", "system2"); configMap.put("stores.store2.changelog", "stream2"); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(configMap)); assertEquals("system1.stream1", config.getChangelogStream("store1")); assertEquals("system2.stream2", config.getChangelogStream("store2")); Map<String, String> configMapErr = new HashMap<>(); configMapErr.put("stores.store4.changelog", "stream4"); // incorrect JavaStorageConfig configErr = new JavaStorageConfig(new MapConfig(configMapErr)); try { configErr.getChangelogStream("store4"); fail("store4 has no system defined. Should've failed."); } catch (Exception e) { // do nothing, it is expected } }
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name))));
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name))));
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name))));
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name))));
Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name))));
@Test public void testStorageConfig() { Map<String, String> map = new HashMap<String, String>(); map.put("stores.test.factory", "testFactory"); map.put("stores.test.changelog", "testSystem.testChangelog"); map.put("stores.test.key.serde", "string"); map.put("stores.test.msg.serde", "integer"); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(map)); assertEquals("testFactory", config.getStorageFactoryClassName("test")); assertEquals("testSystem.testChangelog", config.getChangelogStream("test")); assertEquals("string", config.getStorageKeySerde("test")); assertEquals("integer", config.getStorageMsgSerde("test")); assertEquals("test", config.getStoreNames().get(0)); }