/** * Gets the serialized instance of SideInputsProcessor associated with the {@code storeName}. * * @param storeName name of the store * @return the serialized instance of SideInputsProcessor if present, null otherwise */ public String getSideInputsProcessorSerializedInstance(String storeName) { return get(String.format(SIDE_INPUTS_PROCESSOR_SERIALIZED_INSTANCE, storeName), null); } }
public boolean getAccessLogEnabled(String storeName) { return getBoolean(String.format(ACCESSLOG_ENABLED, storeName), false); }
public int getAccessLogSamplingRatio(String storeName) { return getInt(String.format(ACCESSLOG_SAMPLING_RATIO, storeName), DEFAULT_ACCESSLOG_SAMPLING_RATIO); }
JavaStorageConfig storageConfig = new JavaStorageConfig(config); Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name)))); if (storageConfig.getAccessLogEnabled(storeName)) { String accesslogStream = storageConfig.getAccessLogStream(systemStream.getStream()); StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions);
/** * Get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
@Test public void testStorageConfig() { Map<String, String> map = new HashMap<String, String>(); map.put("stores.test.factory", "testFactory"); map.put("stores.test.changelog", "testSystem.testChangelog"); map.put("stores.test.key.serde", "string"); map.put("stores.test.msg.serde", "integer"); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(map)); assertEquals("testFactory", config.getStorageFactoryClassName("test")); assertEquals("testSystem.testChangelog", config.getChangelogStream("test")); assertEquals("string", config.getStorageKeySerde("test")); assertEquals("integer", config.getStorageMsgSerde("test")); assertEquals("test", config.getStoreNames().get(0)); }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1, new File(dbPath), StorageEngineFactory.StoreMode.ReadWrite); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }
public String getChangelogStream(String storeName) { // If the config specifies 'stores.<storename>.changelog' as '<system>.<stream>' combination - it will take precedence. // If this config only specifies <astream> and there is a value in job.changelog.system=<asystem> - // these values will be combined into <asystem>.<astream> String systemStream = StringUtils.trimToNull(get(String.format(CHANGELOG_STREAM, storeName), null)); String systemStreamRes; if (systemStream != null && !systemStream.contains(".")) { String changelogSystem = getChangelogSystem(); // contains only stream name if (changelogSystem != null) { systemStreamRes = changelogSystem + "." + systemStream; } else { throw new SamzaException("changelog system is not defined:" + systemStream); } } else { systemStreamRes = systemStream; } if (systemStreamRes != null) { systemStreamRes = StreamManager.createUniqueNameForBatch(systemStreamRes, this); } return systemStreamRes; }
@Test public void testEmptyStringOrNullChangelogStream() { Map<String, String> configMap = new HashMap<>(); configMap.put("stores.store1.changelog", ""); configMap.put("stores.store2.changelog", " "); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(configMap)); assertNull(config.getChangelogStream("store1")); assertNull(config.getChangelogStream("store2")); assertNull(config.getChangelogStream("store-changelog-none")); } }
public JavaStorageConfig getStorageConfigs() { HashMap<String, String> configs = new HashMap<>(); configs.put(String.format(StorageConfig.FACTORY(), this.getStoreName()), this.getStoreFactory()); configs.put(String.format(StorageConfig.CHANGELOG_STREAM(), this.getStoreName()), this.getChangelogStream()); configs.putAll(this.getOtherProperties()); return new JavaStorageConfig(new MapConfig(configs)); }
JavaStorageConfig storageConfig = new JavaStorageConfig(config); Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name)))); if (storageConfig.getAccessLogEnabled(storeName)) { String accesslogStream = storageConfig.getAccessLogStream(systemStream.getStream()); StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions);
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }
public String getChangelogStream(String storeName) { // If the config specifies 'stores.<storename>.changelog' as '<system>.<stream>' combination - it will take precedence. // If this config only specifies <astream> and there is a value in job.changelog.system=<asystem> - // these values will be combined into <asystem>.<astream> String systemStream = StringUtils.trimToNull(get(String.format(CHANGELOG_STREAM, storeName), null)); String systemStreamRes; if (systemStream != null && !systemStream.contains(".")) { String changelogSystem = getChangelogSystem(); // contains only stream name if (changelogSystem != null) { systemStreamRes = changelogSystem + "." + systemStream; } else { throw new SamzaException("changelog system is not defined:" + systemStream); } } else { systemStreamRes = systemStream; } if (systemStreamRes != null) { systemStreamRes = StreamManager.createUniqueNameForBatch(systemStreamRes, this); } return systemStreamRes; }
@Test public void testIsChangelogSystemSetting() { Map<String, String> configMap = new HashMap<>(); configMap.put("stores.store1.changelog", "system1.stream1"); configMap.put("job.changelog.system", "system2"); configMap.put("stores.store2.changelog", "stream2"); JavaStorageConfig config = new JavaStorageConfig(new MapConfig(configMap)); assertEquals("system1.stream1", config.getChangelogStream("store1")); assertEquals("system2.stream2", config.getChangelogStream("store2")); Map<String, String> configMapErr = new HashMap<>(); configMapErr.put("stores.store4.changelog", "stream4"); // incorrect JavaStorageConfig configErr = new JavaStorageConfig(new MapConfig(configMapErr)); try { configErr.getChangelogStream("store4"); fail("store4 has no system defined. Should've failed."); } catch (Exception e) { // do nothing, it is expected } }
public JavaStorageConfig getStorageConfigs() { HashMap<String, String> configs = new HashMap<>(); configs.put(String.format(StorageConfig.FACTORY(), this.getStoreName()), this.getStoreFactory()); configs.put(String.format(StorageConfig.CHANGELOG_STREAM(), this.getStoreName()), this.getChangelogStream()); configs.putAll(this.getOtherProperties()); return new JavaStorageConfig(new MapConfig(configs)); }
JavaStorageConfig storageConfig = new JavaStorageConfig(config); Map<String, SystemStream> storeNameSystemStreamMapping = storageConfig.getStoreNames() .stream() .filter(name -> StringUtils.isNotBlank(storageConfig.getChangelogStream(name))) .collect(Collectors.toMap(name -> name, name -> StreamUtil.getSystemStreamFromNames(storageConfig.getChangelogStream(name)))); if (storageConfig.getAccessLogEnabled(storeName)) { String accesslogStream = storageConfig.getAccessLogStream(systemStream.getStream()); StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions);
/** * get the changelog streams and the storage factories from the config file * and put them into the maps */ private void getChangeLogSystemStreamsAndStorageFactories() { JavaStorageConfig config = new JavaStorageConfig(jobConfig); List<String> storeNames = config.getStoreNames(); log.info("Got store names: " + storeNames.toString()); for (String storeName : storeNames) { String streamName = config.getChangelogStream(storeName); log.info("stream name for " + storeName + " is " + streamName); if (streamName != null) { changeLogSystemStreams.put(storeName, StreamUtil.getSystemStreamFromNames(streamName)); } String factoryClass = config.getStorageFactoryClassName(storeName); if (factoryClass != null) { storageEngineFactories.put(storeName, Util.getObj(factoryClass, StorageEngineFactory.class)); } else { throw new SamzaException("Missing storage factory for " + storeName + "."); } } }
/** * Gets the SideInputsProcessorFactory associated with the {@code storeName}. * * @param storeName name of the store * @return the class name of SideInputsProcessorFactory if present, null otherwise */ public String getSideInputsProcessorFactory(String storeName) { return get(String.format(SIDE_INPUTS_PROCESSOR_FACTORY, storeName), null); }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }