/** * Construct a table manager instance * @param config job configuration */ public TableManager(Config config) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { addTable(tableId, config); logger.debug("Added table " + tableId); }); logger.info(String.format("Added %d tables", tableContexts.size())); }
/** * Get registry keys of value serde for this table * @param tableId Id of the table * @return serde retistry key */ public String getValueSerde(String tableId) { return get(String.format(TABLE_VALUE_SERDE, tableId), null); } }
/** * Get Id's of all tables * @return list of table Id's */ public List<String> getTableIds() { Config subConfig = subset(TABLES_PREFIX, true); Set<String> tableNames = subConfig.keySet().stream() .filter(k -> k.endsWith(TABLE_PROVIDER_FACTORY_SUFFIX)) .map(k -> k.replace(TABLE_PROVIDER_FACTORY_SUFFIX, "")) .collect(Collectors.toSet()); return new LinkedList<>(tableNames); }
/** * Get table config value for a key * @param tableId Id of the table * @param key Key for the config item * @param defaultValue default value if absent in config * @return config value for the key */ public String getForTable(String tableId, String key, String defaultValue) { return get(buildKey(tableId, key), defaultValue); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
mergedConfigMap.putAll(generatedConfig); JobConfig mergedJobConfig = new JobConfig(new MapConfig(mergedConfigMap)); JavaTableConfig tableConfig = new JavaTableConfig(mergedJobConfig); String keySerde = tableConfig.getKeySerde(tableSpec.getId()); storeConfig.put(String.format(StorageConfig.KEY_SERDE(), tableSpec.getId()), keySerde); String valueSerde = tableConfig.getValueSerde(tableSpec.getId()); storeConfig.put(String.format(StorageConfig.MSG_SERDE(), tableSpec.getId()), valueSerde);
@Test public void testGetTableProperties() { Map<String, String> map = new HashMap<>(); map.put("stores.t1.key.serde", "key-serde"); map.put("stores.t1.msg.serde", "msg-serde"); map.put("tables.t1.provider.factory", "t1-provider-factory"); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); assertEquals("t1-provider-factory", tableConfig.getTableProviderFactory("t1")); assertEquals("key-serde", tableConfig.getKeySerde("t1")); assertEquals("msg-serde", tableConfig.getMsgSerde("t1")); }
@Override public ReadWriteTable getTable() { Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId)); JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig()); Cache guavaCache = SerdeUtils.deserialize(GuavaCacheTableDescriptor.GUAVA_CACHE, tableConfig.getForTable(tableId, GuavaCacheTableDescriptor.GUAVA_CACHE)); GuavaCacheTable table = new GuavaCacheTable(tableId, guavaCache); table.init(this.context); guavaTables.add(table); return table; }
@Test public void testGetForTable() { Map<String, String> map = new HashMap<>(); map.put(JavaTableConfig.buildKey("t1", "abc"), "xyz"); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); Assert.assertEquals("xyz", tableConfig.getForTable("t1", "abc")); Assert.assertNull(tableConfig.getForTable("t1", "aaa")); Assert.assertEquals("xyz", tableConfig.getForTable("t1", "aaa", "xyz")); Assert.assertNull(tableConfig.getForTable("tt", "abc")); Assert.assertEquals("xyz", tableConfig.getForTable("tt", "abc", "xyz")); } }
private void addTable(String tableId, Config config) { if (tableContexts.containsKey(tableId)) { throw new SamzaException("Table " + tableId + " already exists"); } JavaTableConfig tableConfig = new JavaTableConfig(config); String providerFactoryClassName = tableConfig.getTableProviderFactory(tableId); TableProviderFactory tableProviderFactory = Util.getObj(providerFactoryClassName, TableProviderFactory.class); TableCtx ctx = new TableCtx(); ctx.tableProvider = tableProviderFactory.getTableProvider(tableId); tableContexts.put(tableId, ctx); }
/** * Helper method to add a config item to table configuration * @param key key of the config item * @param value value of the config item * @param tableConfig table configuration */ protected void addTableConfig(String key, String value, Map<String, String> tableConfig) { tableConfig.put(JavaTableConfig.buildKey(tableId, key), value); }
/** * Get table config value for a key * @param tableId Id of the table * @param key Key for the config item * @return config value for the key */ public String getForTable(String tableId, String key) { return getForTable(tableId, key, null); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
mergedConfigMap.putAll(generatedConfig); JobConfig mergedJobConfig = new JobConfig(new MapConfig(mergedConfigMap)); JavaTableConfig tableConfig = new JavaTableConfig(mergedJobConfig); String keySerde = tableConfig.getKeySerde(tableSpec.getId()); storeConfig.put(String.format(StorageConfig.KEY_SERDE(), tableSpec.getId()), keySerde); String valueSerde = tableConfig.getValueSerde(tableSpec.getId()); storeConfig.put(String.format(StorageConfig.MSG_SERDE(), tableSpec.getId()), valueSerde);
@Test public void testWithSerdes() { List<TableDescriptor> descriptors = Arrays.asList( new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())) ); Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors)); JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig); assertNotNull(javaTableConfig.getKeySerde("t1")); assertNotNull(javaTableConfig.getMsgSerde("t1")); assertNotNull(javaTableConfig.getKeySerde("t2")); assertNotNull(javaTableConfig.getMsgSerde("t2")); MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors)); javaTableConfig = new JavaTableConfig(tableConfig); assertNotNull(javaTableConfig.getTableProviderFactory("t1")); assertNotNull(javaTableConfig.getTableProviderFactory("t2")); }
@Override public ReadWriteTable getTable() { Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId)); JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig()); String realTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.REAL_TABLE_ID); ReadWriteTable table = this.context.getTaskContext().getTable(realTableId); String cacheTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.CACHE_TABLE_ID); ReadWriteTable cache; if (cacheTableId != null) { cache = this.context.getTaskContext().getTable(cacheTableId); } else { cache = createDefaultCacheTable(realTableId, tableConfig); defaultCaches.add(cache); } boolean isWriteAround = Boolean.parseBoolean(tableConfig.getForTable(tableId, CachingTableDescriptor.WRITE_AROUND)); CachingTable cachingTable = new CachingTable(tableId, table, cache, isWriteAround); cachingTable.init(this.context); return cachingTable; }
private void assertEquals(String expectedValue, String key, String tableId, Map<String, String> config) { String realKey = JavaTableConfig.buildKey(tableId, key); Assert.assertEquals(expectedValue, config.get(realKey)); } }
private <T> T deserializeObject(JavaTableConfig tableConfig, String key) { String entry = tableConfig.getForTable(tableId, key, ""); if (entry.isEmpty()) { return null; } return SerdeUtils.deserialize(key, entry); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
@Test public void testGetTableIds() { Set<String> ids = Sets.newHashSet("t1", "t2"); Map<String, String> map = ids.stream() .map(id -> String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, id)) .collect(Collectors.toMap(key -> key, key -> key + "-provider-factory")); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); assertEquals(2, tableConfig.getTableIds().size()); ids.removeAll(tableConfig.getTableIds()); assertTrue(ids.isEmpty()); }