/** * Construct a table manager instance * @param config job configuration */ public TableManager(Config config) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { addTable(tableId, config); logger.debug("Added table " + tableId); }); logger.info(String.format("Added %d tables", tableContexts.size())); }
private void addTable(String tableId, Config config) { if (tableContexts.containsKey(tableId)) { throw new SamzaException("Table " + tableId + " already exists"); } JavaTableConfig tableConfig = new JavaTableConfig(config); String providerFactoryClassName = tableConfig.getTableProviderFactory(tableId); TableProviderFactory tableProviderFactory = Util.getObj(providerFactoryClassName, TableProviderFactory.class); TableCtx ctx = new TableCtx(); ctx.tableProvider = tableProviderFactory.getTableProvider(tableId); tableContexts.put(tableId, ctx); }
@Override public ReadWriteTable getTable() { Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId)); JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig()); Cache guavaCache = SerdeUtils.deserialize(GuavaCacheTableDescriptor.GUAVA_CACHE, tableConfig.getForTable(tableId, GuavaCacheTableDescriptor.GUAVA_CACHE)); GuavaCacheTable table = new GuavaCacheTable(tableId, guavaCache); table.init(this.context); guavaTables.add(table); return table; }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
@Test public void testGetTableIds() { Set<String> ids = Sets.newHashSet("t1", "t2"); Map<String, String> map = ids.stream() .map(id -> String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, id)) .collect(Collectors.toMap(key -> key, key -> key + "-provider-factory")); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); assertEquals(2, tableConfig.getTableIds().size()); ids.removeAll(tableConfig.getTableIds()); assertTrue(ids.isEmpty()); }
mergedConfigMap.putAll(generatedConfig); JobConfig mergedJobConfig = new JobConfig(new MapConfig(mergedConfigMap)); JavaTableConfig tableConfig = new JavaTableConfig(mergedJobConfig);
mergedConfigMap.putAll(generatedConfig); JobConfig mergedJobConfig = new JobConfig(new MapConfig(mergedConfigMap)); JavaTableConfig tableConfig = new JavaTableConfig(mergedJobConfig);
JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig());
@Override public ReadWriteTable getTable() { Preconditions.checkNotNull(context, String.format("Table %s not initialized", tableId)); JavaTableConfig tableConfig = new JavaTableConfig(context.getJobContext().getConfig()); String realTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.REAL_TABLE_ID); ReadWriteTable table = this.context.getTaskContext().getTable(realTableId); String cacheTableId = tableConfig.getForTable(tableId, CachingTableDescriptor.CACHE_TABLE_ID); ReadWriteTable cache; if (cacheTableId != null) { cache = this.context.getTaskContext().getTable(cacheTableId); } else { cache = createDefaultCacheTable(realTableId, tableConfig); defaultCaches.add(cache); } boolean isWriteAround = Boolean.parseBoolean(tableConfig.getForTable(tableId, CachingTableDescriptor.WRITE_AROUND)); CachingTable cachingTable = new CachingTable(tableId, table, cache, isWriteAround); cachingTable.init(this.context); return cachingTable; }
@Test public void testGetTableProperties() { Map<String, String> map = new HashMap<>(); map.put("stores.t1.key.serde", "key-serde"); map.put("stores.t1.msg.serde", "msg-serde"); map.put("tables.t1.provider.factory", "t1-provider-factory"); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); assertEquals("t1-provider-factory", tableConfig.getTableProviderFactory("t1")); assertEquals("key-serde", tableConfig.getKeySerde("t1")); assertEquals("msg-serde", tableConfig.getMsgSerde("t1")); }
@Test public void testGetForTable() { Map<String, String> map = new HashMap<>(); map.put(JavaTableConfig.buildKey("t1", "abc"), "xyz"); JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map)); Assert.assertEquals("xyz", tableConfig.getForTable("t1", "abc")); Assert.assertNull(tableConfig.getForTable("t1", "aaa")); Assert.assertEquals("xyz", tableConfig.getForTable("t1", "aaa", "xyz")); Assert.assertNull(tableConfig.getForTable("tt", "abc")); Assert.assertEquals("xyz", tableConfig.getForTable("tt", "abc", "xyz")); } }
@Test public void testWithSerdes() { List<TableDescriptor> descriptors = Arrays.asList( new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())) ); Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors)); JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig); assertNotNull(javaTableConfig.getKeySerde("t1")); assertNotNull(javaTableConfig.getMsgSerde("t1")); assertNotNull(javaTableConfig.getKeySerde("t2")); assertNotNull(javaTableConfig.getMsgSerde("t2")); MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors)); javaTableConfig = new JavaTableConfig(tableConfig); assertNotNull(javaTableConfig.getTableProviderFactory("t1")); assertNotNull(javaTableConfig.getTableProviderFactory("t2")); }