private ExecutableDao(KylinConfig config) { logger.info("Using metadata url: " + config); this.store = MetadataManager.getInstance(config).getStore(); }
private SnapshotTable load(String resourcePath, boolean loadData) throws IOException { logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData); ResourceStore store = MetadataManager.getInstance(this.config).getStore(); SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER); if (loadData) logger.debug("Loaded snapshot at " + resourcePath); return table; }
DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); logger.debug("Going to load DictionaryInfo from " + resourcePath); DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); if (loadDictObj) logger.debug("Loaded dictionary at " + resourcePath); return info; }
public void removeSnapshot(String resourcePath) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); store.deleteResource(resourcePath); snapshotCache.remove(resourcePath); }
public void removeDictionary(String resourcePath) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); store.deleteResource(resourcePath); dictCache.remove(resourcePath); }
void save(DictionaryInfo dict) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); String path = dict.getResourcePath(); logger.info("Saving dictionary at " + path); store.putResource(path, dict, DictionaryInfoSerializer.FULL_SERIALIZER); }
public void saveTableExd(String tableId, Map<String, String> tableExdProperties) throws IOException { if (tableId == null) { throw new IllegalArgumentException("tableId couldn't be null"); } TableDesc srcTable = srcTableMap.get(tableId); if (srcTable == null) { throw new IllegalArgumentException("Couldn't find Source Table with identifier: " + tableId); } String path = TableDesc.concatExdResourcePath(tableId); ByteArrayOutputStream os = new ByteArrayOutputStream(); JsonUtil.writeValueIndent(os, tableExdProperties); os.flush(); InputStream is = new ByteArrayInputStream(os.toByteArray()); getStore().putResource(path, is, System.currentTimeMillis()); os.close(); is.close(); srcTableExdMap.putLocal(tableId, tableExdProperties); }
private TableDesc reloadSourceTableAt(String path) throws IOException { ResourceStore store = getStore(); TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER); if (t == null) { logger.error("Didn't load table at " + path); return null; } t.init(); String tableIdentity = t.getIdentity(); srcTableMap.putLocal(tableIdentity, t); return t; }
private void reloadAllSourceTableExd() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading SourceTable exd info from folder " + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT)); srcTableExdMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { reloadSourceTableExdAt(path); } logger.debug("Loaded " + srcTableExdMap.size() + " SourceTable EXD(s)"); }
private void reloadAllSourceTable() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading SourceTable from folder " + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT)); srcTableMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { reloadSourceTableAt(path); } logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)"); }
private void save(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.putResource(path, snapshot, SnapshotTableSerializer.FULL_SERIALIZER); }
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); ArrayList<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; for (String existing : existings) { logger.info("Checking dup dict :" + existing); DictionaryInfo existingInfo = load(existing, true); // skip cache, // direct load // from store if (existingInfo == null) logger.info("existingInfo is null"); if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject())) return existing; } return null; }
private void reloadAllDataModel() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading DataModel from folder " + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT)); dataModelDescMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { try { reloadDataModelDescAt(path); } catch (IllegalStateException e) { logger.error("Error to load DataModel at " + path, e); continue; } } logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)"); }
private DataModelDesc reloadDataModelDescAt(String path) { ResourceStore store = getStore(); try { DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER); dataModelDesc.init(this.getAllTablesMap()); dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc); return dataModelDesc; } catch (IOException e) { throw new IllegalStateException("Error to load" + path, e); } }
public void saveSourceTable(TableDesc srcTable) throws IOException { if (srcTable.getUuid() == null || srcTable.getIdentity() == null) { throw new IllegalArgumentException(); } srcTable.init(); String path = srcTable.getResourcePath(); getStore().putResource(path, srcTable, TABLE_SERIALIZER); srcTableMap.put(srcTable.getIdentity(), srcTable); }
private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; for (String existing : existings) { SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store if (existingTable != null && existingTable.equals(snapshot)) return existing; } return null; }
private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException { dataModelDesc.init(this.getAllTablesMap()); String path = dataModelDesc.getResourcePath(); getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER); dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc); return dataModelDesc; }
public void removeDictionaries(String srcTable, String srcCol) throws IOException { DictionaryInfo info = new DictionaryInfo(); info.setSourceTable(srcTable); info.setSourceColumn(srcCol); ResourceStore store = MetadataManager.getInstance(config).getStore(); ArrayList<String> existings = store.listResources(info.getResourceDir()); if (existings == null) return; for (String existing : existings) removeDictionary(existing); }
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); ArrayList<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; TableSignature input = dictInfo.getInput(); for (String existing : existings) { DictionaryInfo existingInfo = load(existing, false); // skip cache, // direct // load from // store if (input.equals(existingInfo.getInput())) return existing; } return null; }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct // load from // store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }