private void reloadAllIIDesc() throws IOException { ResourceStore store = getStore(); logger.info("Reloading all II desc from folder " + store.getReadableResourcePath(ResourceStore.II_DESC_RESOURCE_ROOT)); iiDescMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.II_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { logger.info("loading II Desc from path" + path); IIDesc desc; try { desc = loadIIDesc(path); } catch (Exception e) { logger.error("Error loading II desc " + path, e); continue; } if (!path.equals(desc.getResourcePath())) { logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); continue; } if (iiDescMap.containsKey(desc.getName())) { logger.error("Dup IIDesc name '" + desc.getName() + "' on path " + path); continue; } iiDescMap.putLocal(desc.getName(), desc); } logger.debug("Loaded " + iiDescMap.size() + " II desc(s)"); }
public DataModelDesc getDataModelDesc(String name) { return dataModelDescMap.get(name); }
private void afterIIUpdated(IIInstance updatedII) { iiMap.put(updatedII.getName(), updatedII); }
/** * Get table extended info. Keys are defined in {@link MetadataConstants} * * @param tableName * @return */ public Map<String, String> getTableDescExd(String tableName) { String tableIdentity = tableName; Map<String, String> result = new HashMap<String, String>(); if (srcTableExdMap.containsKey(tableIdentity)) { Map<String, String> tmp = srcTableExdMap.get(tableIdentity); Iterator<Entry<String, String>> it = tmp.entrySet().iterator(); while (it.hasNext()) { Entry<String, String> entry = it.next(); result.put(entry.getKey(), entry.getValue()); } result.put(MetadataConstants.TABLE_EXD_STATUS_KEY, "true"); } else { result.put(MetadataConstants.TABLE_EXD_STATUS_KEY, "false"); } return result; }
private void reloadAllSourceTableExd() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading SourceTable exd info from folder " + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT)); srcTableExdMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { reloadSourceTableExdAt(path); } logger.debug("Loaded " + srcTableExdMap.size() + " SourceTable EXD(s)"); }
public void saveTableExd(String tableId, Map<String, String> tableExdProperties) throws IOException { if (tableId == null) { throw new IllegalArgumentException("tableId couldn't be null"); } TableDesc srcTable = srcTableMap.get(tableId); if (srcTable == null) { throw new IllegalArgumentException("Couldn't find Source Table with identifier: " + tableId); } String path = TableDesc.concatExdResourcePath(tableId); ByteArrayOutputStream os = new ByteArrayOutputStream(); JsonUtil.writeValueIndent(os, tableExdProperties); os.flush(); InputStream is = new ByteArrayInputStream(os.toByteArray()); getStore().putResource(path, is, System.currentTimeMillis()); os.close(); is.close(); srcTableExdMap.putLocal(tableId, tableExdProperties); }
/** * Create a new IIDesc * * @param iiDesc * @return * @throws IOException */ public IIDesc createIIDesc(IIDesc iiDesc) throws IOException { if (iiDesc.getUuid() == null || iiDesc.getName() == null) throw new IllegalArgumentException(); if (iiDescMap.containsKey(iiDesc.getName())) throw new IllegalArgumentException("IIDesc '" + iiDesc.getName() + "' already exists"); iiDesc.init(getMetadataManager()); // Check base validation // Semantic validation // TODO iiDesc.setSignature(iiDesc.calculateSignature()); String path = iiDesc.getResourcePath(); getStore().putResource(path, iiDesc, II_DESC_SERIALIZER); iiDescMap.put(iiDesc.getName(), iiDesc); return iiDesc; }
public void removeIIDescLocal(String name) throws IOException { iiDescMap.remove(name); }
@SuppressWarnings("unchecked") private Map<String, String> reloadSourceTableExdAt(String path) throws IOException { Map<String, String> attrs = Maps.newHashMap(); ResourceStore store = getStore(); InputStream is = store.getResource(path); if (is == null) { logger.warn("Failed to get table exd info from " + path); return null; } try { attrs.putAll(JsonUtil.readValue(is, HashMap.class)); } finally { if (is != null) is.close(); } // parse table identity from file name String file = path; if (file.indexOf("/") > -1) { file = file.substring(file.lastIndexOf("/") + 1); } String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase(); srcTableExdMap.putLocal(tableIdentity, attrs); return attrs; }
public List<ProjectInstance> listAllProjects() { return new ArrayList<ProjectInstance>(projectMap.values()); }
private void reloadAllProjects() throws IOException { ResourceStore store = getStore(); List<String> paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); for (String path : paths) { reloadProjectAt(path); } wireProjectAndRealizations(projectMap.values()); logger.debug("Loaded " + projectMap.size() + " Project(s)"); }
public void removeIILocalCache(String name) { iiMap.removeLocal(name); //TODO // for (IISegment segment : ii.getSegments()) { // usedStorageLocation.remove(segment.getName()); // } }
public DataModelDesc createDataModelDesc(DataModelDesc dataModelDesc) throws IOException { String name = dataModelDesc.getName(); if (dataModelDescMap.containsKey(name)) throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists"); return saveDataModelDesc(dataModelDesc); }
private IIManager(KylinConfig config) throws IOException { logger.info("Initializing IIManager with config " + config); this.config = config; this.iiMap = new CaseInsensitiveStringCache<IIInstance>(config, Broadcaster.TYPE.INVERTED_INDEX); loadAllIIInstance(); }
private void reloadAllSourceTable() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading SourceTable from folder " + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT)); srcTableMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { reloadSourceTableAt(path); } logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)"); }
/** * Update IIDesc with the input. Broadcast the event into cluster * * @param desc * @return * @throws IOException */ public IIDesc updateIIDesc(IIDesc desc) throws IOException { // Validate IIDesc if (desc.getUuid() == null || desc.getName() == null) { throw new IllegalArgumentException(); } String name = desc.getName(); if (!iiDescMap.containsKey(name)) { throw new IllegalArgumentException("IIDesc '" + name + "' does not exist."); } desc.init(getMetadataManager()); // TODO: Semantic validation desc.setSignature(desc.calculateSignature()); // Save Source String path = desc.getResourcePath(); getStore().putResource(path, desc, II_DESC_SERIALIZER); // Reload the IIDesc IIDesc ndesc = loadIIDesc(path); // Here replace the old one iiDescMap.put(ndesc.getName(), desc); return ndesc; }
/** * After cube deletion, remove cube related cache * * @param cube */ public void removeCubeCache(CubeInstance cube) { final String cubeName = cube.getName().toUpperCase(); cubeMap.remove(cubeName); usedStorageLocation.removeAll(cubeName); }
private TableDesc reloadSourceTableAt(String path) throws IOException { ResourceStore store = getStore(); TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER); if (t == null) { logger.error("Didn't load table at " + path); return null; } t.init(); String tableIdentity = t.getIdentity(); srcTableMap.putLocal(tableIdentity, t); return t; }
public List<CubeInstance> listAllCubes() { return new ArrayList<CubeInstance>(cubeMap.values()); }
private void reloadAllProjects() throws IOException { ResourceStore store = getStore(); List<String> paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); for (String path : paths) { reloadProjectAt(path); } wireProjectAndRealizations(projectMap.values()); logger.debug("Loaded " + projectMap.size() + " Project(s)"); }