public String getResourcePath() { return getResourcePath(tableName, uuid); }
@Override public void removeSnapshotCache(ExtTableSnapshotInfo extTableSnapshotInfo) { tablesCache.invalidate(extTableSnapshotInfo.getResourcePath()); }
public void updateSnapshot(ExtTableSnapshotInfo extTableSnapshot) throws IOException { save(extTableSnapshot); snapshotCache.invalidate(extTableSnapshot.getResourcePath()); }
public ExtTableSnapshotInfo getSnapshot(String tableName, String snapshotID) { return getSnapshot(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); }
private void initSnapshotState(String tableName, File snapshotCacheFolder) { String snapshotID = snapshotCacheFolder.getName(); File stateFile = getCacheStateFile(snapshotCacheFolder.getAbsolutePath()); if (stateFile.exists()) { try { String stateStr = Files.toString(stateFile, Charsets.UTF_8); String resourcePath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); if (CacheState.AVAILABLE.name().equals(stateStr)) { tablesCache.put(resourcePath, new CachedTableInfo(snapshotCacheFolder.getAbsolutePath())); } } catch (IOException e) { logger.error("error to read state file:" + stateFile.getAbsolutePath()); } } }
private void saveSnapshotCacheState(ExtTableSnapshotInfo extTableSnapshotInfo, String cachePath) { File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); try { Files.write(CacheState.AVAILABLE.name(), stateFile, Charsets.UTF_8); tablesCache.put(extTableSnapshotInfo.getResourcePath(), new CachedTableInfo(cachePath)); } catch (IOException e) { throw new RuntimeException("error when write cache state for snapshot:" + extTableSnapshotInfo.getResourcePath()); } }
public void removeSnapshot(String tableName, String snapshotID) throws IOException { String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); snapshotCache.invalidate(snapshotResPath); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); store.deleteResource(snapshotResPath); }
public static ILookupTable getExtLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupTableCache extLookupTableCache = getExtLookupProvider(extTableSnapshot.getStorageType()).getLocalCache(); if (extLookupTableCache == null) { return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); } ILookupTable cachedLookupTable = extLookupTableCache.getCachedLookupTable(tableDesc, extTableSnapshot, true); if (cachedLookupTable != null) { logger.info("try to use cached lookup table:{}", extTableSnapshot.getResourcePath()); return cachedLookupTable; } logger.info("use ext lookup table:{}", extTableSnapshot.getResourcePath()); return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); }
@Override public ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { logger.info("cache is in building for snapshot:" + resourcePath); return null; } CachedTableInfo cachedTableInfo = tablesCache.getIfPresent(resourcePath); if (cachedTableInfo == null) { if (buildIfNotExist) { buildSnapshotCache(tableDesc, extTableSnapshotInfo, getSourceLookupTable(tableDesc, extTableSnapshotInfo)); } logger.info("no available cache ready for the table snapshot:" + extTableSnapshotInfo.getResourcePath()); return null; } String[] keyColumns = extTableSnapshotInfo.getKeyColumns(); String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); return new RocksDBLookupTable(tableDesc, keyColumns, dbPath); }
@Override public CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { return CacheState.IN_BUILDING; } File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); if (!stateFile.exists()) { return CacheState.NONE; } try { String stateString = Files.toString(stateFile, Charsets.UTF_8); return CacheState.valueOf(stateString); } catch (IOException e) { logger.error("error when read state file", e); } return CacheState.NONE; }
public void save(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.checkAndPutResource(path, snapshot, SNAPSHOT_SERIALIZER); }
String snapshotID = file.getName(); allCachedSnapshots .add(new Pair<>(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID), file));
if (extTableSnapshotInfo.getSignature().getSize() / 1024 > maxCacheSizeInKB * 2 / 3) { logger.warn("the size is to large to build to cache for snapshot:{}, size:{}, skip cache building", extTableSnapshotInfo.getResourcePath(), extTableSnapshotInfo.getSignature().getSize()); return; final String cachePath = getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); final String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); final String snapshotResPath = extTableSnapshotInfo.getResourcePath();
ExtTableSnapshotInfo snapshot = createSnapshotResource(extSnapshotInfoManager, tableName, lookupSnapshotID, keyColumns, hTableNameAndShard.getFirst(), hTableNameAndShard.getSecond(), sourceTable); logger.info("created snapshot information at:{}", snapshot.getResourcePath());
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
response.setSourceTableSize(extTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(extTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(extTableSnapshot.getResourcePath())); result.add(response);
@Override public void removeSnapshotCache(ExtTableSnapshotInfo extTableSnapshotInfo) { tablesCache.invalidate(extTableSnapshotInfo.getResourcePath()); }
private void saveSnapshotCacheState(ExtTableSnapshotInfo extTableSnapshotInfo, String cachePath) { File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); try { Files.write(CacheState.AVAILABLE.name(), stateFile, Charsets.UTF_8); tablesCache.put(extTableSnapshotInfo.getResourcePath(), new CachedTableInfo(cachePath)); } catch (IOException e) { throw new RuntimeException("error when write cache state for snapshot:" + extTableSnapshotInfo.getResourcePath()); } }
public void removeSnapshot(String tableName, String snapshotID) throws IOException { String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); snapshotCache.invalidate(snapshotResPath); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); store.deleteResource(snapshotResPath); }
public void save(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.checkAndPutResource(path, snapshot, SNAPSHOT_SERIALIZER); }