@Override public CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { return CacheState.IN_BUILDING; } File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); if (!stateFile.exists()) { return CacheState.NONE; } try { String stateString = Files.toString(stateFile, Charsets.UTF_8); return CacheState.valueOf(stateString); } catch (IOException e) { logger.error("error when read state file", e); } return CacheState.NONE; }
private void saveSnapshotCacheState(ExtTableSnapshotInfo extTableSnapshotInfo, String cachePath) { File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); try { Files.write(CacheState.AVAILABLE.name(), stateFile, Charsets.UTF_8); tablesCache.put(extTableSnapshotInfo.getResourcePath(), new CachedTableInfo(cachePath)); } catch (IOException e) { throw new RuntimeException("error when write cache state for snapshot:" + extTableSnapshotInfo.getResourcePath()); } }
@Override public ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { logger.info("cache is in building for snapshot:" + resourcePath); return null; } CachedTableInfo cachedTableInfo = tablesCache.getIfPresent(resourcePath); if (cachedTableInfo == null) { if (buildIfNotExist) { buildSnapshotCache(tableDesc, extTableSnapshotInfo, getSourceLookupTable(tableDesc, extTableSnapshotInfo)); } logger.info("no available cache ready for the table snapshot:" + extTableSnapshotInfo.getResourcePath()); return null; } String[] keyColumns = extTableSnapshotInfo.getKeyColumns(); String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); return new RocksDBLookupTable(tableDesc, keyColumns, dbPath); }
@Test public void testCheckCacheState() throws Exception { ExtTableSnapshotInfo snapshotInfo = buildSnapshotCache(RandomUtil.randomUUID().toString(), 1000); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); ILookupTable cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNotNull(cachedLookupTable); cachedLookupTable.close(); ExtTableSnapshotInfoManager.getInstance(kylinConfig).removeSnapshot(snapshotInfo.getTableName(), snapshotInfo.getId()); cache.checkCacheState(); String cacheLocalPath = cache.getSnapshotCachePath(snapshotInfo.getTableName(), snapshotInfo.getId()); // won't cleanup because it is newly created in last 1 hour assertTrue(new File(cacheLocalPath).exists()); // change the volatile value kylinConfig.setProperty("kylin.snapshot.ext.local.cache.check.volatile", "0"); cache.checkCacheState(); // this time it should be removed. assertFalse(new File(cacheLocalPath).exists()); cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNull(cachedLookupTable); }
for (ExtTableSnapshotInfo extTableSnapshot : extTableSnapshots) { TableSnapshotResponse response = new TableSnapshotResponse(); response.setSnapshotID(extTableSnapshot.getId()); response.setSnapshotType(TableSnapshotResponse.TYPE_EXT); response.setLastBuildTime(extTableSnapshot.getLastBuildTime());
@Override public CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { return CacheState.IN_BUILDING; } File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); if (!stateFile.exists()) { return CacheState.NONE; } try { String stateString = Files.toString(stateFile, Charsets.UTF_8); return CacheState.valueOf(stateString); } catch (IOException e) { logger.error("error when read state file", e); } return CacheState.NONE; }
private void saveSnapshotCacheState(ExtTableSnapshotInfo extTableSnapshotInfo, String cachePath) { File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); try { Files.write(CacheState.AVAILABLE.name(), stateFile, Charsets.UTF_8); tablesCache.put(extTableSnapshotInfo.getResourcePath(), new CachedTableInfo(cachePath)); } catch (IOException e) { throw new RuntimeException("error when write cache state for snapshot:" + extTableSnapshotInfo.getResourcePath()); } }
@Override public ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { logger.info("cache is in building for snapshot:" + resourcePath); return null; } CachedTableInfo cachedTableInfo = tablesCache.getIfPresent(resourcePath); if (cachedTableInfo == null) { if (buildIfNotExist) { buildSnapshotCache(tableDesc, extTableSnapshotInfo, getSourceLookupTable(tableDesc, extTableSnapshotInfo)); } logger.info("no available cache ready for the table snapshot:" + extTableSnapshotInfo.getResourcePath()); return null; } String[] keyColumns = extTableSnapshotInfo.getKeyColumns(); String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); return new RocksDBLookupTable(tableDesc, keyColumns, dbPath); }