public static CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo) { IExtLookupTableCache tablesCache = getExtLookupProvider(extTableSnapshotInfo.getStorageType()).getLocalCache(); if (tablesCache != null) { return tablesCache.getCacheState(extTableSnapshotInfo); } return CacheState.NONE; }
public static void removeLocalCache(ExtTableSnapshotInfo extTableSnapshotInfo) { IExtLookupTableCache tablesCache = getExtLookupProvider(extTableSnapshotInfo.getStorageType()).getLocalCache(); if (tablesCache != null) { tablesCache.removeSnapshotCache(extTableSnapshotInfo); } }
public static void rebuildLocalCache(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo) { IExtLookupTableCache tablesCache = getExtLookupProvider(extTableSnapshotInfo.getStorageType()).getLocalCache(); if (tablesCache != null) { tablesCache.buildSnapshotCache(tableDesc, extTableSnapshotInfo, getExtLookupTableWithoutCache(tableDesc, extTableSnapshotInfo)); } }
public String getSnapshotLocalCacheState(String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } CacheState cacheState = LookupProviderFactory.getCacheState(extTableSnapshotInfo); return cacheState.name(); }
public LookupTable<String> initLookupTable() throws Exception { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[] { "CAL_DT" }; String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName, "default"); LookupTable<String> lt = new LookupStringTable(tableDesc, pkCols, snapshot); System.out.println(lt); return lt; }
private SnapshotManager getSnapshotManager() { return SnapshotManager.getInstance(config); }
public static ILookupTable getExtLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupTableCache extLookupTableCache = getExtLookupProvider(extTableSnapshot.getStorageType()).getLocalCache(); if (extLookupTableCache == null) { return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); } ILookupTable cachedLookupTable = extLookupTableCache.getCachedLookupTable(tableDesc, extTableSnapshot, true); if (cachedLookupTable != null) { logger.info("try to use cached lookup table:{}", extTableSnapshot.getResourcePath()); return cachedLookupTable; } logger.info("use ext lookup table:{}", extTableSnapshot.getResourcePath()); return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); }
public ExtTableSnapshotInfo getSnapshot(String tableName, String snapshotID) { return getSnapshot(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); }
private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; for (String existing : existings) { SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store if (existingTable != null && existingTable.equals(snapshot)) return existing; } return null; }
public void updateSnapshot(ExtTableSnapshotInfo extTableSnapshot) throws IOException { save(extTableSnapshot); snapshotCache.invalidate(extTableSnapshot.getResourcePath()); }
public static ILookupTable getExtLookupTableWithoutCache(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupProvider provider = getExtLookupProvider(extTableSnapshot.getStorageType()); return provider.getLookupTable(tableDesc, extTableSnapshot); }
/** * * @param signature source table signature * @param tableName * @return latest snapshot info * @throws IOException */ public ExtTableSnapshotInfo getLatestSnapshot(TableSignature signature, String tableName) throws IOException { ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo(signature, tableName); snapshot.updateRandomUuid(); ExtTableSnapshotInfo dupSnapshot = checkDupByInfo(snapshot); return dupSnapshot; }
public static ILookupTable getInMemLookupTable(TableDesc tableDesc, String[] pkCols, IReadableTable readableTable) throws IOException { return new LookupStringTable(tableDesc, pkCols, readableTable); }
public static <T> T createEngineAdapter(String lookupStorageType, Class<T> engineInterface) { IExtLookupProvider provider = getExtLookupProvider(lookupStorageType); return provider.adaptToBuildEngine(engineInterface); }
public static String getResourcePath(String tableName, String uuid) { return getResourceDir(tableName) + "/" + uuid + ".snapshot"; }
@Override public void reset() { this.iterator = lookupTable.iterator(); }
public static ILookupMaterializer getExtLookupMaterializer(String lookupStorageType) { return LookupProviderFactory.createEngineAdapter(lookupStorageType, ILookupMaterializer.class); }
public String[] getValueData(String[] row) { return extractColValues(row, valueIndexes); }
@Override public ILookupTable getLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { return getLookupTableWithRandomData(extTableSnapshot.getRowCnt()); }
public String getResourceDir() { return getResourceDir(tableName); }