public ExtTableSnapshotInfo getSnapshot(String tableName, String snapshotID) { return getSnapshot(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); }
public void removeSnapshotLocalCache(String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } LookupProviderFactory.removeLocalCache(extTableSnapshotInfo); }
public String getSnapshotLocalCacheState(String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } CacheState cacheState = LookupProviderFactory.getCacheState(extTableSnapshotInfo); return cacheState.name(); }
private static List<String> getAllUsedExtLookupTables() throws IOException { List<String> result = Lists.newArrayList(); KylinConfig config = KylinConfig.getInstanceFromEnv(); final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths(); for (String extSnapshotResource : activeSnapshotSet) { try { ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( extSnapshotResource); if (extTableSnapshot != null) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(extTableSnapshot.getStorageType())) { result.add(extTableSnapshot.getStorageLocationIdentifier()); } } } catch (Exception e) { logger.error("error fetch ext table snapshot:" + extSnapshotResource, e); } } return result; }
private void removeSnapshotIfExist(ExtTableSnapshotInfoManager extSnapshotInfoManager, KylinConfig kylinConfig, String tableName, String lookupSnapshotID) throws IOException { ExtTableSnapshotInfo snapshotInfo = null; try { snapshotInfo = extSnapshotInfoManager.getSnapshot(tableName, lookupSnapshotID); } catch (Exception e) { // swallow the exception, means not snapshot exist of this snapshot id } if (snapshotInfo == null) { return; } logger.info("the table:{} snapshot:{} exist, remove it", tableName, lookupSnapshotID); extSnapshotInfoManager.removeSnapshot(tableName, lookupSnapshotID); String hTableName = snapshotInfo.getStorageLocationIdentifier(); logger.info("remove related HBase table:{} for snapshot:{}", hTableName, lookupSnapshotID); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin(); admin.deleteTable(TableName.valueOf(hTableName)); }
public void updateSnapshotLocalCache(String project, String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } LookupProviderFactory.rebuildLocalCache(tableDesc, extTableSnapshotInfo); }
private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot); }
ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID); long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1")); logger.info("update table:{} snapshot row count:{}", tableName, srcTableRowCnt);
public ExtTableSnapshotInfo getSnapshot(String tableName, String snapshotID) { return getSnapshot(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); }
private void removeSnapshotIfExist(ExtTableSnapshotInfoManager extSnapshotInfoManager, KylinConfig kylinConfig, String tableName, String lookupSnapshotID) throws IOException { ExtTableSnapshotInfo snapshotInfo = null; try { snapshotInfo = extSnapshotInfoManager.getSnapshot(tableName, lookupSnapshotID); } catch (Exception e) { // swallow the exception, means not snapshot exist of this snapshot id } if (snapshotInfo == null) { return; } logger.info("the table:{} snapshot:{} exist, remove it", tableName, lookupSnapshotID); extSnapshotInfoManager.removeSnapshot(tableName, lookupSnapshotID); String hTableName = snapshotInfo.getStorageLocationIdentifier(); logger.info("remove related HBase table:{} for snapshot:{}", hTableName, lookupSnapshotID); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin(); admin.deleteTable(TableName.valueOf(hTableName)); }
private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot); }
ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID); long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1")); logger.info("update table:{} snapshot row count:{}", tableName, srcTableRowCnt);