totalSize += info.getInput().getSize();
@Override public void buildSnapshotCache(final TableDesc tableDesc, final ExtTableSnapshotInfo extTableSnapshotInfo, final ILookupTable sourceTable) { if (extTableSnapshotInfo.getSignature().getSize() / 1024 > maxCacheSizeInKB * 2 / 3) { logger.warn("the size is to large to build to cache for snapshot:{}, size:{}, skip cache building", extTableSnapshotInfo.getResourcePath(), extTableSnapshotInfo.getSignature().getSize()); return;
KylinConfig kylinConfig) throws IOException { TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin();
totalSize += info.getInput().getSize();
KylinConfig kylinConfig) throws IOException { TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin();
@Override public void buildSnapshotCache(final TableDesc tableDesc, final ExtTableSnapshotInfo extTableSnapshotInfo, final ILookupTable sourceTable) { if (extTableSnapshotInfo.getSignature().getSize() / 1024 > maxCacheSizeInKB * 2 / 3) { logger.warn("the size is to large to build to cache for snapshot:{}, size:{}, skip cache building", extTableSnapshotInfo.getResourcePath(), extTableSnapshotInfo.getSignature().getSize()); return;
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
response.setLastBuildTime(extTableSnapshot.getLastBuildTime()); response.setStorageType(extTableSnapshot.getStorageType()); response.setSourceTableSize(extTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(extTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(extTableSnapshot.getResourcePath())); response.setLastBuildTime(metaStoreTableSnapshot.getLastBuildTime()); response.setStorageType(SnapshotTable.STORAGE_TYPE_METASTORE); response.setSourceTableSize(metaStoreTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(metaStoreTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(metaStoreTableSnapshot.getResourcePath()));
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }