private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct load from store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
response.setLastBuildTime(metaStoreTableSnapshot.getLastBuildTime()); response.setStorageType(SnapshotTable.STORAGE_TYPE_METASTORE); response.setSourceTableSize(metaStoreTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(metaStoreTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(metaStoreTableSnapshot.getResourcePath())); result.add(response);
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct // load from // store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct load from store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct load from store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { SnapshotTable snapshot = new SnapshotTable(table); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if (snapshot.getSignature().getSize() / 1024 / 1024 > config.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + config.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }