public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException { String dupTable = checkDupByContent(snapshotTable); if (dupTable != null) { logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable); return getSnapshotTable(dupTable); } save(snapshotTable); snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable); return snapshotTable; }
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join, SnapshotTableDesc snapshotTableDesc) { String tableName = join.getPKSide().getTableIdentity(); String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); String[] pkCols = join.getPrimaryKey(); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException( "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
public LookupTable<String> initLookupTable() throws Exception { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[] { "CAL_DT" }; String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName, "default"); LookupTable<String> lt = new LookupStringTable(tableDesc, pkCols, snapshot); System.out.println(lt); return lt; }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.setUuid(overwriteUUID); snapshot.takeSnapshot(table, tableDesc); SnapshotTable existing = getSnapshotTable(snapshot.getResourcePath()); snapshot.setLastModified(existing.getLastModified()); save(snapshot); snapshotCache.put(snapshot.getResourcePath(), snapshot); return snapshot; }
SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig); SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig); SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig); SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig); SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException { String dupTable = checkDupByContent(snapshotTable); if (dupTable != null) { logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable); return getSnapshotTable(dupTable); } save(snapshotTable); snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable); return snapshotTable; }
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { SnapshotTable snapshot = new SnapshotTable(table); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { String tableName = dim.getTable(); String[] pkCols = dim.getJoin().getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName); return new LookupStringTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
public LookupTable initLookupTable() throws Exception { MetadataManager metaMgr = MetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[]{"CAL_DT"}; String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName); LookupTable lt = new LookupStringTable(tableDesc, pkCols, snapshot); System.out.println(lt); return lt; }
SnapshotManager dstSnapMgr = SnapshotManager.getInstance(dstConfig); SnapshotManager srcSnapMgr = SnapshotManager.getInstance(srcConfig); SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item);
public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException { String dupTable = checkDupByContent(snapshotTable); if (dupTable != null) { logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable); return getSnapshotTable(dupTable); } save(snapshotTable); snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable); return snapshotTable; }
public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException { String dupTable = checkDupByContent(snapshotTable); if (dupTable != null) { logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable); return getSnapshotTable(dupTable); } save(snapshotTable); snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable); return snapshotTable; }
@Test public void basicTest() throws Exception { String tableName = "EDW.TEST_SITES"; HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName); TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName); String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath(); snapshotMgr.wipeoutCache(); SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath); // compare hive & snapshot TableReader hiveReader = hiveTable.getReader(); TableReader snapshotReader = snapshot.getReader(); while (true) { boolean hiveNext = hiveReader.next(); boolean snapshotNext = snapshotReader.next(); assertEquals(hiveNext, snapshotNext); if (hiveNext == false) break; String[] hiveRow = hiveReader.getRow(); String[] snapshotRow = snapshotReader.getRow(); assertArrayEquals(hiveRow, snapshotRow); } } }
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { String tableName = dim.getTable(); String[] pkCols = dim.getJoin().getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName); return new LookupStringTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join, SnapshotTableDesc snapshotTableDesc) { String tableName = join.getPKSide().getTableIdentity(); String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); String[] pkCols = join.getPrimaryKey(); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException( "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { SnapshotTable snapshot = new SnapshotTable(table); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if (snapshot.getSignature().getSize() / 1024 / 1024 > config.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + config.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.setUuid(overwriteUUID); snapshot.takeSnapshot(table, tableDesc); SnapshotTable existing = getSnapshotTable(snapshot.getResourcePath()); snapshot.setLastModified(existing.getLastModified()); save(snapshot); snapshotCache.put(snapshot.getResourcePath(), snapshot); return snapshot; }