SnapshotTable(IReadableTable table, String tableName) throws IOException { this.tableName = tableName; this.signature = table.getSignature(); this.useDictionary = true; }
private ExtTableSnapshotInfo createSnapshotResource(ExtTableSnapshotInfoManager extSnapshotInfoManager, String tableName, String snapshotID, String[] keyColumns, String hTableName, int shardNum, IReadableTable sourceTable) throws IOException { return extSnapshotInfoManager.createSnapshot(sourceTable.getSignature(), tableName, snapshotID, keyColumns, shardNum, ExtTableSnapshotInfo.STORAGE_TYPE_HBASE, hTableName); }
private DictionaryInfo createDictionaryInfo(TblColRef col, IReadableTable inpTable) throws IOException { TableSignature inputSig = inpTable.getSignature(); if (inputSig == null) // table does not exists throw new IllegalStateException("Input table does not exist: " + inpTable); DictionaryInfo dictInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype(), inputSig); return dictInfo; }
public List<TableSnapshotResponse> getLookupTableSnapshots(String project, String tableName) throws IOException { TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); TableSignature signature = hiveTable.getSignature(); return internalGetLookupTableSnapshots(tableName, signature); }
TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig);
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public void takeSnapshot(IReadableTable table, TableDesc tableDesc) throws IOException { this.signature = table.getSignature();
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
SnapshotTable(IReadableTable table, String tableName) throws IOException { this.tableName = tableName; this.signature = table.getSignature(); this.useDictionary = true; }
private ExtTableSnapshotInfo createSnapshotResource(ExtTableSnapshotInfoManager extSnapshotInfoManager, String tableName, String snapshotID, String[] keyColumns, String hTableName, int shardNum, IReadableTable sourceTable) throws IOException { return extSnapshotInfoManager.createSnapshot(sourceTable.getSignature(), tableName, snapshotID, keyColumns, shardNum, ExtTableSnapshotInfo.STORAGE_TYPE_HBASE, hTableName); }
private DictionaryInfo createDictionaryInfo(TblColRef col, IReadableTable inpTable) throws IOException { TableSignature inputSig = inpTable.getSignature(); if (inputSig == null) // table does not exists throw new IllegalStateException("Input table does not exist: " + inpTable); DictionaryInfo dictInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype(), inputSig); return dictInfo; }
TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig);
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public void takeSnapshot(IReadableTable table, TableDesc tableDesc) throws IOException { this.signature = table.getSignature();
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }