@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
totalSize += info.getInput().getSize(); signature.setSize(totalSize); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath("merged_with_no_original_path");
public static String getOldResourceDir(TableSignature signature) { return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName(); }
totalSize += info.getInput().getSize(); signature.setSize(totalSize); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath("merged_with_no_original_path");
KylinConfig kylinConfig) throws IOException { TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin();
@Override public void buildSnapshotCache(final TableDesc tableDesc, final ExtTableSnapshotInfo extTableSnapshotInfo, final ILookupTable sourceTable) { if (extTableSnapshotInfo.getSignature().getSize() / 1024 > maxCacheSizeInKB * 2 / 3) { logger.warn("the size is to large to build to cache for snapshot:{}, size:{}, skip cache building", extTableSnapshotInfo.getResourcePath(), extTableSnapshotInfo.getSignature().getSize()); return;
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = DFSFileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (this.hiveTableMeta.isNative == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public static String getOldResourceDir(TableSignature signature) { return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName(); }
@Override public TableSignature getSignature() throws IOException { Pair<Long, Long> sizeAndLastModified; try { sizeAndLastModified = getSizeAndLastModified(path); } catch (FileNotFoundException ex) { sizeAndLastModified = Pair.newPair(-1L, 0L); } return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond()); }
KylinConfig kylinConfig) throws IOException { TableSignature signature = sourceTable.getSignature(); int shardNum = calculateShardNum(kylinConfig, signature.getSize()); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin();
@Override public void buildSnapshotCache(final TableDesc tableDesc, final ExtTableSnapshotInfo extTableSnapshotInfo, final ILookupTable sourceTable) { if (extTableSnapshotInfo.getSignature().getSize() / 1024 > maxCacheSizeInKB * 2 / 3) { logger.warn("the size is to large to build to cache for snapshot:{}, size:{}, skip cache building", extTableSnapshotInfo.getResourcePath(), extTableSnapshotInfo.getSignature().getSize()); return;
response.setLastBuildTime(extTableSnapshot.getLastBuildTime()); response.setStorageType(extTableSnapshot.getStorageType()); response.setSourceTableSize(extTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(extTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(extTableSnapshot.getResourcePath())); result.add(response); response.setLastBuildTime(metaStoreTableSnapshot.getLastBuildTime()); response.setStorageType(SnapshotTable.STORAGE_TYPE_METASTORE); response.setSourceTableSize(metaStoreTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(metaStoreTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(metaStoreTableSnapshot.getResourcePath())); result.add(response);
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = DFSFileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (this.hiveTableMeta.isNative == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
private ExtTableSnapshotInfo buildSnapshotCache(String snapshotID, int rowCnt) throws Exception { ExtTableSnapshotInfo snapshotInfo = new ExtTableSnapshotInfo(); snapshotInfo.setTableName(TABLE_COUNTRY); snapshotInfo.setUuid(snapshotID); snapshotInfo.setStorageType(MOCK_EXT_LOOKUP); snapshotInfo.setKeyColumns(new String[] { "COUNTRY" }); snapshotInfo.setRowCnt(rowCnt); snapshotInfo.setSignature(new TableSignature("/test", rowCnt, System.currentTimeMillis())); ExtTableSnapshotInfoManager.getInstance(kylinConfig).save(snapshotInfo); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); cache.buildSnapshotCache(tableDesc, snapshotInfo, getLookupTableWithRandomData(rowCnt)); while (cache.getCacheState(snapshotInfo) == CacheState.IN_BUILDING) { Thread.sleep(500); } return snapshotInfo; }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public static IReadableTable newNonExistTable(String path) { TableSignature sig = new TableSignature(path, -1, 0); return new MockupReadableTable(null, sig, false); }
@Override public TableSignature getSignature() throws IOException { String path = String.format(Locale.ROOT, "%s.%s", database, tableName); long lastModified = System.currentTimeMillis(); // assume table is ever changing int size = 0; return new TableSignature(path, size, lastModified); }