@Override public TableSignature getSignature() throws IOException { String path = String.format(Locale.ROOT, "%s.%s", database, tableName); long lastModified = System.currentTimeMillis(); // assume table is ever changing int size = 0; return new TableSignature(path, size, lastModified); }
@Override public TableSignature getSignature() throws IOException { String path = String.format(Locale.ROOT, "%s.%s", database, tableName); long lastModified = System.currentTimeMillis(); // assume table is ever changing int size = 0; return new TableSignature(path, size, lastModified); }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = DFSFileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (this.hiveTableMeta.isNative == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
@Override public TableSignature getSignature() throws IOException { Pair<Long, Long> sizeAndLastModified; try { sizeAndLastModified = getSizeAndLastModified(path); } catch (FileNotFoundException ex) { sizeAndLastModified = Pair.newPair(-1L, 0L); } return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond()); }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = DFSFileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (this.hiveTableMeta.isNative == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
private ExtTableSnapshotInfo buildSnapshotCache(String snapshotID, int rowCnt) throws Exception { ExtTableSnapshotInfo snapshotInfo = new ExtTableSnapshotInfo(); snapshotInfo.setTableName(TABLE_COUNTRY); snapshotInfo.setUuid(snapshotID); snapshotInfo.setStorageType(MOCK_EXT_LOOKUP); snapshotInfo.setKeyColumns(new String[] { "COUNTRY" }); snapshotInfo.setRowCnt(rowCnt); snapshotInfo.setSignature(new TableSignature("/test", rowCnt, System.currentTimeMillis())); ExtTableSnapshotInfoManager.getInstance(kylinConfig).save(snapshotInfo); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); cache.buildSnapshotCache(tableDesc, snapshotInfo, getLookupTableWithRandomData(rowCnt)); while (cache.getCacheState(snapshotInfo) == CacheState.IN_BUILDING) { Thread.sleep(500); } return snapshotInfo; }
public static IReadableTable newNonExistTable(String path) { TableSignature sig = new TableSignature(path, -1, 0); return new MockupReadableTable(null, sig, false); }
public static IReadableTable newSingleColumnTable(String path, String... values) { TableSignature sig = new TableSignature(path, values.length, 0); List<String[]> content = new ArrayList<>(); for (String v : values) { content.add(new String[] { v }); } return new MockupReadableTable(content, sig, true); }
@Test public void testGetTableSnapshots() throws IOException { TableSignature tableSignature = new TableSignature("TEST_CAL_DT.csv", 100, System.currentTimeMillis()); List<TableSnapshotResponse> snapshotResponseList = tableService.internalGetLookupTableSnapshots("EDW.TEST_CAL_DT", tableSignature); Assert.assertEquals(8, snapshotResponseList.size()); } }
@Override public TableSignature getSignature() throws IOException { Pair<Long, Long> sizeAndLastModified; try { sizeAndLastModified = getSizeAndLastModified(path); } catch (FileNotFoundException ex) { sizeAndLastModified = Pair.newPair(-1L, 0L); } return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond()); }
@Override public TableSignature getSignature() throws IOException { return new IReadableTable.TableSignature("", 2 * 1024 * 1024, System.currentTimeMillis()); } @Override