@Override public TableSignature getSignature() throws IOException { FileSystem fs = HadoopUtil.getFileSystem(path); FileStatus status = fs.getFileStatus(new Path(path)); return new TableSignature(path, status.getLen(), status.getModificationTime()); }
public String getResourcePath() { return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName() + "/" + uuid + ".snapshot"; }
private DictionaryInfo makeSharedDict() throws IOException { TableSignature signature = new TableSignature(); signature.setSize(100); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath("fake_common_dict"); DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature, ""); List<byte[]> values = new ArrayList<byte[]>(); values.add(new byte[] { 101, 101, 101 }); values.add(new byte[] { 102, 102, 102 }); Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values); dictionaryManager.trySaveNewDict(dict, newDictInfo); ((TrieDictionary) dict).dump(System.out); return newDictInfo; }
public DictionaryInfo mergeDictionary(List<DictionaryInfo> dicts) throws IOException { DictionaryInfo firstDictInfo = null; int totalSize = 0; for (DictionaryInfo info : dicts) { // check if (firstDictInfo == null) { firstDictInfo = info; } else { if (!firstDictInfo.isDictOnSameColumn(info)) { throw new IllegalArgumentException("Merging dictionaries are not structurally equal(regardless of signature)."); } } totalSize += info.getInput().getSize(); } if (firstDictInfo == null) { throw new IllegalArgumentException("DictionaryManager.mergeDictionary input cannot be null"); } DictionaryInfo newDictInfo = new DictionaryInfo(firstDictInfo); TableSignature signature = newDictInfo.getInput(); signature.setSize(totalSize); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath("merged_with_no_original_path"); String dupDict = checkDupByInfo(newDictInfo); if (dupDict != null) { logger.info("Identical dictionary input " + newDictInfo.getInput() + ", reuse existing dictionary at " + dupDict); return getDictionaryInfo(dupDict); } Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(newDictInfo, dicts); return trySaveNewDict(newDict, newDictInfo); }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct // load from // store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
for (CubeSegment segment : cube.getSegments()) { TableSignature signature = new TableSignature(); signature.setSize(100); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); ArrayList<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; TableSignature input = dictInfo.getInput(); for (String existing : existings) { DictionaryInfo existingInfo = load(existing, false); // skip cache, // direct // load from // store if (input.equals(existingInfo.getInput())) return existing; } return null; }
public String getResourceDir() { return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName(); }