DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { ResourceStore store = getStore(); logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath); DictionaryInfo info = store.getResource(resourcePath, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); return info; }
public void removeDictionary(String resourcePath) throws IOException { logger.info("Remvoing dict: " + resourcePath); ResourceStore store = getStore(); store.deleteResource(resourcePath); dictCache.invalidate(resourcePath); }
void save(DictionaryInfo dict) throws IOException { ResourceStore store = getStore(); String path = dict.getResourcePath(); logger.info("Saving dictionary at " + path); store.putBigResource(path, dict, System.currentTimeMillis(), DictionaryInfoSerializer.FULL_SERIALIZER); }
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException { ResourceStore store = getStore(); NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; logger.info("{} existing dictionaries of the same column", existings.size()); if (existings.size() > 100) { logger.warn("Too many dictionaries under {}, dict count: {}", dictInfo.getResourceDir(), existings.size()); } for (String existing : existings) { DictionaryInfo existingInfo = getDictionaryInfo(existing); if (existingInfo != null) { if ((config.isDictResuable() && existingInfo.getDictionaryObject().contains(dict)) || dict.equals(existingInfo.getDictionaryObject())) { return existing; } } } return null; }
private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; for (DictionaryInfo dictionaryInfo : allResources) { if (largestDict == null) { largestDict = dictionaryInfo; continue; } if (largestDict.getCardinality() < dictionaryInfo.getCardinality()) { largestDict = dictionaryInfo; } } return largestDict; }
public void removeDictionaries(String srcTable, String srcCol) throws IOException { DictionaryInfo info = new DictionaryInfo(); info.setSourceTable(srcTable); info.setSourceColumn(srcCol); ResourceStore store = getStore(); NavigableSet<String> existings = store.listResources(info.getResourceDir()); if (existings == null) return; for (String existing : existings) removeDictionary(existing); }
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); for (DictionaryInfo dictionaryInfo : allResources) { if (input.equals(dictionaryInfo.getInput())) { return dictionaryInfo.getResourcePath(); } } return null; }
DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException { ResourceStore store = getStore(); logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath); DictionaryInfo info = store.getResource(resourcePath, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER); return info; }
public void removeDictionary(String resourcePath) throws IOException { logger.info("Remvoing dict: " + resourcePath); ResourceStore store = getStore(); store.deleteResource(resourcePath); dictCache.invalidate(resourcePath); }
void save(DictionaryInfo dict) throws IOException { ResourceStore store = getStore(); String path = dict.getResourcePath(); logger.info("Saving dictionary at " + path); store.putBigResource(path, dict, System.currentTimeMillis(), DictionaryInfoSerializer.FULL_SERIALIZER); }
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException { ResourceStore store = getStore(); NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; logger.info("{} existing dictionaries of the same column", existings.size()); if (existings.size() > 100) { logger.warn("Too many dictionaries under {}, dict count: {}", dictInfo.getResourceDir(), existings.size()); } for (String existing : existings) { DictionaryInfo existingInfo = getDictionaryInfo(existing); if (existingInfo != null) { if ((config.isDictResuable() && existingInfo.getDictionaryObject().contains(dict)) || dict.equals(existingInfo.getDictionaryObject())) { return existing; } } } return null; }
private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; for (DictionaryInfo dictionaryInfo : allResources) { if (largestDict == null) { largestDict = dictionaryInfo; continue; } if (largestDict.getCardinality() < dictionaryInfo.getCardinality()) { largestDict = dictionaryInfo; } } return largestDict; }
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { final ResourceStore store = getStore(); final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); for (DictionaryInfo dictionaryInfo : allResources) { if (input.equals(dictionaryInfo.getInput())) { return dictionaryInfo.getResourcePath(); } } return null; }
public void removeDictionaries(String srcTable, String srcCol) throws IOException { DictionaryInfo info = new DictionaryInfo(); info.setSourceTable(srcTable); info.setSourceColumn(srcCol); ResourceStore store = getStore(); NavigableSet<String> existings = store.listResources(info.getResourceDir()); if (existings == null) return; for (String existing : existings) removeDictionary(existing); }