public static Dictionary getDictionary(AbsoluteTableIdentifier absoluteTableIdentifier, ColumnIdentifier columnIdentifier, DataType dataType) throws IOException { return getDictionary( new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, dataType)); }
protected void initPaths() { this.dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); this.dictionaryMetaFilePath = dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); }
protected void initPath() { String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); long dictOffset = CarbonUtil.getFileSize(dictionaryFilePath); this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(dictOffset); cleanUpOldSortIndex(dictionaryFilePath); }
/** * This method will remove the cache for a given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier */ @Override public void invalidate( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { carbonLRUCache.remove( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY)); }
/** * This method will get the value for the given key. If value does not exist * for the given key, it will check and load the value. * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return dictionary * @throws IOException in case memory is not sufficient to load dictionary into memory */ private Dictionary getDictionary( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { Dictionary forwardDictionary = null; // dictionary is only for primitive data type assert (!dictionaryColumnUniqueIdentifier.getDataType().isComplexType()); String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(); ColumnDictionaryInfo columnDictionaryInfo = getColumnDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier); // load sort index file in case of forward dictionary checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnDictionaryInfo, getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY), true); forwardDictionary = new ForwardDictionary(columnDictionaryInfo); return forwardDictionary; }
/** * This method will form the path for dictionary file for a given column */ protected void initFileLocation() { this.columnDictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); }
/** * This method will form the path for dictionary metadata file for a given column */ protected void initFileLocation() { this.columnDictionaryMetadataFilePath = dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); }
protected void initPath() { try { CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry(); long dictOffset = chunkMetaObjectForLastSegmentEntry.getEnd_offset(); this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(dictOffset); if (!FileFactory .isFileExist(this.sortIndexFilePath, FileFactory.getFileType(this.sortIndexFilePath))) { this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(); } } catch (IOException e) { this.sortIndexFilePath = dictionaryColumnUniqueIdentifier.getSortIndexFilePath(); } }
/** * This method will check and create columnDictionaryInfo object for the given column * * @param dictionaryColumnUniqueIdentifier * @param columnIdentifier * @return */ private ColumnDictionaryInfo getColumnDictionaryInfo( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String columnIdentifier) { ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY)); if (null == columnDictionaryInfo) { synchronized (dictionaryColumnUniqueIdentifier) { columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY)); if (null == columnDictionaryInfo) { columnDictionaryInfo = new ColumnDictionaryInfo(dictionaryColumnUniqueIdentifier.getDataType()); } } } return columnDictionaryInfo; }
/** * This method will remove the cache for a given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier */ @Override public void invalidate( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { carbonLRUCache.remove( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY)); }
/** * This method will get the value for the given key. If value does not exist * for the given key, it will check and load the value. * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return dictionary * @throws IOException in case memory is not sufficient to load dictionary into memory */ private Dictionary getDictionary( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { Dictionary reverseDictionary = null; // dictionary is only for primitive data type assert (!dictionaryColumnUniqueIdentifier.getDataType().isComplexType()); String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(); ColumnReverseDictionaryInfo columnReverseDictionaryInfo = getColumnReverseDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier); // do not load sort index file for reverse dictionary checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnReverseDictionaryInfo, getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY), false); reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo); return reverseDictionary; }
/** * This method will return the carbon file objetc based on its type (local, HDFS) * * @param dictionaryColumnUniqueIdentifier * @return */ private CarbonFile getDictionaryMetaCarbonFile( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath); CarbonFile dictFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType); // When rename table triggered parallely with select query, dictionary files may not exist if (!dictFile.exists()) { throw new IOException("Dictionary file does not exist: " + dictionaryFilePath); } return dictFile; }
new ColumnIdentifier(dims.get(i).getColumnId(), null, null); DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier( table.getAbsoluteTableIdentifier(), columnIdentifier, columnIdentifier.getDataType()); CarbonDictionaryWriter writer = writer.commit(); Dictionary dict = (Dictionary) dictCache.get( new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, dims.get(i).getDataType())); CarbonDictionarySortInfoPreparator preparator =
.put(dictionaryColumnUniqueIdentifiers.get(i).getColumnIdentifier().getColumnId(), columnDictionaryList.get(i));
/** * This method will check if dictionary and its metadata file exists for a given column * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return */ public static boolean isFileExistsForGivenColumn( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { String dictionaryFilePath = dictionaryColumnUniqueIdentifier.getDictionaryFilePath(); String dictionaryMetadataFilePath = dictionaryColumnUniqueIdentifier.getDictionaryMetaFilePath(); // check if both dictionary and its metadata file exists for a given column return isFileExists(dictionaryFilePath) && isFileExists(dictionaryMetadataFilePath); }
new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier, columnIdentifier.getDataType()); Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(identifier);
@Override public void clearAccessCount(List<DictionaryColumnUniqueIdentifier> keys) { for (DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier : keys) { Dictionary cacheable = (Dictionary) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY)); cacheable.clear(); } }
.get(CarbonCommonConstants.DICTIONARY_PATH); dictionaryColumnUniqueIdentifiers.add( new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, dimension.getDataType(), dictionaryPath));
@Override public void clearAccessCount(List<DictionaryColumnUniqueIdentifier> keys) { for (DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier : keys) { Dictionary cacheable = (Dictionary) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY)); cacheable.clear(); } }
new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, carbonDimension.getDataType(), dictionaryPath); CacheProvider cacheProvider = CacheProvider.getInstance();