public boolean isNullId(int id) { int nullId = NULL_ID[getSizeOfId()]; return (nullId & id) == nullId; }
@Override public int maxLength() { return dict.getSizeOfId(); }
public DictionaryDimEnc(Dictionary<String> dict, int roundingFlag, byte defaultByte) { this.dict = dict; this.fixedLen = dict.getSizeOfId(); this.roundingFlag = roundingFlag; this.defaultByte = defaultByte; }
@Override public int getStorageBytesEstimate() { return dict.getSizeOfId(); } }
public int nullId() { return NULL_ID[getSizeOfId()]; }
@Override public int peekLength(ByteBuffer in) { return dict.getSizeOfId(); }
@Override public void serialize(Object value, ByteBuffer buf) { int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag); BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf); }
@Override public Object deserialize(ByteBuffer in) { int id = BytesUtil.readUnsigned(in, dict.getSizeOfId()); return dict.getValueFromId(id); }
@Override public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length != 1) throw new IllegalArgumentException(); //source input column value String literal = values[0]; // encode literal using dictionary TblColRef literalCol = getRawColumn(measureDesc.getFunction()); Dictionary<String> dictionary = dictionaryMap.get(literalCol); int keyEncodedValue = dictionary.getIdFromValue(literal); ByteArray key = new ByteArray(dictionary.getSizeOfId()); BytesUtil.writeUnsigned(keyEncodedValue, key.array(), key.offset(), dictionary.getSizeOfId()); List<ByteArray> valueList = new ArrayList<ByteArray>(1); valueList.add(key); return valueList; }
@Override public List<ByteArray> reEncodeDictionary(List<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { TblColRef colRef = getRawColumn(measureDesc.getFunction()); Dictionary<String> sourceDict = oldDicts.get(colRef); Dictionary<String> mergedDict = newDicts.get(colRef); int valueSize = value.size(); byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()]; int bufOffset = 0; for (ByteArray c : value) { int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length()); int newId; String v = sourceDict.getValueFromId(oldId); if (v == null) { newId = mergedDict.nullId(); } else { newId = mergedDict.getIdFromValue(v); } BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId()); c.reset(newIdBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); } return value; } };
public ShrunkenDictionary<T> build(ValueSerializer<T> valueSerializer) { return new ShrunkenDictionary<>(valueSerializer, fullDict.getMinId(), fullDict.getMaxId(), fullDict.getSizeOfId(), fullDict.getSizeOfValue(), valueToIdMap); } }
private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, DictionaryInfo dictInfo) throws IOException { if (dictInfo == null) return; // work on copy instead of cached objects CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid()); Dictionary<?> dict = dictInfo.getDictionaryObject(); segCopy.putDictResPath(col, dictInfo.getResourcePath()); segCopy.getRowkeyStats().add(new Object[] { col.getIdentity(), dict.getSize(), dict.getSizeOfId() }); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); updateCube(update); }
BytesUtil.writeUnsigned(mergedDict.nullId(), newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); continue; } else { mergedDict.getSizeOfId() > newKeyBodyBuf.length - bufOffset) { byte[] oldBuf = newKeyBodyBuf; newKeyBodyBuf = new byte[2 * newKeyBodyBuf.length]; BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); } else {
@Override public int peekLength(ByteBuffer in) { return dict.getSizeOfId(); }
@Override public int maxLength() { return dict.getSizeOfId(); }
public DictionaryDimEnc(Dictionary<String> dict, int roundingFlag, byte defaultByte) { this.dict = dict; this.fixedLen = dict.getSizeOfId(); this.roundingFlag = roundingFlag; this.defaultByte = defaultByte; }
@Override public int getStorageBytesEstimate() { return dict.getSizeOfId(); } }
@Override public void serialize(Object value, ByteBuffer buf) { int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag); BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf); }
@Override public Object deserialize(ByteBuffer in) { int id = BytesUtil.readUnsigned(in, dict.getSizeOfId()); return dict.getValueFromId(id); }
@Override public int getColumnLength(TblColRef col) { Dictionary<?> dict = getDictionary(col); if (dict == null) { return this.getCubeDesc().getRowkey().getColumnLength(col); } else { return dict.getSizeOfId(); } }