public int getColumnLength(TblColRef col) { return dimEncMap.get(col).getLengthOfEncoding(); }
private ByteArray encode(DimensionEncoding enc, Object value) { byte[] buf = new byte[enc.getLengthOfEncoding()]; String valueStr = value == null ? null : value.toString(); enc.encode(valueStr, buf, 0); return new ByteArray(buf); }
protected String fromBytes(byte[] bytes) { if (DimensionEncoding.isNull(bytes, 0, bytes.length)) { return null; } return Bytes.toString(bytes); } }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); Counter<ByteArray> counter = topNCounterIterator.next(); int offset = counter.getItem().offset(); for (int i = 0; i < dimensionEncodings.length; i++) { String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); tuple.setDimensionValue(literalTupleIdx[i], colValue); offset += dimensionEncodings[i].getLengthOfEncoding(); } tuple.setMeasureValue(numericTupleIdx, counter.getCount()); } };
int innerBuffOffset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); newDimensionEncodings[i].encode(dimValue, newIdBuf, bufOffset + innerBuffOffset); innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding(); offset += dimensionEncodings[i].getLengthOfEncoding();
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();
public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) { DimensionEncoding dimEnc = dimEncMap.get(col); if (dimEnc instanceof DictionaryDimEnc) dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue); dimEnc.encode(value, output, outputOffset); }
private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) { final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()]; for (int i = 0; i < literalCols.size(); i++) { TblColRef colRef = literalCols.get(i); Pair<String, String> topNEncoding = TopNMeasureType.getEncoding(function, colRef); String encoding = topNEncoding.getFirst(); String encodingVersionStr = topNEncoding.getSecond(); if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef)); } else { int encodingVersion = 1; if (!StringUtils.isEmpty(encodingVersionStr)) { try { encodingVersion = Integer.parseInt(encodingVersionStr); } catch (NumberFormatException e) { throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName() + " has to be an integer"); } } Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); String encodingName = (String) encodingConf[0]; String[] encodingArgs = (String[]) encodingConf[1]; encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType()); dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion); } } return dimensionEncodings; }
@Test public void testFixLenHexEncSerDser() { FixedLenHexDimEnc enc = new FixedLenHexDimEnc(6); ByteBuffer buff = ByteBuffer.allocate(1024); writeDimensionEncoding(enc, buff); buff.flip(); DimensionEncoding dimensionEncoding = readDimensionEncoding(buff); Assert.assertEquals(3, dimensionEncoding.asDataTypeSerializer().peekLength(null)); } }
public String readColumnString(TblColRef col, byte[] bytes, int offset, int length) { DimensionEncoding dimEnc = dimEncMap.get(col); return dimEnc.decode(bytes, offset, length); }
int innerBuffOffset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); newDimensionEncodings[i].encode(dimValue, newIdBuf, bufOffset + innerBuffOffset); innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding(); offset += dimensionEncodings[i].getLengthOfEncoding();
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); Counter<ByteArray> counter = topNCounterIterator.next(); int offset = counter.getItem().offset(); for (int i = 0; i < dimensionEncodings.length; i++) { String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); tuple.setDimensionValue(literalTupleIdx[i], colValue); offset += dimensionEncodings[i].getLengthOfEncoding(); } tuple.setMeasureValue(numericTupleIdx, counter.getCount()); } };
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();
public void writeColumn(TblColRef col, String value, int roundingFlag, byte defaultValue, byte[] output, int outputOffset) { DimensionEncoding dimEnc = dimEncMap.get(col); if (dimEnc instanceof DictionaryDimEnc) dimEnc = ((DictionaryDimEnc) dimEnc).copy(roundingFlag, defaultValue); dimEnc.encode(value, output, outputOffset); }
public void init(int index, CubeDesc cubeDesc) { bitIndex = index; colRef = cubeDesc.getModel().findColumn(column); column = colRef.getIdentity(); Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); encodingName = (String) encodingConf[0]; encodingArgs = (String[]) encodingConf[1]; if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); // convert date/time dictionary on date/time column to DimensionEncoding implicitly // however date/time dictionary on varchar column is still required DataType type = colRef.getType(); if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { if (type.isDate()) { encoding = encodingName = DateDimEnc.ENCODING_NAME; } if (type.isTimeFamily()) { encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) { logger.warn(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } }
@Override public void init(GTInfo info) { this.info = info; ImmutableBitSet dDims = info.getDynamicDims(); this.serializers = new DataTypeSerializer[info.getColumnCount()]; for (int i = 0; i < serializers.length; i++) { DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null; if (dimEnc != null) { // for dimensions serializers[i] = dimEnc.asDataTypeSerializer(); } else { DataTypeSerializer dSerializer = DataTypeSerializer.create(info.getColumnType(i)); if (dDims != null && dDims.get(i)) { // for dynamic dimensions dSerializer = new DynamicDimSerializer(dSerializer); } else { // for measures } serializers[i] = dSerializer; } } }
public String readColumnString(TblColRef col, byte[] bytes, int offset, int length) { DimensionEncoding dimEnc = dimEncMap.get(col); return dimEnc.decode(bytes, offset, length); }
private void initialize(MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, oldDicts); keyLength = 0; boolean hasDictEncoding = false; for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); if (encoding instanceof DictionaryDimEnc) { hasDictEncoding = true; } } newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, newDicts); newKeyLength = 0; for (DimensionEncoding encoding : newDimensionEncodings) { newKeyLength += encoding.getLengthOfEncoding(); } needReEncode = hasDictEncoding; } };
@Override public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { double counter = values[0] == null ? 0 : Double.parseDouble(values[0]); if (dimensionEncodings == null) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, dictionaryMap); for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); } if (values.length != (literalCols.size() + 1)) { throw new IllegalArgumentException(); } } final ByteArray key = new ByteArray(keyLength); int offset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { if (values[i + 1] == null) { Arrays.fill(key.array(), offset, offset + dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL); } else { dimensionEncodings[i].encode(values[i + 1], key.array(), offset); } offset += dimensionEncodings[i].getLengthOfEncoding(); } TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>( dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE); topNCounter.offer(key, counter); return topNCounter; }
@Override public boolean isNull(ByteArray code) { return DimensionEncoding.isNull(code.array(), code.offset(), code.length()); }