public int getColumnLength(TblColRef col) { return dimEncMap.get(col).getLengthOfEncoding(); }
private void initialize(MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, oldDicts); keyLength = 0; boolean hasDictEncoding = false; for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); if (encoding instanceof DictionaryDimEnc) { hasDictEncoding = true; } } newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, newDicts); newKeyLength = 0; for (DimensionEncoding encoding : newDimensionEncodings) { newKeyLength += encoding.getLengthOfEncoding(); } needReEncode = hasDictEncoding; } };
public static void writeDimensionEncoding(DimensionEncoding encoding, ByteBuffer out) { try { if (encoding == null) { BytesUtil.writeVInt(1, out); } else { BytesUtil.writeVInt(0, out); if (encoding instanceof DictionaryDimEnc) { encoding = new TrimmedDimEnc(encoding.getLengthOfEncoding()); } ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); oos.writeObject(encoding); BytesUtil.writeByteArray(baos.toByteArray(), out); } } catch (IOException e) { throw new RuntimeException(e); } }
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap, long sourceRowCount) { final CubeDesc cubeDesc = cubeSegment.getCubeDesc(); final List<Integer> rowkeyColumnSize = Lists.newArrayList(); final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); final List<TblColRef> columnList = baseCuboid.getColumns(); final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap(); final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId()); for (int i = 0; i < columnList.size(); i++) { rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding()); } Map<Long, Double> sizeMap = Maps.newHashMap(); for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) { sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount)); } return sizeMap; }
private String translate(TblColRef column, String v, int roundingFlag) { byte[] id = new byte[dimEncMap.get(column).getLengthOfEncoding()]; columnIO.writeColumn(column, v, roundingFlag, DimensionEncoding.NULL, id, 0); return Dictionary.dictIdToString(id, 0, id.length); } }
@Override public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { double counter = values[0] == null ? 0 : Double.parseDouble(values[0]); if (dimensionEncodings == null) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, dictionaryMap); for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); } if (values.length != (literalCols.size() + 1)) { throw new IllegalArgumentException(); } } final ByteArray key = new ByteArray(keyLength); int offset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { if (values[i + 1] == null) { Arrays.fill(key.array(), offset, offset + dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL); } else { dimensionEncodings[i].encode(values[i + 1], key.array(), offset); } offset += dimensionEncodings[i].getLengthOfEncoding(); } TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>( dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE); topNCounter.offer(key, counter); return topNCounter; }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); Counter<ByteArray> counter = topNCounterIterator.next(); int offset = counter.getItem().offset(); for (int i = 0; i < dimensionEncodings.length; i++) { String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); tuple.setDimensionValue(literalTupleIdx[i], colValue); offset += dimensionEncodings[i].getLengthOfEncoding(); } tuple.setMeasureValue(numericTupleIdx, counter.getCount()); } };
private ByteArray encode(DimensionEncoding enc, Object value) { byte[] buf = new byte[enc.getLengthOfEncoding()]; String valueStr = value == null ? null : value.toString(); enc.encode(valueStr, buf, 0); return new ByteArray(buf); }
for (int i = 0; i < dimensionEncodings.length; i++) { String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); newDimensionEncodings[i].encode(dimValue, newIdBuf, bufOffset + innerBuffOffset); innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding(); offset += dimensionEncodings[i].getLengthOfEncoding();
DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig) { final List<Pair<byte[], byte[]>> selectedColumns = Lists.newArrayList(); selectedColumns.add(new Pair<>(FAM, COL_M)); int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN; int bodyLength = 0; ImmutableBitSet primaryKey = gtInfo.getPrimaryKey(); for (int i = 0; i < primaryKey.trueBitCount(); i++) { bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding(); } //Mock start key byte[] start = new byte[headerLength + bodyLength]; BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock end key byte[] end = new byte[headerLength + bodyLength + 1]; for (int i = 0; i < end.length - 1; i++) { end[i] = RowConstants.ROWKEY_UPPER_BYTE; } BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock fuzzy key List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList(); return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(), kylinConfig.getHBaseScanMaxResultSize()); }
public int getColumnLength(TblColRef col) { return dimEncMap.get(col).getLengthOfEncoding(); }
private void initialize(MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, oldDicts); keyLength = 0; boolean hasDictEncoding = false; for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); if (encoding instanceof DictionaryDimEnc) { hasDictEncoding = true; } } newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, newDicts); newKeyLength = 0; for (DimensionEncoding encoding : newDimensionEncodings) { newKeyLength += encoding.getLengthOfEncoding(); } needReEncode = hasDictEncoding; } };
public static void writeDimensionEncoding(DimensionEncoding encoding, ByteBuffer out) { try { if (encoding == null) { BytesUtil.writeVInt(1, out); } else { BytesUtil.writeVInt(0, out); if (encoding instanceof DictionaryDimEnc) { encoding = new TrimmedDimEnc(encoding.getLengthOfEncoding()); } ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); oos.writeObject(encoding); BytesUtil.writeByteArray(baos.toByteArray(), out); } } catch (IOException e) { throw new RuntimeException(e); } }
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap, long sourceRowCount) { final CubeDesc cubeDesc = cubeSegment.getCubeDesc(); final List<Integer> rowkeyColumnSize = Lists.newArrayList(); final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); final List<TblColRef> columnList = baseCuboid.getColumns(); final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap(); final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId()); for (int i = 0; i < columnList.size(); i++) { rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding()); } Map<Long, Double> sizeMap = Maps.newHashMap(); for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) { sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount)); } return sizeMap; }
private String translate(TblColRef column, String v, int roundingFlag) { byte[] id = new byte[dimEncMap.get(column).getLengthOfEncoding()]; columnIO.writeColumn(column, v, roundingFlag, DimensionEncoding.NULL, id, 0); return Dictionary.dictIdToString(id, 0, id.length); } }
@Override public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { double counter = values[0] == null ? 0 : Double.parseDouble(values[0]); if (dimensionEncodings == null) { literalCols = getTopNLiteralColumn(measureDesc.getFunction()); dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(), literalCols, dictionaryMap); for (DimensionEncoding encoding : dimensionEncodings) { keyLength += encoding.getLengthOfEncoding(); } if (values.length != (literalCols.size() + 1)) { throw new IllegalArgumentException(); } } final ByteArray key = new ByteArray(keyLength); int offset = 0; for (int i = 0; i < dimensionEncodings.length; i++) { if (values[i + 1] == null) { Arrays.fill(key.array(), offset, offset + dimensionEncodings[i].getLengthOfEncoding(), DimensionEncoding.NULL); } else { dimensionEncodings[i].encode(values[i + 1], key.array(), offset); } offset += dimensionEncodings[i].getLengthOfEncoding(); } TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>( dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE); topNCounter.offer(key, counter); return topNCounter; }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); Counter<ByteArray> counter = topNCounterIterator.next(); int offset = counter.getItem().offset(); for (int i = 0; i < dimensionEncodings.length; i++) { String colValue = dimensionEncodings[i].decode(counter.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); tuple.setDimensionValue(literalTupleIdx[i], colValue); offset += dimensionEncodings[i].getLengthOfEncoding(); } tuple.setMeasureValue(numericTupleIdx, counter.getCount()); } };
for (int i = 0; i < dimensionEncodings.length; i++) { String dimValue = dimensionEncodings[i].decode(c.getItem().array(), offset, dimensionEncodings[i].getLengthOfEncoding()); newDimensionEncodings[i].encode(dimValue, newIdBuf, bufOffset + innerBuffOffset); innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding(); offset += dimensionEncodings[i].getLengthOfEncoding();
DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();