public CubeDimEncMap getDimensionEncodingMap() { return new CubeDimEncMap(this); }
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap, long sourceRowCount) { final CubeDesc cubeDesc = cubeSegment.getCubeDesc(); final List<Integer> rowkeyColumnSize = Lists.newArrayList(); final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); final List<TblColRef> columnList = baseCuboid.getColumns(); final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap(); final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId()); for (int i = 0; i < columnList.size(); i++) { rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding()); } Map<Long, Double> sizeMap = Maps.newHashMap(); for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) { sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount)); } return sizeMap; }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap) ); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); // MemDiskStore store = new MemDiskStore(info, MemoryBudgetController.ZERO_BUDGET); IGTStore store = new ConcurrentDiskStore(info); GridTable gridTable = new GridTable(info, store); return gridTable; }
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap, long sourceRowCount) { final CubeDesc cubeDesc = cubeSegment.getCubeDesc(); final List<Integer> rowkeyColumnSize = Lists.newArrayList(); final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); final List<TblColRef> columnList = baseCuboid.getColumns(); final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap(); final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId()); for (int i = 0; i < columnList.size(); i++) { rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding()); } Map<Long, Double> sizeMap = Maps.newHashMap(); for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) { sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(), baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount)); } return sizeMap; }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap)); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); // MemDiskStore store = new MemDiskStore(info, MemoryBudgetController.ZERO_BUDGET); IGTStore store = new ConcurrentDiskStore(info); GridTable gridTable = new GridTable(info, store); return gridTable; }
@Override protected void doSetup(Mapper.Context context) throws IOException { super.doSetup(context); long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); GTInfo gtInfo = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, baseCuboid), new CubeDimEncMap(cubeDesc, dictionaryMap)); keyValueBuffer = ByteBuffer.allocate(gtInfo.getMaxRecordLength()); keyOffset = cubeSegment.getRowKeyPreambleSize(); }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
this.gtInfo = CubeGridTable.newGTInfo(cuboid, new CubeDimEncMap(cubeSegment), mapping);
@Test public void testGTInfo() { CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready"); CubeSegment segment = cube.getFirstSegment(); Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor()); GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment)); GTInfo.serializer.serialize(info, buffer); buffer.flip(); GTInfo sInfo = GTInfo.serializer.deserialize(buffer); this.compareTwoGTInfo(info, sInfo); }
public CubeDimEncMap getDimensionEncodingMap() { return new CubeDimEncMap(this); }
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap) ); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); // MemDiskStore store = new MemDiskStore(info, MemoryBudgetController.ZERO_BUDGET); IGTStore store = new ConcurrentDiskStore(info); GridTable gridTable = new GridTable(info, store); return gridTable; }
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap)); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); // MemDiskStore store = new MemDiskStore(info, MemoryBudgetController.ZERO_BUDGET); IGTStore store = new ConcurrentDiskStore(info); GridTable gridTable = new GridTable(info, store); return gridTable; }
@Override protected void doSetup(Mapper.Context context) throws IOException { super.doSetup(context); long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc); GTInfo gtInfo = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, baseCuboid), new CubeDimEncMap(cubeDesc, dictionaryMap)); keyValueBuffer = ByteBuffer.allocate(gtInfo.getMaxRecordLength()); keyOffset = cubeSegment.getRowKeyPreambleSize(); }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
this.gtInfo = CubeGridTable.newGTInfo(cuboid, new CubeDimEncMap(cubeSegment), mapping);