public boolean isMemoryHungry() { for (HBaseColumnDesc hBaseColumnDesc : columns) { for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { return true; } } } return false; }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName()); this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier()); this.projectionIndex = new BitSet(); this.measures = hbaseColumn.getMeasures(); this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
@Test(expected = IllegalArgumentException.class) public void testError() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("11111111111111111111333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); Long item_count = new Long(100); codec.encode(new Object[] { sum, min, max, count, item_count }); } }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); Long count = new Long(2); Long item_count = new Long(100); ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count }); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasure(aggrFunc); rowValueDecoder.setProjectIndex(index); } rowValueDecoder.decodeAndConvertJavaObj(valueBytes); Object[] measureValues = rowValueDecoder.getValues(); //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues)); }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.projectionIndex = new BitSet(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, // StorageContext context) { Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap(); for (FunctionDesc aggrFunc : metrics) { Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc); if (hbCols.isEmpty()) { throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression()); } HBaseColumnDesc bestHBCol = null; int bestIndex = -1; for (HBaseColumnDesc hbCol : hbCols) { bestHBCol = hbCol; bestIndex = hbCol.findMeasureIndex(aggrFunc); MeasureDesc measure = hbCol.getMeasures()[bestIndex]; // criteria for holistic measure: Exact Aggregation && Exact Cuboid if (measure.isHolisticCountDistinct() && context.isExactAggregation()) { logger.info("Holistic count distinct chosen for " + aggrFunc); break; } } RowValueDecoder codec = codecMap.get(bestHBCol); if (codec == null) { codec = new RowValueDecoder(bestHBCol); codecMap.put(bestHBCol, codec); } codec.setIndex(bestIndex); } return new ArrayList<RowValueDecoder>(codecMap.values()); }
private static HCol buildHCol(HBaseColumnDesc desc) { byte[] family = Bytes.toBytes(desc.getColumnFamilyName()); byte[] qualifier = Bytes.toBytes(desc.getQualifier()); MeasureDesc[] measures = desc.getMeasures(); String[] funcNames = new String[measures.length]; String[] dataTypes = new String[measures.length]; for (int i = 0; i < measures.length; i++) { funcNames[i] = measures[i].getFunction().getExpression(); dataTypes[i] = measures[i].getFunction().getReturnType(); } return new HCol(family, qualifier, funcNames, dataTypes); }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.projectionIndex = new BitSet(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public boolean isMemoryHungry() { for (HBaseColumnDesc hBaseColumnDesc : columns) { for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { return true; } } } return false; }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName()); this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier()); this.projectionIndex = new BitSet(); this.measures = hbaseColumn.getMeasures(); this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }