public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName()); this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier()); this.projectionIndex = new BitSet(); this.measures = hbaseColumn.getMeasures(); this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public void init(CubeDesc cubeDesc) { cubeRef = cubeDesc; for (HBaseColumnFamilyDesc cf : columnFamily) { cf.setName(cf.getName().toUpperCase(Locale.ROOT)); for (HBaseColumnDesc c : cf.getColumns()) { c.setQualifier(c.getQualifier().toUpperCase(Locale.ROOT)); StringUtil.toUpperCaseArray(c.getMeasureRefs(), c.getMeasureRefs()); } } }
public void initAsSeparatedColumns(CubeDesc cubeDesc) { cubeRef = cubeDesc; int cfNum = cubeDesc.getMeasures().size(); columnFamily = new HBaseColumnFamilyDesc[cfNum]; for (int i = 0; i < cfNum; i++) { HBaseColumnFamilyDesc cf = new HBaseColumnFamilyDesc(); HBaseColumnDesc col = new HBaseColumnDesc(); String measureRef = cubeDesc.getMeasures().get(i).getName(); col.setMeasureRefs(new String[] { measureRef }); col.setQualifier("M"); cf.setColumns(new HBaseColumnDesc[] { col }); cf.setName("F" + (i + 1)); columnFamily[i] = cf; } }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
public boolean isMemoryHungry() { for (HBaseColumnDesc hBaseColumnDesc : columns) { for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { return true; } } } return false; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = System.currentTimeMillis(); List<MeasureDesc> measures = cubeDesc.getMeasures(); String[] measureNames = getMeasureNames(cubeDesc); String[] refs = colDesc.getMeasureRefs(); refIndex = new int[refs.length]; refMeasures = new MeasureDesc[refs.length]; for (int i = 0; i < refs.length; i++) { refIndex[i] = indexOf(measureNames, refs[i]); refMeasures[i] = measures.get(refIndex[i]); } codec = new MeasureCodec(refMeasures); colValues = new Object[refs.length]; isFullCopy = true; for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs(); MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length]; int[] measureIndex = new int[colMeasureRefs.length]; c.setMeasures(measureDescs); c.setMeasureIndex(measureIndex); c.setColumnFamilyName(cf.getName());
private Scan buildScan(HBaseKeyRange keyRange) { Scan scan = new Scan(); scan.setCaching(SCAN_CACHE); scan.setCacheBlocks(true); scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); for (RowValueDecoder valueDecoder : this.rowValueDecoders) { HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn(); byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName()); byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier()); scan.addColumn(byteFamily, byteQualifier); } scan.setStartRow(keyRange.getStartKey()); scan.setStopRow(keyRange.getStopKey()); return scan; }
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, // StorageContext context) { Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap(); for (FunctionDesc aggrFunc : metrics) { Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc); if (hbCols.isEmpty()) { throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression()); } HBaseColumnDesc bestHBCol = null; int bestIndex = -1; for (HBaseColumnDesc hbCol : hbCols) { bestHBCol = hbCol; bestIndex = hbCol.findMeasureIndex(aggrFunc); MeasureDesc measure = hbCol.getMeasures()[bestIndex]; // criteria for holistic measure: Exact Aggregation && Exact Cuboid if (measure.isHolisticCountDistinct() && context.isExactAggregation()) { logger.info("Holistic count distinct chosen for " + aggrFunc); break; } } RowValueDecoder codec = codecMap.get(bestHBCol); if (codec == null) { codec = new RowValueDecoder(bestHBCol); codecMap.put(bestHBCol, codec); } codec.setIndex(bestIndex); } return new ArrayList<RowValueDecoder>(codecMap.values()); }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); Long count = new Long(2); Long item_count = new Long(100); ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count }); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasure(aggrFunc); rowValueDecoder.setProjectIndex(index); } rowValueDecoder.decodeAndConvertJavaObj(valueBytes); Object[] measureValues = rowValueDecoder.getValues(); //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues)); }
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
private void validateColumnFamily(CubeDesc cubeDesc) { Set<String> columnFamilyMetricsSet = Sets.newHashSet(); for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) { for (String columnName : hBaseColumnDesc.getMeasureRefs()) { columnFamilyMetricsSet.add(columnName); } } } for (MeasureDesc measureDesc : cubeDesc.getMeasures()) { if (!columnFamilyMetricsSet.contains(measureDesc.getName())) { throw new BadRequestException("column family lack measure:" + measureDesc.getName()); } } if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) { throw new BadRequestException( "the number of input measure and the number of measure defined in cubedesc are not consistent"); } }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp List<MeasureDesc> measures = cubeDesc.getMeasures(); String[] measureNames = getMeasureNames(cubeDesc); String[] refs = colDesc.getMeasureRefs(); refIndex = new int[refs.length]; refMeasures = new MeasureDesc[refs.length]; for (int i = 0; i < refs.length; i++) { refIndex[i] = indexOf(measureNames, refs[i]); refMeasures[i] = measures.get(refIndex[i]); } codec = new MeasureCodec(refMeasures); colValues = new Object[refs.length]; isFullCopy = true; for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs(); MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length]; int[] measureIndex = new int[colMeasureRefs.length]; c.setMeasures(measureDescs); c.setMeasureIndex(measureIndex); c.setColumnFamilyName(cf.getName());
String columnFamily = hbaseColumn.getColumnFamilyName(); String qualifier = hbaseColumn.getQualifier();
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }