public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
private void setThreshold(Collection<TblColRef> dimensions, List<RowValueDecoder> valueDecoders, StorageContext context) { if (RowValueDecoder.hasMemHungryCountDistinct(valueDecoders) == false) { return; } int rowSizeEst = dimensions.size() * 3; for (RowValueDecoder decoder : valueDecoders) { MeasureDesc[] measures = decoder.getMeasures(); BitSet projectionIndex = decoder.getProjectionIndex(); for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) { FunctionDesc func = measures[i].getFunction(); rowSizeEst += func.getReturnDataType().getSpaceEstimate(); } } long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; context.setThreshold((int) rowEst); }
public void decode(byte[] bytes) { codec.decode(ByteBuffer.wrap(bytes), values); convertToJavaObjects(values, values); }
HBaseColumnDesc hbaseColumn = rowValueDecoder.getHBaseColumn(); String columnFamily = hbaseColumn.getColumnFamilyName(); String qualifier = hbaseColumn.getQualifier(); rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); BitSet projectionIndex = rowValueDecoder.getProjectionIndex(); for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) { tuple.setMeasureValue(measureNames.get(i), measureValues[i]);
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, // StorageContext context) { Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap(); for (FunctionDesc aggrFunc : metrics) { Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc); if (hbCols.isEmpty()) { throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression()); } HBaseColumnDesc bestHBCol = null; int bestIndex = -1; for (HBaseColumnDesc hbCol : hbCols) { bestHBCol = hbCol; bestIndex = hbCol.findMeasureIndex(aggrFunc); MeasureDesc measure = hbCol.getMeasures()[bestIndex]; // criteria for holistic measure: Exact Aggregation && Exact Cuboid if (measure.isHolisticCountDistinct() && context.isExactAggregation()) { logger.info("Holistic count distinct chosen for " + aggrFunc); break; } } RowValueDecoder codec = codecMap.get(bestHBCol); if (codec == null) { codec = new RowValueDecoder(bestHBCol); codecMap.put(bestHBCol, codec); } codec.setIndex(bestIndex); } return new ArrayList<RowValueDecoder>(codecMap.values()); }
List<String> names = rowValueDecoder.getNames(); MeasureDesc[] measures = rowValueDecoder.getMeasures(); for (int i = 0; i < measures.length; i++) { String dataType = measures[i].getFunction().getSQLType();
public static boolean hasMemHungryCountDistinct(Collection<RowValueDecoder> rowValueDecoders) { for (RowValueDecoder decoder : rowValueDecoders) { if (decoder.hasMemHungryCountDistinct()) return true; } return false; }
public static ObserverAggregators fromValueDecoders(Collection<RowValueDecoder> rowValueDecoders) { // each decoder represents one HBase column HCol[] hcols = new HCol[rowValueDecoders.size()]; int i = 0; for (RowValueDecoder rowValueDecoder : rowValueDecoders) { hcols[i++] = buildHCol(rowValueDecoder.getHBaseColumn()); } ObserverAggregators aggrs = new ObserverAggregators(hcols); return aggrs; }
private static boolean isCoprocessorBeneficial(CubeInstance cube, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) { String forceFlag = System.getProperty(FORCE_COPROCESSOR); if (forceFlag != null) { return Boolean.parseBoolean(forceFlag); } Boolean cubeOverride = CUBE_OVERRIDES.get(cube.getName()); if (cubeOverride != null) { return cubeOverride.booleanValue(); } if (RowValueDecoder.hasMemHungryCountDistinct(rowValueDecoders)) { logger.info("Coprocessor is disabled because there is memory hungry count distinct"); return false; } if (context.isExactAggregation()) { logger.info("Coprocessor is disabled because exactAggregation is true"); return false; } Cuboid cuboid = context.getCuboid(); Set<TblColRef> toAggr = Sets.newHashSet(cuboid.getAggregationColumns()); toAggr.removeAll(groupBy); if (toAggr.isEmpty()) { logger.info("Coprocessor is disabled because no additional columns to aggregate"); return false; } logger.info("Coprocessor is enabled to aggregate " + toAggr + ", returning " + groupBy); return true; }
private Scan buildScan(HBaseKeyRange keyRange) { Scan scan = new Scan(); scan.setCaching(SCAN_CACHE); scan.setCacheBlocks(true); scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); for (RowValueDecoder valueDecoder : this.rowValueDecoders) { HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn(); byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName()); byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier()); scan.addColumn(byteFamily, byteQualifier); } scan.setStartRow(keyRange.getStartKey()); scan.setStopRow(keyRange.getStopKey()); return scan; }
public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public void decode(byte[] bytes) { codec.decode(ByteBuffer.wrap(bytes), values); convertToJavaObjects(values, values); }