public ByteBuffer[] getHColValues(MeasureAggregator[] aggrs) { int i = 0; for (int ci = 0; ci < nHCols; ci++) { HCol col = hcols[ci]; for (int j = 0; j < col.nMeasures; j++) col.measureValues[j] = aggrs[i++].getState(); col.measureBuf.clear(); col.measureCodec.encode(col.measureValues, col.measureBuf); hColValues[ci] = col.measureBuf; } return hColValues; }
public KeyValue create(Text key, Object[] measureValues) { for (int i = 0; i < colValues.length; i++) { colValues[i] = measureValues[refIndex[i]]; } valueBuf.clear(); codec.encode(colValues, valueBuf); return create(key, valueBuf.array(), 0, valueBuf.position()); }
private void buildValue(SplittedBytes[] splitBuffers) { for (int i = 0; i < measures.length; i++) { byte[] valueBytes = getValueBytes(splitBuffers, i); measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes); } valueBuf.clear(); measureCodec.encode(measures, valueBuf); }
@Override public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { codec.decode(value, input); aggs.aggregate(input); } aggs.collectStates(result); valueBuf.clear(); codec.encode(result, valueBuf); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); counter++; if (counter % BatchConstants.COUNTER_MAX == 0) { logger.info("Handled " + counter + " records!"); } }
private void buildValue(SplittedBytes[] splitBuffers) { for (int i = 0; i < measures.length; i++) { byte[] valueBytes = getValueBytes(splitBuffers, i); measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes); } valueBuf.clear(); measureCodec.encode(measures, valueBuf); }
private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) { Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) }; buf.clear(); codec.encode(values, buf); Text t = new Text(); t.set(buf.array(), 0, buf.position()); return t; }
private Cell newCell(byte[] key, HCol col, String decimal, int number) { Object[] values = number == Integer.MIN_VALUE ? // new Object[] { new BigDecimal(decimal) } // : new Object[] { new BigDecimal(decimal), new LongWritable(number) }; buf.clear(); col.measureCodec.encode(values, buf); Cell keyValue = new KeyValue(key, 0, key.length, // col.family, 0, col.family.length, // col.qualifier, 0, col.qualifier.length, // HConstants.LATEST_TIMESTAMP, Type.Put, // buf.array(), 0, buf.position()); return keyValue; }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
@Test public void basicTest() { MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") }; MeasureCodec codec = new MeasureCodec(descs); DoubleWritable d = new DoubleWritable(1.0); LongWritable l = new LongWritable(2); BigDecimal b = new BigDecimal("333.1234567"); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); Object values[] = new Object[] { d, l, b, hllc, hllc2 }; ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(values, buf); buf.flip(); System.out.println("size: " + buf.limit()); Object copy[] = new Object[values.length]; codec.decode(buf, copy); assertTrue(Arrays.equals(values, copy)); }