for (HBaseColumnFamilyDesc cf : getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { String[] colMeasureRefs = c.getMeasureRefs();
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); inputCodec = new MeasureCodec(cubeDesc.getMeasures()); inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } } }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
private void validateColumnFamily(CubeDesc cubeDesc) { Set<String> columnFamilyMetricsSet = Sets.newHashSet(); for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) { for (String columnName : hBaseColumnDesc.getMeasureRefs()) { columnFamilyMetricsSet.add(columnName); } } } for (MeasureDesc measureDesc : cubeDesc.getMeasures()) { if (!columnFamilyMetricsSet.contains(measureDesc.getName())) { throw new BadRequestException("column family lack measure:" + measureDesc.getName()); } } if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) { throw new BadRequestException( "the number of input measure and the number of measure defined in cubedesc are not consistent"); } }
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (int i = 0; i < familyDesc.getColumns().length; i++) { metricsColBlocks.add(new BitSet()); for (HBaseColumnFamilyDesc familyDesc : cuboid.getCubeDesc().getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (hbaseColDesc.containsMeasure(measure.getName())) {
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
final List<KeyValueCreator> keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
@Test(expected = IllegalArgumentException.class) public void testError() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("11111111111111111111333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); Long item_count = new Long(100); codec.encode(new Object[] { sum, min, max, count, item_count }); } }
for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { HColumnDescriptor cf = createColumnFamily(kylinConfig, cfDesc.getName(), cfDesc.isMemoryHungry()); tableDesc.addFamily(cf);
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0]; BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); Long count = new Long(2); Long item_count = new Long(100); ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count }); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasure(aggrFunc); rowValueDecoder.setProjectIndex(index); } rowValueDecoder.decodeAndConvertJavaObj(valueBytes); Object[] measureValues = rowValueDecoder.getValues(); //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues)); }
@Test public void testBadInit15() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs( new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }
@Test public void testBadInit14() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); String measureInfoForTransCnt = measureForTransCnt.toString(); thrown.expectMessage( "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }
newCubeDesc.setDictionaries(cubeDesc.getDictionaries()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature()); newCubeDesc.setNotifyList(cubeDesc.getNotifyList());
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); inputCodec = new MeasureCodec(cubeDesc.getMeasures()); inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } } }
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
public static CubeDesc getCopyOf(CubeDesc cubeDesc) { CubeDesc newCubeDesc = new CubeDesc(); newCubeDesc.setName(cubeDesc.getName()); newCubeDesc.setModelName(cubeDesc.getModelName()); newCubeDesc.setDescription(cubeDesc.getDescription()); newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); newCubeDesc.setDimensions(cubeDesc.getDimensions()); newCubeDesc.setMeasures(cubeDesc.getMeasures()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature()); newCubeDesc.setNotifyList(cubeDesc.getNotifyList()); newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges()); newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); newCubeDesc.setConfig(cubeDesc.getConfig()); newCubeDesc.updateRandomUuid(); return newCubeDesc; }
newCubeDesc.setDictionaries(cubeDesc.getDictionaries()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature()); newCubeDesc.setNotifyList(cubeDesc.getNotifyList());