@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); inputCodec = new MeasureCodec(cubeDesc.getMeasures()); inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } } }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
CubeDesc cubeDesc = cube.getDescriptor(); FsPermission permission = new FsPermission((short) 0777); for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { String cfName = cf.getName(); fs.setPermission(new Path(input + cfName), permission);
for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName()); cf.setMaxVersions(1);
List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHBaseMapping(), metrics, context);
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); inputCodec = new MeasureCodec(cubeDesc.getMeasures()); inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } } }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
CubeDesc cubeDesc = cube.getDescriptor(); FsPermission permission = new FsPermission((short) 0777); for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) { String cfName = cf.getName(); Path columnFamilyPath = new Path(input, cfName);
for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName()); cf.setMaxVersions(1);