public KVGTRecordWriter(CubeDesc cubeDesc, CubeSegment cubeSegment) { this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.measureCount = cubeDesc.getMeasures().size(); }
SplitMerger() { reuseAggrs = new MeasureAggregators(cubeDesc.getMeasures()); reuseMetricsArray = new Object[cubeDesc.getMeasures().size()]; }
Merger() { reuseAggrs = new MeasureAggregators(cubeDesc.getMeasures()); reuseMetricsArray = new Object[cubeDesc.getMeasures().size()]; }
public InputConverterUnitForRawData(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { this.flatDesc = new CubeJoinedFlatTableEnrich(flatDesc, cubeDesc); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures()); this.dictionaryMap = dictionaryMap; this.kvBuilder = new KeyValueBuilder(this.flatDesc); }
public void resetAggrs() { for (int i = 0; i < cubeDesc.getMeasures().size(); i++) { aggrIngesters[i].reset(); } } }
@Override public List<MeasureDesc> getMeasures() { return getDescriptor().getMeasures(); }
public InMemCubeBuilder2(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); }
public Object[] buildValueObjects(String[] flatRow) { Object[] measures = new Object[cubeDesc.getMeasures().size()]; for (int i = 0; i < measures.length; i++) { String[] colValues = kvBuilder.buildValueOf(i, flatRow); MeasureDesc measure = measureDescList.get(i); measures[i] = aggrIngesters[i].valueOf(colValues, measure, dictionaryMap); } return measures; }
public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); this.totalCuboidCount = cuboidScheduler.getCuboidCount(); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); }
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc, AbstractRowKeyEncoder rowKeyEncoder, MeasureIngester<?>[] aggrIngesters, Map<TblColRef, Dictionary<String>> dictionaryMap) { this.kylinConfig = kylinConfig; this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.intermediateTableDesc = intermediateTableDesc; this.dictionaryMap = dictionaryMap; this.rowKeyEncoder = rowKeyEncoder; this.aggrIngesters = aggrIngesters; measureDescList = cubeDesc.getMeasures(); measureCodec = new BufferedMeasureCodec(measureDescList); kvBuilder = new KeyValueBuilder(intermediateTableDesc); }
@Override public Iterator<Tuple2<RowKeyWritable, KeyValue>> call(Tuple2<Text, Text> textTextTuple2) throws Exception { List<Tuple2<RowKeyWritable, KeyValue>> result = Lists.newArrayListWithExpectedSize(cfNum); Object[] inputMeasures = new Object[cubeDesc.getMeasures().size()]; inputCodec.decode(ByteBuffer.wrap(textTextTuple2._2.getBytes(), 0, textTextTuple2._2.getLength()), inputMeasures); for (int i = 0; i < cfNum; i++) { KeyValue outputValue = keyValueCreators.get(i).create(textTextTuple2._1, inputMeasures); result.add(new Tuple2<>(new RowKeyWritable(outputValue.createKeyOnly(false).getKey()), outputValue)); } return result.iterator(); } });
public void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); cubeDesc = cubeInstance.getDescriptor(); aggregators = new MeasureAggregators(cubeDesc.getMeasures()); measureNum = cubeDesc.getMeasures().size(); } }
public void initAsSeparatedColumns(CubeDesc cubeDesc) { cubeRef = cubeDesc; int cfNum = cubeDesc.getMeasures().size(); columnFamily = new HBaseColumnFamilyDesc[cfNum]; for (int i = 0; i < cfNum; i++) { HBaseColumnFamilyDesc cf = new HBaseColumnFamilyDesc(); HBaseColumnDesc col = new HBaseColumnDesc(); String measureRef = cubeDesc.getMeasures().get(i).getName(); col.setMeasureRefs(new String[] { measureRef }); col.setQualifier("M"); cf.setColumns(new HBaseColumnDesc[] { col }); cf.setName("F" + (i + 1)); columnFamily[i] = cf; } }
private FunctionDesc findAggrFuncFromCubeDesc(FunctionDesc aggrFunc) { for (MeasureDesc measure : cubeDesc.getMeasures()) { if (measure.getFunction().equals(aggrFunc)) return measure.getFunction(); } return aggrFunc; }
@Test(expected = IllegalStateException.class) public void testValidateMeasureNamesDuplicated() throws IOException { File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json"); CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); MeasureDesc measureDescDuplicated = desc.getMeasures().get(1); List<MeasureDesc> newMeasures = Lists.newArrayList(desc.getMeasures()); newMeasures.add(measureDescDuplicated); desc.setMeasures(newMeasures); desc.init(config); } }
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { this.kylinConfig = kylinConfig; this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.intermediateTableDesc = intermediateTableDesc; this.dictionaryMap = dictionaryMap; Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid); measureDescList = cubeDesc.getMeasures(); aggrIngesters = MeasureIngester.create(measureDescList); measureCodec = new BufferedMeasureCodec(measureDescList); kvBuilder = new KeyValueBuilder(intermediateTableDesc); }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
@Before public void setup() throws Exception { this.createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); codec = new MeasureCodec(desc.getMeasures()); }
@Test public void testBadInit15() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs( new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }
@Test public void testBadInit14() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); String measureInfoForTransCnt = measureForTransCnt.toString(); thrown.expectMessage( "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }