/** * @return all functions from each measure. */ public List<FunctionDesc> listAllFunctions() { List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); for (MeasureDesc m : measures) { functions.add(m.getFunction()); } return functions; }
FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) { for (MeasureDesc m : measures) { if (aggFunc.equals(m.getFunction())) return m.getFunction(); } return aggFunc; }
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) { for (MeasureDesc measureDesc : measureDescs) { sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs()); sqlDigest.aggregations.remove(measureDesc.getFunction()); } } }
public int findMeasure(FunctionDesc function) { for (int i = 0; i < measures.length; i++) { if (measures[i].getFunction().equals(function)) { return i; } } return -1; }
private FunctionDesc findAggrFuncFromCubeDesc(FunctionDesc aggrFunc) { for (MeasureDesc measure : cubeDesc.getMeasures()) { if (measure.getFunction().equals(aggrFunc)) return measure.getFunction(); } return aggrFunc; }
public boolean hasMemHungryMeasures() { for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) { FunctionDesc func = measures[i].getFunction(); if (func.getMeasureType().isMemoryHungry()) return true; } return false; }
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) { Map<String, List<MeasureDesc>> map = Maps.newHashMap(); for (MeasureDesc measure : cubeDesc.getMeasures()) { MeasureType<?> measureType = measure.getFunction().getMeasureType(); String key = measureType.getClass().getCanonicalName(); List<MeasureDesc> temp = null; if ((temp = map.get(key)) != null) { temp.add(measure); } else { map.put(key, Lists.<MeasureDesc> newArrayList(measure)); } } for (List<MeasureDesc> sublist : map.values()) { sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest); } }
public InMemCubeBuilder2(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); }
public MeasureCodec(MeasureDesc... measureDescs) { String[] dataTypes = new String[measureDescs.length]; for (int i = 0; i < dataTypes.length; i++) { dataTypes[i] = measureDescs[i].getFunction().getReturnType(); } init(dataTypes); }
public boolean hasMemoryHungryMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().getMeasureType().isMemoryHungry()) { return true; } } return false; }
public static MeasureIngester<?> create(MeasureDesc measure) { return measure.getFunction().getMeasureType().newIngester(); }
public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); this.totalCuboidCount = cuboidScheduler.getCuboidCount(); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); }
@SuppressWarnings("unused") MeasureDesc findMeasure(TblColRef col) { for (MeasureDesc measure : this.context.realization.getMeasures()) { if (col.getName().equals(measure.getFunction().getRewriteFieldName())) { return measure; } } return null; }
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MeasureDesc that = (MeasureDesc) o; if (!function.equals(that.getFunction())) return false; if (dependentMeasureRef != null && that.getDependentMeasureRef() == null || dependentMeasureRef == null && that.getDependentMeasureRef() != null) return false; if (dependentMeasureRef == null && that.getDependentMeasureRef() == null) return true; return dependentMeasureRef.equals(that.getDependentMeasureRef()); }
private List<MeasureDesc> dropPercentile(List<MeasureDesc> measures) { ArrayList<MeasureDesc> result = new ArrayList<>(); for (MeasureDesc m : measures) { if (!m.getFunction().getExpression().toUpperCase(Locale.ROOT).contains("PERCENTILE")) result.add(m); } return result; }
public boolean isMemoryHungry() { for (HBaseColumnDesc hBaseColumnDesc : columns) { for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { return true; } } } return false; }
@Override public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length != 1) throw new IllegalArgumentException(); //source input column value String literal = values[0]; // encode literal using dictionary TblColRef literalCol = getRawColumn(measureDesc.getFunction()); Dictionary<String> dictionary = dictionaryMap.get(literalCol); int keyEncodedValue = dictionary.getIdFromValue(literal); ByteArray key = new ByteArray(dictionary.getSizeOfId()); BytesUtil.writeUnsigned(keyEncodedValue, key.array(), key.offset(), dictionary.getSizeOfId()); List<ByteArray> valueList = new ArrayList<ByteArray>(1); valueList.add(key); return valueList; }
public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) { Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>(); HBaseMappingDesc hbaseMapping = cubeRef.getHbaseMapping(); if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) { return result; } for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { for (MeasureDesc m : c.getMeasures()) { if (m.getFunction().equals(function)) { result.add(c); } } } } return result; }
@Before public void setUp() throws Exception { this.createTestMetadata(); cube = getTestKylinCubeWithSeller(); cubeDesc = cube.getDescriptor(); dimensions = Sets.newHashSet(); metrics = Lists.newArrayList(); for (MeasureDesc measureDesc : cubeDesc.getMeasures()) { Collections.addAll(metrics, measureDesc.getFunction()); } FunctionDesc mockUpFuncDesc = new FunctionDesc(); Field field = FunctionDesc.class.getDeclaredField("measureType"); field.setAccessible(true); field.set(mockUpFuncDesc, new MockUpMeasureType()); metrics.add(mockUpFuncDesc); }
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }