public boolean needRewrite() { if (getMeasureType() == null) return false; return getMeasureType().needRewrite(); }
public boolean hasMemHungryMeasures() { for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) { FunctionDesc func = measures[i].getFunction(); if (func.getMeasureType().isMemoryHungry()) return true; } return false; }
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) { Map<String, List<MeasureDesc>> map = Maps.newHashMap(); for (MeasureDesc measure : cubeDesc.getMeasures()) { MeasureType<?> measureType = measure.getFunction().getMeasureType(); String key = measureType.getClass().getCanonicalName(); List<MeasureDesc> temp = null; if ((temp = map.get(key)) != null) { temp.add(measure); } else { map.put(key, Lists.<MeasureDesc> newArrayList(measure)); } } for (List<MeasureDesc> sublist : map.values()) { sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest); } }
public boolean needRewriteField() { if (!needRewrite()) return false; return getMeasureType().needRewriteField(); }
public boolean hasMemoryHungryMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().getMeasureType().isMemoryHungry()) { return true; } } return false; }
public static MeasureIngester<?> create(MeasureDesc measure) { return measure.getFunction().getMeasureType().newIngester(); }
public boolean isMemoryHungry() { for (HBaseColumnDesc hBaseColumnDesc : columns) { for (MeasureDesc measureDesc : hBaseColumnDesc.getMeasures()) { if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) { return true; } } } return false; }
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) { CubeDesc cubeDesc = cube.getDescriptor(); List<String> influencingMeasures = Lists.newArrayList(); for (MeasureDesc measure : cubeDesc.getMeasures()) { // if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty()) // break; MeasureType<?> measureType = measure.getFunction().getMeasureType(); if (measureType instanceof BasicMeasureType) continue; CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure); if (inf != null) { result.influences.add(inf); influencingMeasures.add(measure.getName() + "@" + measureType.getClass()); } } if (influencingMeasures.size() != 0) logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(), StringUtils.join(influencingMeasures, ",")); }
public MeasureAggregators(MeasureDesc... measureDescs) { descLength = measureDescs.length; aggs = new MeasureAggregator[descLength]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descLength; i++) { FunctionDesc func = measureDescs[i].getFunction(); aggs[i] = func.getMeasureType().newAggregator(); measureIndexMap.put(measureDescs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descLength; i++) { String depMsrRef = measureDescs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
/** * Get columns that have dictionary */ public Set<TblColRef> getAllColumnsHaveDictionary() { Set<TblColRef> result = Sets.newLinkedHashSet(); // dictionaries in dimensions result.addAll(getAllDimsHaveDictionary()); // dictionaries in measures for (MeasureDesc measure : measures) { MeasureType<?> aggrType = measure.getFunction().getMeasureType(); result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction())); } // any additional dictionaries if (dictionaries != null) { for (DictionaryDesc dictDesc : dictionaries) { TblColRef col = dictDesc.getColumnRef(); result.add(col); } } return result; }
public DataType getRewriteFieldType() { if (getMeasureType() instanceof BasicMeasureType) { if (isMax() || isMin()) { return parameter.getColRefs().get(0).getType(); } else if (isSum()) { return parameter.isColumnType() ? DataType.getType(returnType) : DataType.getType("bigint"); } else if (isCount()) { return DataType.getType("bigint"); } else { throw new IllegalArgumentException("unknown measure type " + getMeasureType()); } } else { return DataType.ANY; } }
public static long toCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) { for (FunctionDesc metric : metrics) { if (metric.getMeasureType().onlyAggrInBaseCuboid()) return Cuboid.getBaseCuboidId(cubeDesc); } long cuboidID = 0; for (TblColRef column : dimensions) { int index = cubeDesc.getRowkey().getColumnBitIndex(column); cuboidID |= 1L << index; } return cuboidID; }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(Locale.ROOT); // only used in Build job, not in Merge job cuboidLevel = context.getConfiguration().getInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 0); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor(); measuresDescs = cubeDesc.getMeasures(); codec = new BufferedMeasureCodec(measuresDescs); aggs = new MeasureAggregators(measuresDescs); input = new Object[measuresDescs.size()]; result = new Object[measuresDescs.size()]; List<Integer> needAggMeasuresList = Lists.newArrayList(); for (int i = 0; i < measuresDescs.size(); i++) { if (cuboidLevel == 0) { needAggMeasuresList.add(i); } else { if (!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid()) { needAggMeasuresList.add(i); } } } needAggrMeasures = new int[needAggMeasuresList.size()]; for (int i = 0; i < needAggMeasuresList.size(); i++) { needAggrMeasures[i] = needAggMeasuresList.get(i); } }
if (cubeFunc.getMeasureType() != null && cubeFunc.getMeasureType().needRewriteField()) { aggCall = new KylinAggregateCall(aggCall, cubeFunc);
int memoryHungryMeasures = 0; for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) { if (measure.getFunction().getMeasureType().isMemoryHungry()) { logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression()); memoryHungryMeasures++;
for (int i = 0; i < measureDescs.size(); i++) { MeasureDesc measureDesc = measureDescs.get(i); MeasureType measureType = measureDesc.getFunction().getMeasureType(); List<TblColRef> columns = measureType.getColumnsNeedDictionary(measureDesc.getFunction()); boolean needReEncode = false;
SqlAggFunction newAgg = aggCall.getAggregation(); Map<String, Class<?>> udafMap = func.getMeasureType().getRewriteCalciteAggrFunctions(); if (func.isCount()) { newAgg = SqlStdOperatorTable.SUM0;
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
Field field = FunctionDesc.class.getDeclaredField("measureType"); field.setAccessible(true); MeasureType origMeasureType = functionDesc.getMeasureType(); field.set(functionDesc, new MockUpMeasureType(origMeasureType));
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs) .setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }