public static boolean isCountDistinct(FunctionDesc func) { return FUNC_COUNT_DISTINCT.equalsIgnoreCase(func.getExpression()); }
private boolean isTopN(FunctionDesc functionDesc) { return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression()); }
private static void removeUnmatchedGroupingAgg(Collection<FunctionDesc> unmatchedAggregations) { if (CollectionUtils.isEmpty(unmatchedAggregations)) return; Iterator<FunctionDesc> iterator = unmatchedAggregations.iterator(); while (iterator.hasNext()) { if (FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(iterator.next().getExpression())) { iterator.remove(); } } }
public String[] makeAggrFuncs(Collection<FunctionDesc> metrics) { //metrics are represented in ImmutableBitSet, which loses order information //sort the aggrFuns to align with metrics natural order List<FunctionDesc> metricList = Lists.newArrayList(metrics); Collections.sort(metricList, new Comparator<FunctionDesc>() { @Override public int compare(FunctionDesc o1, FunctionDesc o2) { int a = getIndexOf(o1); int b = getIndexOf(o2); return a - b; } }); String[] result = new String[metricList.size()]; int i = 0; for (FunctionDesc metric : metricList) { result[i++] = metric.getExpression(); } return result; }
@Override public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { checkArgument(FUNC_COUNT_DISTINCT.equals(functionDesc.getExpression()), "BitmapMeasureType only support function %s, got %s", FUNC_COUNT_DISTINCT, functionDesc.getExpression()); checkArgument(functionDesc.getParameterCount() == 1, "BitmapMeasureType only support 1 parameter, got %d", functionDesc.getParameterCount()); String returnType = functionDesc.getReturnDataType().getName(); checkArgument(DATATYPE_BITMAP.equals(returnType), "BitmapMeasureType's return type must be %s, got %s", DATATYPE_BITMAP, returnType); }
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) { for (FunctionDesc func : sqlDigest.aggregations) { if (!func.isDimensionAsMetric() && !FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(func.getExpression())) { // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision metrics.add(findAggrFuncFromCubeDesc(func)); } } for (TblColRef column : sqlDigest.allColumns) { // skip measure columns if ((sqlDigest.metricColumns.contains(column) || sqlDigest.rtMetricColumns.contains(column)) && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column) || sqlDigest.rtDimensionColumns.contains(column))) { continue; } dimensions.add(column); } }
public InMemCubeBuilder2(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); }
public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { validate(functionDesc.getExpression(), functionDesc.getReturnDataType(), true); }
public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { validate(functionDesc.getExpression(), functionDesc.getReturnDataType(), true); }
public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { validate(functionDesc.getExpression(), functionDesc.getReturnDataType(), true); }
public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { super(cuboidScheduler, flatDesc, dictionaryMap); this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); this.totalCuboidCount = cuboidScheduler.getCuboidCount(); this.measureCount = cubeDesc.getMeasures().size(); this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]); List<String> metricsAggrFuncsList = Lists.newArrayList(); for (int i = 0; i < measureCount; i++) { MeasureDesc measureDesc = measureDescs[i]; metricsAggrFuncsList.add(measureDesc.getFunction().getExpression()); } this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]); }
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) { Iterator<FunctionDesc> it = unmatchedAggregations.iterator(); while (it.hasNext()) { FunctionDesc functionDesc = it.next(); // let calcite handle count if (functionDesc.isCount()) { it.remove(); continue; } // calcite can do aggregation from columns on-the-fly ParameterDesc parameterDesc = functionDesc.getParameter(); if (parameterDesc == null) { continue; } List<TblColRef> neededCols = parameterDesc.getColRefs(); if (neededCols.size() > 0 && dimCols.containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); it.remove(); continue; } } }
private List<MeasureDesc> dropPercentile(List<MeasureDesc> measures) { ArrayList<MeasureDesc> result = new ArrayList<>(); for (MeasureDesc m : measures) { if (!m.getFunction().getExpression().toUpperCase(Locale.ROOT).contains("PERCENTILE")) result.add(m); } return result; }
private void reInitMeasureType() { if (isDimensionAsMetric && isCountDistinct()) { // create DimCountDis measureType = MeasureTypeFactory.createNoRewriteFieldsMeasureType(getExpression(), getReturnDataType()); returnDataType = DataType.getType("dim_dc"); } else { measureType = MeasureTypeFactory.create(getExpression(), getReturnDataType()); } }
for (MeasureDesc m : c.getMeasures()) { FunctionDesc f = m.getFunction(); if (f.getExpression().equals(PercentileMeasureType.FUNC_PERCENTILE)) { f.setExpression(PercentileMeasureType.FUNC_PERCENTILE_APPROX);
for (MeasureDesc measureDesc : measureDescList) { if (measureDesc.getFunction().isCountDistinct() || measureDesc.getFunction().getExpression().equals(PercentileMeasureType.FUNC_PERCENTILE)) { largeMeasureList.add(measureDesc.getName()); } else {
@SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return; } for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase(Locale.ROOT)); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT)); } FunctionDesc func = m.getFunction(); func.init(model); allColumns.addAll(func.getParameter().getColRefs()); if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) { FunctionDesc functionDesc = m.getFunction(); List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc); TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc); initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn); } } }
for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) { if (measure.getFunction().getMeasureType().isMemoryHungry()) { logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression()); memoryHungryMeasures++;
if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(measureDesc.getFunction().getExpression())) {
@Test public void test() { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; for (MeasureDesc measureDesc : desc.getMeasures()) { if (measureDesc.getName().equals("TOP_SELLER")) { topSellerMeasure = measureDesc; break; } } TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); } }