public boolean needRewrite() { if (getMeasureType() == null) return false; return getMeasureType().needRewrite(); }
public static List<TblColRef> getExtendedColumnHosts(FunctionDesc functionDesc) { List<TblColRef> ret = Lists.newArrayList(); List<TblColRef> params = functionDesc.getParameter().getColRefs(); for (int i = 0; i < params.size() - 1; i++) { ret.add(params.get(i)); } return ret; }
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) { Iterator<FunctionDesc> it = unmatchedAggregations.iterator(); while (it.hasNext()) { FunctionDesc functionDesc = it.next(); // let calcite handle count if (functionDesc.isCount()) { it.remove(); continue; } // calcite can do aggregation from columns on-the-fly ParameterDesc parameterDesc = functionDesc.getParameter(); if (parameterDesc == null) { continue; } List<TblColRef> neededCols = parameterDesc.getColRefs(); if (neededCols.size() > 0 && dimCols.containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); it.remove(); continue; } } }
public String getRewriteFieldName() { if (isCount()) { return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same } else if (isCountDistinct()) { return "_KY_" + getFullExpressionInAlphabetOrder().replaceAll("[(),. ]", "_"); } else { return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_"); } }
private FunctionDesc constructSumFunction(TblColRef column) { FunctionDesc function = new FunctionDesc(); function.setParameter(ParameterDesc.newInstance(column)); function.setExpression(FUNC_SUM); function.setReturnType("decimal"); return function; } }
@Override public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { checkArgument(FUNC_COUNT_DISTINCT.equals(functionDesc.getExpression()), "BitmapMeasureType only support function %s, got %s", FUNC_COUNT_DISTINCT, functionDesc.getExpression()); checkArgument(functionDesc.getParameterCount() == 1, "BitmapMeasureType only support 1 parameter, got %d", functionDesc.getParameterCount()); String returnType = functionDesc.getReturnDataType().getName(); checkArgument(DATATYPE_BITMAP.equals(returnType), "BitmapMeasureType's return type must be %s, got %s", DATATYPE_BITMAP, returnType); }
MeasureDesc measure = it.next(); FunctionDesc func = measure.getFunction(); ParameterDesc parameter = func.getParameter(); if (parameter == null) { context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName()); return; String type = func.getParameter().getType(); String value = func.getParameter().getValue(); if (StringUtils.isEmpty(type)) { context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName()); return; context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName()); return; if (StringUtils.isEmpty(func.getReturnType())) { context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName()); return; func.getMeasureType().validate(func); } catch (IllegalArgumentException ex) { context.addResult(ResultLevel.ERROR, ex.getMessage()); if (func.isCount()) countFuncs.add(func); if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) { if (parameter.getNextParameter() == null) {
public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) { String[] funcNames = new String[metrics.size()]; String[] dataTypes = new String[metrics.size()]; MetricInfo[] metricInfos = new MetricInfo[metrics.size()]; for (int i = 0; i < metrics.size(); i++) { FunctionDesc functionDesc = metrics.get(i); //TODO: what if funcionDesc's type is different from tablDesc? cause scale difference funcNames[i] = functionDesc.getExpression(); dataTypes[i] = functionDesc.getReturnType(); if (functionDesc.isCount()) { metricInfos[i] = new MetricInfo(MetricType.Count); } else if (functionDesc.isDimensionAsMetric()) { metricInfos[i] = new MetricInfo(MetricType.DimensionAsMetric); } else { int index = tableInfo.findFactTableColumn(functionDesc.getParameter().getValue()); if (index < 0) { throw new IllegalStateException("Column " + functionDesc.getParameter().getValue() + " is not found in II"); } if (functionDesc.isCountDistinct()) { metricInfos[i] = new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision()); } else { metricInfos[i] = new MetricInfo(MetricType.Normal, index); } } } return new EndpointAggregators(funcNames, dataTypes, metricInfos, tableInfo.getDigest()); }
private void initMeasureColumns(Map<String, TableDesc> tables) { if (measures == null || measures.isEmpty()) { return; } TableDesc factTable = tables.get(getFactTable()); for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase()); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase()); } FunctionDesc f = m.getFunction(); f.setExpression(f.getExpression().toUpperCase()); f.setReturnDataType(DataType.getInstance(f.getReturnType())); ParameterDesc p = f.getParameter(); p.normalizeColumnValue(); if (p.isColumnType()) { ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (String cName : p.getValue().split("\\s*,\\s*")) { ColumnDesc sourceColumn = factTable.findColumnByName(cName); TblColRef colRef = new TblColRef(sourceColumn); colRefs.add(colRef); allColumns.add(colRef); } if (colRefs.isEmpty() == false) p.setColRefs(colRefs); } } }
private void validateReturnType(ValidateContext context, CubeDesc cube, FunctionDesc funcDesc) { String func = funcDesc.getExpression(); DataType rtype = funcDesc.getReturnDataType(); if (funcDesc.isCount()) { if (rtype.isIntegerFamily() == false) { context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.INTEGER_FAMILY); } } else if (funcDesc.isCountDistinct()) { if (rtype.isHLLC() == false && funcDesc.isHolisticCountDistinct() == false) { context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be hllc(10), hllc(12) etc."); } } else if (funcDesc.isMax() || funcDesc.isMin() || funcDesc.isSum()) { if (rtype.isNumberFamily() == false) { context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY); } } else { if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) { context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]"); } } }
if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(measureDesc.getFunction().getExpression())) { Map<String, String> configuration = measureDesc.getFunction().getConfiguration(); ParameterDesc parameter = measureDesc.getFunction().getParameter(); parameter = parameter.getNextParameter(); int keyLength = 0; DataType returnType = DataType.getType(measureDesc.getFunction().getReturnType()); DataType newReturnType = new DataType(returnType.getName(), returnType.getPrecision(), keyLength); measureDesc.getFunction().setReturnType(newReturnType.toString());
if (functionDesc.isCount()) { functionDesc.setReturnType("bigint"); functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType())); } else { boolean updated = false; for (TblColRef column : columns) { if (column.isSameAs(factTableName, functionDesc.getParameter().getValue())) { if (functionDesc.isCountDistinct()) { functionDesc.setReturnType(iiDefaultHLLC); functionDesc.setReturnDataType(DataType.getInstance(iiDefaultHLLC)); } else { functionDesc.setReturnType(column.getColumn().getType().toString()); functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType())); functionDesc.getParameter().setColRefs(ImmutableList.of(column)); updated = true; break;
public DataType getRewriteFieldType() { if (getMeasureType() instanceof BasicMeasureType) { if (isMax() || isMin()) { return parameter.getColRefs().get(0).getType(); } else if (isSum()) { return parameter.isColumnType() ? DataType.getType(returnType) : DataType.getType("bigint"); } else if (isCount()) { return DataType.getType("bigint"); } else { throw new IllegalArgumentException("unknown measure type " + getMeasureType()); } } else { return DataType.ANY; } }
public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { validate(functionDesc.getExpression(), functionDesc.getReturnDataType(), true); }
@Test public void test() { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; for (MeasureDesc measureDesc : desc.getMeasures()) { if (measureDesc.getName().equals("TOP_SELLER")) { topSellerMeasure = measureDesc; break; } } TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); } }
private void reInitMeasureType() { if (isDimensionAsMetric && isCountDistinct()) { // create DimCountDis measureType = MeasureTypeFactory.createNoRewriteFieldsMeasureType(getExpression(), getReturnDataType()); returnDataType = DataType.getType("dim_dc"); } else { measureType = MeasureTypeFactory.create(getExpression(), getReturnDataType()); } }
private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) { if (sum == null) return false; if (!isTopN(topN)) return false; TblColRef topnNumCol = getTopNNumericColumn(topN); if (topnNumCol == null) { if (sum.isCount()) return true; return false; } if (sum.isSum() == false) return false; if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0) return false; TblColRef sumCol = sum.getParameter().getColRefs().get(0); return sumCol.equals(topnNumCol); }
SqlAggFunction newAgg = aggCall.getAggregation(); Map<String, Class<?>> udafMap = func.getMeasureType().getRewriteCalciteAggrFunctions(); if (func.isCount()) { newAgg = SqlStdOperatorTable.SUM0; } else if (udafMap != null && udafMap.containsKey(callName)) { newArgList = truncArgList(newArgList, udafMap.get(callName)); if (func.needRewriteField()) { RelDataTypeField field = getInput().getRowType().getField(func.getRewriteFieldName(), true, false); if (newArgList.isEmpty()) { newArgList.add(field.getIndex());
public String getRewriteFieldName() { if (isSum()) { return getParameter().getValue(); } else if (isCount()) { return "COUNT__"; // ignores parameter, count(*), count(1), // count(col) are all the same } else { return getFullExpression().replaceAll("[(), ]", "_"); } }
public static boolean isCountDistinct(FunctionDesc func) { return FUNC_COUNT_DISTINCT.equalsIgnoreCase(func.getExpression()); }