public static List<TblColRef> getExtendedColumnHosts(FunctionDesc functionDesc) { List<TblColRef> ret = Lists.newArrayList(); List<TblColRef> params = functionDesc.getParameter().getColRefs(); for (int i = 0; i < params.size() - 1; i++) { ret.add(params.get(i)); } return ret; }
@Override public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) { TblColRef literalCol = functionDesc.getParameter().getColRefs().get(0); return Collections.singletonList(literalCol); }
private TblColRef getRawColumn(FunctionDesc functionDesc) { return functionDesc.getParameter().getColRefs().get(0); }
public static TblColRef getExtendedColumn(FunctionDesc functionDesc) { List<TblColRef> params = functionDesc.getParameter().getColRefs(); return params.get(params.size() - 1); }
@Override public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) { if (needDictionaryColumn(functionDesc)) { return Collections.singletonList(functionDesc.getParameter().getColRefs().get(0)); } else { return Collections.emptyList(); } }
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) { for (MeasureDesc measureDesc : measureDescs) { sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs()); sqlDigest.aggregations.remove(measureDesc.getFunction()); } } }
private List<TblColRef> getTopNLiteralColumn(FunctionDesc functionDesc) { List<TblColRef> allColumns = functionDesc.getParameter().getColRefs(); if (functionDesc.getParameter().isColumnType() == false) { return allColumns; } return allColumns.subList(1, allColumns.size()); }
@Override public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) { List<TblColRef> columnsNeedDict = Lists.newArrayList(); List<TblColRef> allCols = functionDesc.getParameter().getColRefs(); int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0; for (int i = start; i < allCols.size(); i++) { TblColRef tblColRef = allCols.get(i); String encoding = getEncoding(functionDesc, tblColRef).getFirst(); if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { columnsNeedDict.add(tblColRef); } } return columnsNeedDict; }
private boolean belongToTable(FunctionDesc func, String table, DataModelDesc model) { // measure belong to the first column parameter's table List<TblColRef> cols = func.getParameter().getColRefs(); String belongTo = cols.isEmpty() ? model.getRootFactTable().getTableIdentity() : cols.get(0).getTable(); return belongTo.equals(table); }
private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) { if (functionDesc.getParameter().isColumnType() == true) { return functionDesc.getParameter().getColRefs().get(0); } return null; }
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) { Iterator<FunctionDesc> it = unmatchedAggregations.iterator(); while (it.hasNext()) { FunctionDesc functionDesc = it.next(); // let calcite handle count if (functionDesc.isCount()) { it.remove(); continue; } // calcite can do aggregation from columns on-the-fly ParameterDesc parameterDesc = functionDesc.getParameter(); if (parameterDesc == null) { continue; } List<TblColRef> neededCols = parameterDesc.getColRefs(); if (neededCols.size() > 0 && dimCols.containsAll(neededCols) && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) { result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc)); it.remove(); continue; } } }
private boolean needDictionaryColumn(FunctionDesc functionDesc) { DataType dataType = functionDesc.getParameter().getColRefs().get(0).getType(); if (dataType.isIntegerFamily() && !dataType.isBigInt()) { return false; } return true; }
@Override public BitmapCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { checkArgument(values.length == 1, "expect 1 value, got %s", Arrays.toString(values)); current.clear(); if (values[0] == null) { return current; } int id; if (needDictionaryColumn(measureDesc.getFunction())) { TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(0); Dictionary<String> dictionary = dictionaryMap.get(literalCol); id = dictionary.getIdFromValue(values[0]); } else { id = Integer.parseInt(values[0]); } current.add(id); return current; }
private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) { if (sum == null) return false; if (!isTopN(topN)) return false; TblColRef topnNumCol = getTopNNumericColumn(topN); if (topnNumCol == null) { if (sum.isCount()) return true; return false; } if (sum.isSum() == false) return false; if (sum.getParameter() == null || sum.getParameter().getColRefs() == null || sum.getParameter().getColRefs().size() == 0) return false; TblColRef sumCol = sum.getParameter().getColRefs().get(0); return sumCol.equals(topnNumCol); }
private void parseCubeDesc() { Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); // build index for rowkey columns List<TblColRef> cuboidColumns = baseCuboid.getColumns(); int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length; rowKeyColumnIndexes = new int[rowkeyColCount]; for (int i = 0; i < rowkeyColCount; i++) { TblColRef col = cuboidColumns.get(i); rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col); } List<MeasureDesc> measures = cubeDesc.getMeasures(); int measureSize = measures.size(); measureColumnIndexes = new int[measureSize][]; for (int i = 0; i < measureSize; i++) { FunctionDesc func = measures.get(i).getFunction(); List<TblColRef> colRefs = func.getParameter().getColRefs(); if (colRefs == null) { measureColumnIndexes[i] = null; } else { measureColumnIndexes[i] = new int[colRefs.size()]; for (int j = 0; j < colRefs.size(); j++) { TblColRef c = colRefs.get(j); measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c); } } } }
@SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return; } for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase(Locale.ROOT)); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT)); } FunctionDesc func = m.getFunction(); func.init(model); allColumns.addAll(func.getParameter().getColRefs()); if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) { FunctionDesc functionDesc = m.getFunction(); List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc); TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc); initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn); } } }
public DataType getRewriteFieldType() { if (getMeasureType() instanceof BasicMeasureType) { if (isMax() || isMin()) { return parameter.getColRefs().get(0).getType(); } else if (isSum()) { return parameter.isColumnType() ? DataType.getType(returnType) : DataType.getType("bigint"); } else if (isCount()) { return DataType.getType("bigint"); } else { throw new IllegalArgumentException("unknown measure type " + getMeasureType()); } } else { return DataType.ANY; } }
void buildRewriteFieldsAndMetricsColumns() { ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType(); RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); for (int i = 0; i < this.aggregations.size(); i++) { FunctionDesc aggFunc = this.aggregations.get(i); if (aggFunc.isDimensionAsMetric()) { addToContextGroupBy(aggFunc.getParameter().getColRefs()); continue; // skip rewrite, let calcite handle } if (aggFunc.needRewriteField()) { String rewriteFieldName = aggFunc.getRewriteFieldName(); RelDataType rewriteFieldType = OLAPTable.createSqlType(typeFactory, aggFunc.getRewriteFieldType(), true); this.context.rewriteFields.put(rewriteFieldName, rewriteFieldType); TblColRef column = buildRewriteColumn(aggFunc); this.context.metricsColumns.add(column); } AggregateCall aggCall = this.rewriteAggCalls.get(i); if (!aggCall.getArgList().isEmpty()) { for (Integer index : aggCall.getArgList()) { TblColRef column = inputColumnRowType.getColumnByIndex(index); if (!column.isInnerColumn() && this.context.belongToContextTables(column)) { this.context.metricsColumns.add(column); } } } } }
for (int i = 0; i < measureSize; i++) { FunctionDesc func = measures.get(i).getFunction(); List<TblColRef> colRefs = func.getParameter().getColRefs(); if (colRefs != null) { for (int j = 0; j < colRefs.size(); j++) {
@Test public void test() { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; for (MeasureDesc measureDesc : desc.getMeasures()) { if (measureDesc.getName().equals("TOP_SELLER")) { topSellerMeasure = measureDesc; break; } } TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); } }