public Set<Long> generateMandatoryCuboids(List<Set<String>> mandatoryDimensionSetList) { Map<String, RowKeyColDesc> rowKeyColDescMap = Maps.newHashMap(); for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) { rowKeyColDescMap.put(entry.getColumn(), entry); } Set<Long> mandatoryCuboids = Sets.newHashSetWithExpectedSize(mandatoryDimensionSetList.size()); for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) { long cuboid = 0L; for (String columnName : mandatoryDimensionSet) { TblColRef tblColRef = model.findColumn(columnName); RowKeyColDesc rowKeyColDesc = rowKeyColDescMap.get(tblColRef.getIdentity()); // check if mandatory dimension set list is valid if (rowKeyColDesc == null) { logger.warn("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); throw new IllegalStateException( "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); } cuboid |= 1L << rowKeyColDesc.getBitIndex(); } mandatoryCuboids.add(cuboid); } return mandatoryCuboids; }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getColumn() == null || rd.getColumn().length() == 0) { context.addResult(ResultLevel.ERROR, "Rowkey column empty"); } } } }
for (int i = 0; i < rowKeyColDescList.length; i++) { if ((cuboid & (1L << rowKeyColDescList[i].getBitIndex())) > 0) { dimensionSet.add(rowKeyColDescList[i].getColumn());
public int getRowKeyIndexByColumnName(String columnName) { if (this.rowkeyColumns == null) return -1; for (int i = 0; i < this.rowkeyColumns.length; ++i) { RowKeyColDesc desc = this.rowkeyColumns[i]; if (desc.getColumn().equalsIgnoreCase(columnName)) { return i; } } return -1; }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) { RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns(); int[] ret = new int[rowKeyColDescs.length]; for (int i = 0; i < rowKeyColDescs.length; ++i) { RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i]; if (rowKeyColDesc.getDictionary() == null) { if (rowKeyColDesc.getLength() == 0) throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0"); ret[i] = rowKeyColDesc.getLength(); } else { ret[i] = estimateDictionaryColSpace(cardinality[i]); } } return ret; }
@Override public void validate(CubeDesc cube, ValidateContext context) { Set<String> mands = new HashSet<String>(); RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns(); if (cols == null || cols.length == 0) { return; } for (int i = 0; i < cols.length; i++) { RowKeyColDesc rowKeyColDesc = cols[i]; if (rowKeyColDesc.isMandatory()) { mands.add(rowKeyColDesc.getColumn()); } } if (mands.isEmpty()) { return; } String[][] groups = cube.getRowkey().getAggregationGroups(); for (int i = 0; i < groups.length; i++) { String[] group = groups[i]; for (int j = 0; j < group.length; j++) { String col = group[j]; if (mands.contains(col)) { context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]"); } } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } if(rcd.length == 0){ context.addResult(ResultLevel.ERROR, "Rowkey columns is empty"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute"); } if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty"); } } }
public int getRowKeyIndexByColumnName(String columnName) { if (this.rowkeyColumns == null) return -1; for (int i = 0; i < this.rowkeyColumns.length; ++i) { RowKeyColDesc desc = this.rowkeyColumns[i]; if (desc.getColumn().equalsIgnoreCase(columnName)) { return i; } } return -1; }
public Set<Long> generateMandatoryCuboids(List<Set<String>> mandatoryDimensionSetList) { Map<String, RowKeyColDesc> rowKeyColDescMap = Maps.newHashMap(); for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) { rowKeyColDescMap.put(entry.getColumn(), entry); } Set<Long> mandatoryCuboids = Sets.newHashSetWithExpectedSize(mandatoryDimensionSetList.size()); for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) { long cuboid = 0L; for (String columnName : mandatoryDimensionSet) { TblColRef tblColRef = model.findColumn(columnName); RowKeyColDesc rowKeyColDesc = rowKeyColDescMap.get(tblColRef.getIdentity()); // check if mandatory dimension set list is valid if (rowKeyColDesc == null) { logger.warn("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); throw new IllegalStateException( "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); } cuboid |= 1L << rowKeyColDesc.getBitIndex(); } mandatoryCuboids.add(cuboid); } return mandatoryCuboids; }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getColumn() == null || rd.getColumn().length() == 0) { context.addResult(ResultLevel.ERROR, "Rowkey column empty"); } } } }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) { RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns(); int[] ret = new int[rowKeyColDescs.length]; for (int i = 0; i < rowKeyColDescs.length; ++i) { RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i]; if (rowKeyColDesc.getDictionary() == null) { if (rowKeyColDesc.getLength() == 0) throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0"); ret[i] = rowKeyColDesc.getLength(); } else { ret[i] = estimateDictionaryColSpace(cardinality[i]); } } return ret; }
@Override public void validate(CubeDesc cube, ValidateContext context) { Set<String> mands = new HashSet<String>(); RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns(); if (cols == null || cols.length == 0) { return; } for (int i = 0; i < cols.length; i++) { RowKeyColDesc rowKeyColDesc = cols[i]; if (rowKeyColDesc.isMandatory()) { mands.add(rowKeyColDesc.getColumn()); } } if (mands.isEmpty()) { return; } String[][] groups = cube.getRowkey().getAggregationGroups(); for (int i = 0; i < groups.length; i++) { String[] group = groups[i]; for (int j = 0; j < group.length; j++) { String col = group[j]; if (mands.contains(col)) { context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]"); } } } }
private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) { DataModelDesc modelDesc = newModel.getModel(); Map<String, String> pkToFK = Maps.newHashMap(); for (LookupDesc lookupDesc : modelDesc.getLookups()) { if (lookupDesc.getJoin() != null) { JoinDesc join = lookupDesc.getJoin(); for (int i = 0; i < join.getForeignKey().length; i++) { pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]); } } } RowKeyDesc rowKey = newModel.getRowkey(); for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) { if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0) rowkeyCol.setDictionary("true"); if (pkToFK.containsKey(rowkeyCol.getColumn())) { rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn())); } } for (String[] aggregationGroup : rowKey.getAggregationGroups()) { for (int i = 0; i < aggregationGroup.length; i++) { if (pkToFK.containsKey(aggregationGroup[i])) { aggregationGroup[i] = pkToFK.get(aggregationGroup[i]); } } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } if (rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns is empty"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary()) && !rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute"); } if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary()) || rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty"); } } }