private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
public Set<Long> generateMandatoryCuboids(List<Set<String>> mandatoryDimensionSetList) { Map<String, RowKeyColDesc> rowKeyColDescMap = Maps.newHashMap(); for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) { rowKeyColDescMap.put(entry.getColumn(), entry); } Set<Long> mandatoryCuboids = Sets.newHashSetWithExpectedSize(mandatoryDimensionSetList.size()); for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) { long cuboid = 0L; for (String columnName : mandatoryDimensionSet) { TblColRef tblColRef = model.findColumn(columnName); RowKeyColDesc rowKeyColDesc = rowKeyColDescMap.get(tblColRef.getIdentity()); // check if mandatory dimension set list is valid if (rowKeyColDesc == null) { logger.warn("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); throw new IllegalStateException( "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); } cuboid |= 1L << rowKeyColDesc.getBitIndex(); } mandatoryCuboids.add(cuboid); } return mandatoryCuboids; }
public static RowKeyColDesc getRowKeyColDesc(String tableName, String column, int id) { RowKeyColDesc rowKeyColDesc = new RowKeyColDesc(); rowKeyColDesc.setIndex(Integer.toString(id)); rowKeyColDesc.setColumn(tableName.substring(tableName.lastIndexOf(".") + 1) + "." + column); rowKeyColDesc.setEncoding(DictionaryDimEnc.ENCODING_NAME); rowKeyColDesc.setShardBy(false); return rowKeyColDesc; }
private void buildRowKey() { columnMap = new HashMap<>(); shardByColumns = new HashSet<>(); for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; rowKeyColDesc.init(rowkeyColumns.length - i - 1, cubeDesc); columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isShardBy()) { shardByColumns.add(rowKeyColDesc.getColRef()); } } this.fullMask = 0L; for (int i = 0; i < this.rowkeyColumns.length; i++) { int index = rowkeyColumns[i].getBitIndex(); this.fullMask |= 1L << index; } }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
/** * Get dimensions that have dictionary */ public Set<TblColRef> getAllDimsHaveDictionary() { Set<TblColRef> result = Sets.newHashSet(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } return result; }
private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) { RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns(); int[] ret = new int[rowKeyColDescs.length]; for (int i = 0; i < rowKeyColDescs.length; ++i) { RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i]; if (rowKeyColDesc.getDictionary() == null) { if (rowKeyColDesc.getLength() == 0) throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0"); ret[i] = rowKeyColDesc.getLength(); } else { ret[i] = estimateDictionaryColSpace(cardinality[i]); } } return ret; }
public int getColumnBitIndex(TblColRef col) { return getColDesc(col).getBitIndex(); }
private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) { DataModelDesc modelDesc = newModel.getModel(); Map<String, String> pkToFK = Maps.newHashMap(); for (LookupDesc lookupDesc : modelDesc.getLookups()) { if (lookupDesc.getJoin() != null) { JoinDesc join = lookupDesc.getJoin(); for (int i = 0; i < join.getForeignKey().length; i++) { pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]); } } } RowKeyDesc rowKey = newModel.getRowkey(); for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) { if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0) rowkeyCol.setDictionary("true"); if (pkToFK.containsKey(rowkeyCol.getColumn())) { rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn())); } } for (String[] aggregationGroup : rowKey.getAggregationGroups()) { for (int i = 0; i < aggregationGroup.length; i++) { if (pkToFK.containsKey(aggregationGroup[i])) { aggregationGroup[i] = pkToFK.get(aggregationGroup[i]); } } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getColumn() == null || rd.getColumn().length() == 0) { context.addResult(ResultLevel.ERROR, "Rowkey column empty"); } } } }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
@Override public void validate(CubeDesc cube, ValidateContext context) { Set<String> mands = new HashSet<String>(); RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns(); if (cols == null || cols.length == 0) { return; } for (int i = 0; i < cols.length; i++) { RowKeyColDesc rowKeyColDesc = cols[i]; if (rowKeyColDesc.isMandatory()) { mands.add(rowKeyColDesc.getColumn()); } } if (mands.isEmpty()) { return; } String[][] groups = cube.getRowkey().getAggregationGroups(); for (int i = 0; i < groups.length; i++) { String[] group = groups[i]; for (int j = 0; j < group.length; j++) { String col = group[j]; if (mands.contains(col)) { context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]"); } } } }
@Test public void testBadDesc() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getRowkey().getRowKeyColumns()[2].setColumn(""); IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); rule.validate(desc, vContext); vContext.print(System.out); assertTrue(vContext.getResults().length == 1); assertTrue("Rowkey column empty".equalsIgnoreCase(vContext.getResults()[0].getMessage())); } }
private void updateRowkeyDictionary(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { RowKeyDesc rowKey = newModel.getRowkey(); for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) { if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0) rowkeyCol.setDictionary("true"); } }
public String getDictionary(TblColRef col) { return getColDesc(col).getDictionary(); }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
private void buildRowKey() { columnMap = new HashMap<>(); shardByColumns = new HashSet<>(); for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; rowKeyColDesc.init(rowkeyColumns.length - i - 1, cubeDesc); columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isShardBy()) { shardByColumns.add(rowKeyColDesc.getColRef()); } } this.fullMask = 0L; for (int i = 0; i < this.rowkeyColumns.length; i++) { int index = rowkeyColumns[i].getBitIndex(); this.fullMask |= 1L << index; } }
public TblColRef getColumnByBitIndex(int bitIndex) { RowKeyColDesc[] rowKeyColumns = this.getRowkey().getRowKeyColumns(); return rowKeyColumns[rowKeyColumns.length - 1 - bitIndex].getColRef(); }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } if(rcd.length == 0){ context.addResult(ResultLevel.ERROR, "Rowkey columns is empty"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute"); } if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) { context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty"); } } }
public int getColumnBitIndex(TblColRef col) { return getColDesc(col).getBitIndex(); }