/** * Get dimensions that have dictionary */ public Set<TblColRef> getAllDimsHaveDictionary() { Set<TblColRef> result = Sets.newHashSet(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } return result; }
private long getQueryFilterMask(Set<TblColRef> filterColumnD) { long filterMask = 0; logger.info("Filter column set for query: %s", filterColumnD); if (filterColumnD.isEmpty() == false) { RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { if (filterColumnD.contains(allColumns[i].getColRef())) { filterMask |= 1L << allColumns[i].getBitIndex(); } } } logger.info("Filter mask is: {0}", filterMask); return filterMask; }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
public int getRowKeyColumnCount() { return getDescriptor().getRowkey().getRowKeyColumns().length; }
public TblColRef getColumnByBitIndex(int bitIndex) { RowKeyColDesc[] rowKeyColumns = this.getRowkey().getRowKeyColumns(); return rowKeyColumns[rowKeyColumns.length - 1 - bitIndex].getColRef(); }
public Set<Long> generateMandatoryCuboids(List<Set<String>> mandatoryDimensionSetList) { Map<String, RowKeyColDesc> rowKeyColDescMap = Maps.newHashMap(); for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) { rowKeyColDescMap.put(entry.getColumn(), entry); } Set<Long> mandatoryCuboids = Sets.newHashSetWithExpectedSize(mandatoryDimensionSetList.size()); for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) { long cuboid = 0L; for (String columnName : mandatoryDimensionSet) { TblColRef tblColRef = model.findColumn(columnName); RowKeyColDesc rowKeyColDesc = rowKeyColDescMap.get(tblColRef.getIdentity()); // check if mandatory dimension set list is valid if (rowKeyColDesc == null) { logger.warn("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); throw new IllegalStateException( "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); } cuboid |= 1L << rowKeyColDesc.getBitIndex(); } mandatoryCuboids.add(cuboid); } return mandatoryCuboids; }
RowKeyColDesc[] rowKeyColDescList = cube.getDescriptor().getRowkey().getRowKeyColumns();
public DefaultCuboidScheduler(CubeDesc cubeDesc) { super(cubeDesc); int size = this.cubeDesc.getRowkey().getRowKeyColumns().length; this.max = (long) Math.pow(2, size) - 1; Pair<Set<Long>, Map<Long, List<Long>>> pair = buildTreeBottomUp(); this.allCuboidIds = pair.getFirst(); this.parent2child = pair.getSecond(); }
checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(), "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", this.rowkey.getRowKeyColumns().length); checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%s) doesn't match dimensions columns count (%s)", rowkey.getRowKeyColumns().length, dimCols.size());
public RowKeySplitter(CubeSegment cubeSeg) { this(cubeSeg, cubeSeg.getCubeDesc().getRowkey().getRowKeyColumns().length + 2, cubeSeg.getConfig().getDimensionEncodingMaxLength()); } public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) { final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc); final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length; final Set<Long> allCuboidIds = cubeDesc.getInitialCuboidScheduler().getAllCuboidIds(); final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
private void parseCubeDesc() { Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); // build index for rowkey columns List<TblColRef> cuboidColumns = baseCuboid.getColumns(); int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length; rowKeyColumnIndexes = new int[rowkeyColCount]; for (int i = 0; i < rowkeyColCount; i++) { TblColRef col = cuboidColumns.get(i); rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col); } List<MeasureDesc> measures = cubeDesc.getMeasures(); int measureSize = measures.size(); measureColumnIndexes = new int[measureSize][]; for (int i = 0; i < measureSize; i++) { FunctionDesc func = measures.get(i).getFunction(); List<TblColRef> colRefs = func.getParameter().getColRefs(); if (colRefs == null) { measureColumnIndexes[i] = null; } else { measureColumnIndexes[i] = new int[colRefs.size()]; for (int j = 0; j < colRefs.size(); j++) { TblColRef c = colRefs.get(j); measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c); } } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getColumn() == null || rd.getColumn().length() == 0) { context.addResult(ResultLevel.ERROR, "Rowkey column empty"); } } } }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
@Test public void testBadDesc() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getRowkey().getRowKeyColumns()[2].setColumn(""); IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); rule.validate(desc, vContext); vContext.print(System.out); assertTrue(vContext.getResults().length == 1); assertTrue("Rowkey column empty".equalsIgnoreCase(vContext.getResults()[0].getMessage())); } }
private static List<Long> getAllPossibleParents(CubeDesc cube, long cuboidID) { List<Long> allPossibleParents = new ArrayList<Long>(); for (int i = 0; i < cube.getRowkey().getRowKeyColumns().length; i++) { long mask = 1L << i; long parentId = cuboidID | mask; if (parentId != cuboidID) { allPossibleParents.add(parentId); } } return allPossibleParents; }
private void updateRowkeyDictionary(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { RowKeyDesc rowKey = newModel.getRowkey(); for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) { if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0) rowkeyCol.setDictionary("true"); } }