private long getQueryFilterMask(Set<TblColRef> filterColumnD) { long filterMask = 0; logger.info("Filter column set for query: %s", filterColumnD); if (filterColumnD.isEmpty() == false) { RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { if (filterColumnD.contains(allColumns[i].getColRef())) { filterMask |= 1L << allColumns[i].getBitIndex(); } } } logger.info("Filter mask is: {0}", filterMask); return filterMask; }
/** * Get dimensions that have dictionary */ public Set<TblColRef> getAllDimsHaveDictionary() { Set<TblColRef> result = Sets.newHashSet(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } return result; }
private void buildPartialCubeFullMask(RowKeyDesc rowKeyDesc) { Preconditions.checkState(this.includes != null); Preconditions.checkState(this.includes.length != 0); partialCubeFullMask = 0L; for (String dim : this.includes) { TblColRef hColumn = cubeDesc.getModel().findColumn(dim); Integer index = rowKeyDesc.getColumnBitIndex(hColumn); long bit = 1L << index; partialCubeFullMask |= bit; } }
public void init(CubeDesc cubeDesc) { setCubeDesc(cubeDesc); buildRowKey(); initColumnsNeedIndex(); }
checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(), "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", this.rowkey.getRowKeyColumns().length); initMeasureColumns(); rowkey.init(this); checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%s) doesn't match dimensions columns count (%s)", rowkey.getRowKeyColumns().length, dimCols.size());
private Boolean checkNeedMerging(TblColRef col) throws IOException { Boolean ret = dictsNeedMerging.get(col); if (ret != null) return ret; else { ret = cubeDesc.getRowkey().isUseDictionary(col); if (ret) { String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable); } dictsNeedMerging.put(col, ret); return ret; } }
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) { RowKeyDesc rowKeyDesc = cubeDesc.getRowkey(); RowKeyColInfo info = new RowKeyColInfo(); info.hierachyColBitIndice = new ArrayList<List<Integer>>(); info.nonHierachyColBitIndice = new ArrayList<Integer>(); HashSet<Integer> heirachyIndexSet = new HashSet<Integer>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { if (dim.getHierarchy() != null) { LinkedList<Integer> hlist = new LinkedList<Integer>(); for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) { int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef()); hlist.add(index); heirachyIndexSet.add(index); } info.hierachyColBitIndice.add(hlist); } } for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) { if (!heirachyIndexSet.contains(i)) { info.nonHierachyColBitIndice.add(i); } } return info; }
@Override public void validate(CubeDesc cube, ValidateContext context) { Set<String> mands = new HashSet<String>(); RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns(); if (cols == null || cols.length == 0) { return; } for (int i = 0; i < cols.length; i++) { RowKeyColDesc rowKeyColDesc = cols[i]; if (rowKeyColDesc.isMandatory()) { mands.add(rowKeyColDesc.getColumn()); } } if (mands.isEmpty()) { return; } String[][] groups = cube.getRowkey().getAggregationGroups(); for (int i = 0; i < groups.length; i++) { String[] group = groups[i]; for (int j = 0; j < group.length; j++) { String col = group[j]; if (mands.contains(col)) { context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]"); } } } }
Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) { final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels(); final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length; final String jobId = result.getId(); final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String intermediateHiveTableLocation = getIntermediateHiveTableLocation(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getFactDistinctColumnsPath(seg, jobId); final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); // base cuboid step final MapReduceExecutable baseCuboidStep = createBaseCuboidStep(seg, intermediateHiveTableLocation, cuboidOutputTempPath); result.addTask(baseCuboidStep); // n dim cuboid steps for (int i = 1; i <= groupRowkeyColumnsCount; i++) { int dimNum = totalRowkeyColumnsCount - i; result.addTask(createNDimensionCuboidStep(seg, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount)); } return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep); }
if (StringUtils.isNotEmpty(builderClass) && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && dimensionColumns.contains(dictCol) && rowKeyDesc.isUseDictionary(dictCol)) { context.addResult(ResultLevel.ERROR, ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol); return;
public static long getBaseCuboidId(CubeDesc cube) { return cube.getRowkey().getFullMask(); }
private static long translateToValidCuboid(CubeDesc cubeDesc, long cuboidID) { long mandatoryColumnMask = rowkey.getMandatoryColumnMask(); if (cuboidID < mandatoryColumnMask) { cuboidID = cuboidID | mandatoryColumnMask; for (int i = hierarchies.length - 1; i >= 0; i--) { TblColRef hColumn = hierarchies[i].getColumnRef(); Integer index = rowkey.getColumnBitIndex(hColumn); long bit = 1L << index; if ((rowkey.getTailMask() & bit) > 0) continue; // ignore levels in tail, they don't participate long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask(); long leftover; for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) { if ((cuboidWithoutMandatory & mask.uniqueMask) > 0) { leftover = cuboidWithoutMandatory & ~mask.groupMask; leftover = cuboidWithoutMandatory & rowkey.getTailMask(); if (leftover == 0) { cuboidID = cuboidID | Long.lowestOneBit(rowkey.getAggrGroupFullMask()); return translateToValidCuboid(cubeDesc, cuboidID); cuboidID = cuboidID | rowkey.getTailMask(); return cuboidID;
public void init(CubeDesc cube) { setCubeRef(cube); Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation(); buildRowKey(colNameAbbr); buildAggregationGroups(colNameAbbr); buildHierarchyMasks(); }
public boolean isUseDictionary(TblColRef col) { return getColDesc(col).isUsingDictionary(); }
public boolean isUseDictionary(TblColRef col) { String useDictionary = getDictionary(col); return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary); }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf); cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); cube = CubeManager.getInstance(config).getCube(cubeName); cubeDesc = cube.getDescriptor(); intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); List<TblColRef> columns = baseCuboid.getColumns(); ArrayList<Integer> factDictCols = new ArrayList<Integer>(); RowKeyDesc rowkey = cubeDesc.getRowkey(); DictionaryManager dictMgr = DictionaryManager.getInstance(config); for (int i = 0; i < columns.size(); i++) { TblColRef col = columns.get(i); if (rowkey.isUseDictionary(col) == false) continue; String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; if (cubeDesc.getModel().isFactTable(scanTable)) { factDictCols.add(i); } } this.factDictCols = new int[factDictCols.size()]; for (int i = 0; i < factDictCols.size(); i++) this.factDictCols[i] = factDictCols.get(i); schema = HCatInputFormat.getTableSchema(context.getConfiguration()); }
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) { RowKeyDesc rowKeyDesc = cubeDesc.getRowkey(); RowKeyColInfo info = new RowKeyColInfo(); info.hierachyColBitIndice = new ArrayList<List<Integer>>(); info.nonHierachyColBitIndice = new ArrayList<Integer>(); HashSet<Integer> heirachyIndexSet = new HashSet<Integer>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { if (dim.getHierarchy() != null) { LinkedList<Integer> hlist = new LinkedList<Integer>(); for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) { int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef()); hlist.add(index); heirachyIndexSet.add(index); } info.hierachyColBitIndice.add(hlist); } } for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) { if (!heirachyIndexSet.contains(i)) { info.nonHierachyColBitIndice.add(i); } } return info; }
public void init(KylinConfig config, Map<String, TableDesc> tables) { this.errors.clear(); this.config = config; if (this.modelName == null || this.modelName.length() == 0) { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); } for (DimensionDesc dim : dimensions) { dim.init(this, tables); } sortDimAndMeasure(); initDimensionColumns(tables); initMeasureColumns(tables); rowkey.init(this); if (hbaseMapping != null) { hbaseMapping.init(this); } initMeasureReferenceToColumnFamily(); // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); if (rowkey.getRowKeyColumns().length != dimCols.size()) { addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); } }
TblColRef col = cuboid.getColumns().get(i); if (cubeDesc.getRowkey().isUseDictionary(col)) {
@Test public void testIdentifyCuboidV2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException { CubeDesc cubeDesc = cube.getDescriptor(); Cuboid ret = Cuboid.findCuboid(cube.getCuboidScheduler(), Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList()); long baseCuboidId = cubeDesc.getRowkey().getFullMask(); assertNotEquals(baseCuboidId, ret.getId()); ret = Cuboid.findCuboid(cube.getCuboidScheduler(), dimensions, metrics); assertEquals(baseCuboidId, ret.getId()); }