public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { TableRef table = dim.getTableRef(); if (cubeSeg.getModel().isLookupTable(table)) {
while (groupByCol != null) { String embeded_groupby = groupByCol.getValue(); for (DimensionDesc dimensionDesc : cube.getDimensions()) { if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) { duplicatedCol.add(embeded_groupby);
assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { DimensionDesc ld = lc.getDimensions().get(i); DimensionDesc id = ic.getDimensions().get(i); assertEquals(ld.getTable(), id.getTable()); assertEquals(ld.getColumn(), id.getColumn());
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
private List<DimensionDesc> getSortedDimentsionDescs() { List<DimensionDesc> dimensions = desc.getDimensions(); Collections.sort(dimensions, new Comparator<DimensionDesc>() { @Override public int compare(DimensionDesc o1, DimensionDesc o2) { JoinDesc j1 = o2.getJoin(); JoinDesc j2 = o1.getJoin(); return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0); } }); return dimensions; }
private void buildHierarchyMasks() { this.hierarchyMasks = new ArrayList<HierarchyMask>(); for (DimensionDesc dimension : this.cubeDesc.getDimensions()) { HierarchyDesc[] hierarchies = dimension.getHierarchy(); if (hierarchies == null || hierarchies.length == 0) continue; HierarchyMask mask = new HierarchyMask(); ArrayList<Long> allMaskList = new ArrayList<Long>(); for (int i = 0; i < hierarchies.length; i++) { TblColRef hColumn = hierarchies[i].getColumnRef(); Integer index = getColumnBitIndex(hColumn); long bit = 1L << index; if ((tailMask & bit) > 0) continue; // ignore levels in tail, they don't participate // aggregation group combination anyway mask.fullMask |= bit; allMaskList.add(mask.fullMask); } mask.allMasks = new long[allMaskList.size()]; for (int i = 0; i < allMaskList.size(); i++) mask.allMasks[i] = allMaskList.get(i); this.hierarchyMasks.add(mask); } }
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) { RowKeyDesc rowKeyDesc = cubeDesc.getRowkey(); RowKeyColInfo info = new RowKeyColInfo(); info.hierachyColBitIndice = new ArrayList<List<Integer>>(); info.nonHierachyColBitIndice = new ArrayList<Integer>(); HashSet<Integer> heirachyIndexSet = new HashSet<Integer>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { if (dim.getHierarchy() != null) { LinkedList<Integer> hlist = new LinkedList<Integer>(); for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) { int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef()); hlist.add(index); heirachyIndexSet.add(index); } info.hierachyColBitIndice.add(hlist); } } for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) { if (!heirachyIndexSet.contains(i)) { info.nonHierachyColBitIndice.add(i); } } return info; }
private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) { CubeDesc cubeDesc = cube.getDescriptor(); List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size()); for (DimensionDesc d : cubeDesc.getDimensions()) { if (d.getJoin() != null) { cubeJoins.add(d.getJoin());
this.factTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { JoinDesc join = dim.getJoin(); if (join != null) {
for (DimensionDesc dim : cubeDesc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
newCubeDesc.setDescription(cubeDesc.getDescription()); newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); newCubeDesc.setDimensions(cubeDesc.getDimensions()); newCubeDesc.setMeasures(cubeDesc.getMeasures()); newCubeDesc.setDictionaries(cubeDesc.getDictionaries());
private void prepare() throws Exception { // load config loadConfig(); TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (DimensionDesc dim : desc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { if (col.getTable().equals(factTableName)) factTableColumns.add(col.getName()); } JoinDesc join = dim.getJoin(); if (join != null) { String lookupTable = dim.getTable(); for (String column : join.getPrimaryKey()) { if (!lookupTableKeys.containsKey(lookupTable)) { lookupTableKeys.put(lookupTable, new LinkedList<String>()); } if (!lookupTableKeys.get(lookupTable).contains(column)) lookupTableKeys.get(lookupTable).add(column); } } } int distinctRowCount = (int) (this.rowCount / this.conflictRatio); distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount; // lookup tables for (String lookupTable : lookupTableKeys.keySet()) { this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount); } }
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
private void checkCubeDesc(String descName) { CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName); Assert.assertNotNull(cubedesc1); DataModelDesc model = cubedesc1.getModel(); Assert.assertNotNull(model); Assert.assertTrue(model.getLookups().length > 0); List<DimensionDesc> dims = cubedesc1.getDimensions(); Assert.assertTrue(dims.size() > 0); for (DimensionDesc dim : dims) { Assert.assertTrue(dim.getColumn().length > 0); } Assert.assertTrue(cubedesc1.getMeasures().size() > 0); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName); Assert.assertTrue(cubes.size() > 0); }
private void buildHierarchyMasks() { this.hierarchyMasks = new ArrayList<HierarchyMask>(); for (DimensionDesc dimension : this.cubeDesc.getDimensions()) { HierarchyDesc[] hierarchies = dimension.getHierarchy(); if (hierarchies == null || hierarchies.length == 0) continue; HierarchyMask mask = new HierarchyMask(); ArrayList<Long> allMaskList = new ArrayList<Long>(); for (int i = 0; i < hierarchies.length; i++) { TblColRef hColumn = hierarchies[i].getColumnRef(); Integer index = getColumnBitIndex(hColumn); long bit = 1L << index; if ((tailMask & bit) > 0) continue; // ignore levels in tail, they don't participate // aggregation group combination anyway mask.fullMask |= bit; allMaskList.add(mask.fullMask); } mask.allMasks = new long[allMaskList.size()]; for (int i = 0; i < allMaskList.size(); i++) mask.allMasks[i] = allMaskList.get(i); this.hierarchyMasks.add(mask); } }
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) { RowKeyDesc rowKeyDesc = cubeDesc.getRowkey(); RowKeyColInfo info = new RowKeyColInfo(); info.hierachyColBitIndice = new ArrayList<List<Integer>>(); info.nonHierachyColBitIndice = new ArrayList<Integer>(); HashSet<Integer> heirachyIndexSet = new HashSet<Integer>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { if (dim.getHierarchy() != null) { LinkedList<Integer> hlist = new LinkedList<Integer>(); for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) { int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef()); hlist.add(index); heirachyIndexSet.add(index); } info.hierachyColBitIndice.add(hlist); } } for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) { if (!heirachyIndexSet.contains(i)) { info.nonHierachyColBitIndice.add(i); } } return info; }
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
public static CubeDesc getCopyOf(CubeDesc cubeDesc) { CubeDesc newCubeDesc = new CubeDesc(); newCubeDesc.setName(cubeDesc.getName()); newCubeDesc.setModelName(cubeDesc.getModelName()); newCubeDesc.setDescription(cubeDesc.getDescription()); newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); newCubeDesc.setDimensions(cubeDesc.getDimensions()); newCubeDesc.setMeasures(cubeDesc.getMeasures()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature()); newCubeDesc.setNotifyList(cubeDesc.getNotifyList()); newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges()); newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); newCubeDesc.setConfig(cubeDesc.getConfig()); newCubeDesc.updateRandomUuid(); return newCubeDesc; }