public DimensionDesc findDimensionByColumn(TblColRef col) { for (DimensionDesc dim : dimensions) { if (ArrayUtils.contains(dim.getColumnRefs(), col)) return dim; } return null; }
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) { LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>(); for (TblColRef colRef : dim.getColumnRefs()) { int dimTableIndex = colRef.getColumn().getZeroBasedIndex(); int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName()); dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex)); } return dimTblColAsRowKey; }
for (TblColRef col : dim.getColumnRefs()) { if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
private void prepare() throws Exception { // load config loadConfig(); TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (DimensionDesc dim : desc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { if (col.getTable().equals(factTableName)) factTableColumns.add(col.getName()); } JoinDesc join = dim.getJoin(); if (join != null) { String lookupTable = dim.getTable(); for (String column : join.getPrimaryKey()) { if (!lookupTableKeys.containsKey(lookupTable)) { lookupTableKeys.put(lookupTable, new LinkedList<String>()); } if (!lookupTableKeys.get(lookupTable).contains(column)) lookupTableKeys.get(lookupTable).add(column); } } } int distinctRowCount = (int) (this.rowCount / this.conflictRatio); distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount; // lookup tables for (String lookupTable : lookupTableKeys.keySet()) { this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount); } }
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
public DimensionDesc findDimensionByColumn(TblColRef col) { for (DimensionDesc dim : dimensions) { if (ArrayUtils.contains(dim.getColumnRefs(), col)) return dim; } return null; }
for (TblColRef col : dim.getColumnRefs()) { if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }