private void initDimensionColumns() { for (DimensionDesc dim : dimensions) { JoinDesc join = dim.getJoin(); String colStr = dim.getColumn(); if ((colStr == null && dim.isDerived()) || ("{FK}".equalsIgnoreCase(colStr))) { dim.setColumnRefs(dimColArray); if (dim.isDerived()) { String[] derived = dim.getDerived(); String[][] split = splitDerivedColumnAndExtra(derived); String[] derivedNames = split[0];
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) { TblColRef col = model.findColumn(dim.getTable(), colName); // for backward compatibility if (KylinVersion.isBefore200(getVersion())) { // always use FK instead PK, FK could be shared by more than one lookup tables JoinDesc join = dim.getJoin(); if (join != null) { int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), col); if (idx >= 0) { col = join.getForeignKeyColumns()[idx]; } } } return initDimensionColRef(col); }
public DimensionDesc findDimensionByColumn(TblColRef col) { for (DimensionDesc dim : dimensions) { if (ArrayUtils.contains(dim.getColumnRefs(), col)) return dim; } return null; }
public static DimensionDesc getDimensionDesc(String tableName, String dimension) { DimensionDesc dimensionDesc = new DimensionDesc(); dimensionDesc.setName(dimension); dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1)); dimensionDesc.setColumn(dimension); return dimensionDesc; }
private void initDimensionColumns(Map<String, TableDesc> tables) { for (DimensionDesc dim : dimensions) { JoinDesc join = dim.getJoin(); String[] colStrs = dim.getColumn(); if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) { for (TblColRef col : join.getForeignKeyColumns()) { dimCols.add(initDimensionColRef(col)); if (dim.isHierarchy()) { for (int i = 0; i < dimCols.size(); i++) dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); dim.setColumnRefs(dimColArray); if (dim.isDerived()) { String[] derived = dim.getDerived(); String[][] split = splitDerivedColumnAndExtra(derived); String[] derivedNames = split[0];
DimensionDesc ld = lc.getDimensions().get(i); DimensionDesc id = ic.getDimensions().get(i); assertEquals(ld.getTable(), id.getTable()); assertEquals(ld.getColumn(), id.getColumn()); assertArrayEquals(ld.getDerived(), id.getDerived());
JoinDesc jDesc = dim.getJoin(); if (jDesc != null) { String[] fks = jDesc.getForeignKey(); int num = fks.length; for (int i = 0; i < num; ++i) { String value = dim.getTable() + "/" + pks[i]; JoinDesc jDesc = dim.getJoin(); if (jDesc == null) { for (String aColumn : dim.getColumn()) { if (!factTableCol2LookupCol.containsKey(aColumn)) usedCols.add(aColumn);
private void prepare() throws Exception { // load config loadConfig(); TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (DimensionDesc dim : desc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { if (col.getTable().equals(factTableName)) factTableColumns.add(col.getName()); } JoinDesc join = dim.getJoin(); if (join != null) { String lookupTable = dim.getTable(); for (String column : join.getPrimaryKey()) { if (!lookupTableKeys.containsKey(lookupTable)) { lookupTableKeys.put(lookupTable, new LinkedList<String>()); } if (!lookupTableKeys.get(lookupTable).contains(column)) lookupTableKeys.get(lookupTable).add(column); } } } int distinctRowCount = (int) (this.rowCount / this.conflictRatio); distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount; // lookup tables for (String lookupTable : lookupTableKeys.keySet()) { this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount); } }
this.lookupTable = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim.getJoin());
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) { DimensionDesc newDim = new DimensionDesc(); newDim.setId(dimId); newDim.setName(name); newDim.setTable(getMetadataManager().appendDBName(dim.getTable())); return newDim; }
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) { TableDesc table = dim.getTableDesc(); ColumnDesc col = table.findColumnByName(colName); if (col == null) throw new IllegalArgumentException("No column '" + colName + "' found in table " + table); TblColRef ref = new TblColRef(col); // always use FK instead PK, FK could be shared by more than one lookup tables JoinDesc join = dim.getJoin(); if (join != null) { int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), ref); if (idx >= 0) { ref = join.getForeignKeyColumns()[idx]; } } return initDimensionColRef(ref); }
public DimensionDesc findDimensionByTable(String lookupTableName) { lookupTableName = lookupTableName.toUpperCase(Locale.ROOT); for (DimensionDesc dim : dimensions) if (dim.getTableRef() != null && dim.getTableRef().getTableIdentity().equals(lookupTableName)) return dim; return null; }
dim.init(this);
private void initDimensionColumns(Map<String, TableDesc> tables) { for (DimensionDesc dim : dimensions) { JoinDesc join = dim.getJoin(); String[] colStrs = dim.getColumn(); if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) { for (TblColRef col : join.getForeignKeyColumns()) { dimCols.add(initDimensionColRef(col)); if (dim.isHierarchy()) { for (int i = 0; i < dimCols.size(); i++) dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); dim.setColumnRefs(dimColArray); if (dim.isDerived()) { String[] derived = dim.getDerived(); String[][] split = splitDerivedColumnAndExtra(derived); String[] derivedNames = split[0];
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
/** * @deprecated use getModel().getAllTables() instead * @return */ public List<TableDesc> listTables() { MetadataManager metaMgr = MetadataManager.getInstance(config); HashSet<String> tableNames = new HashSet<String>(); List<TableDesc> result = new ArrayList<TableDesc>(); tableNames.add(this.getFactTable().toUpperCase()); for (DimensionDesc dim : dimensions) { String table = dim.getTable(); if (table != null) tableNames.add(table.toUpperCase()); } for (String tableName : tableNames) { result.add(metaMgr.getTableDesc(tableName)); } return result; }