@Deprecated public List<TableDesc> getLookupTableDescs() { List<TableDesc> result = Lists.newArrayList(); for (TableRef table : getLookupTables()) { result.add(table.getTableDesc()); } return result; }
public void generate() throws IOException { Set<TableDesc> generated = new HashSet<>(); Set<TableDesc> allTableDesc = new LinkedHashSet<>(); JoinTableDesc[] allTables = model.getJoinTables(); for (int i = allTables.length - 1; i >= -1; i--) { // reverse order needed for FK generation TableDesc table = (i == -1) ? model.getRootFactTable().getTableDesc() : allTables[i].getTableRef().getTableDesc(); allTableDesc.add(table); if (generated.contains(table)) continue; logger.info(String.format(Locale.ROOT, "generating data for %s", table)); boolean gen = generateTable(table); if (gen) generated.add(table); } generateDDL(allTableDesc); }
TblColRef(TableRef table, ColumnDesc column) { checkArgument(table.getTableDesc().getIdentity().equals(column.getTable().getIdentity())); this.table = table; this.column = column; }
private boolean matchChain(Chain chain, Chain anotherChain, Map<String, String> matchUp) { String thisAlias = chain.table.getAlias(); String anotherAlias = anotherChain.table.getAlias(); String curMatch = matchUp.get(thisAlias); if (curMatch != null) return curMatch.equals(anotherAlias); if (curMatch == null && matchUp.values().contains(anotherAlias)) return false; boolean matches = false; if (chain.join == null) { matches = anotherChain.join == null && chain.table.getTableDesc().getIdentity().equals(anotherChain.table.getTableDesc().getIdentity()); } else { matches = joinDescMatcher.matches(chain.join, anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp); } if (matches) { matchUp.put(thisAlias, anotherAlias); } return matches; }
private void addAlias(TableRef ref) { String alias = ref.getAlias(); if (aliasMap.containsKey(alias)) throw new IllegalStateException("Alias '" + alias + "' ref to multiple tables: " + ref.getTableIdentity() + ", " + aliasMap.get(alias).getTableIdentity()); aliasMap.put(alias, ref); TableDesc table = ref.getTableDesc(); addTableName(table.getName(), ref); addTableName(table.getIdentity(), ref); }
private void updateMeta(KylinConfig config, String projectName, String cubeName, DataModelDesc model) { String[] nodes = config.getRestServers(); Map<String, String> tableToProjects = new HashMap<>(); for (TableRef tableRef : model.getAllTables()) { tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject()); } for (String node : nodes) { RestClient restClient = new RestClient(node); try { logger.info("update meta cache for " + node); restClient.clearCacheForCubeMigration(cubeName, projectName, model.getName(), tableToProjects); } catch (IOException e) { logger.error(e.getMessage()); } } }
@Override public int getSourceType() { return getModel().getRootFactTable().getTableDesc().getSourceType(); }
public boolean containsTable(TableDesc table) { for (TableRef t : allTableRefs) { if (t.getTableIdentity().equals(table.getIdentity()) && StringUtil.equals(t.getTableDesc().getProject(), table.getProject())) return true; } return false; }
public static void fixUnknownModel(DataModelDesc model, String alias, TblColRef col) { checkArgument(col.table.getModel() == UNKNOWN_MODEL || col.table.getModel() == model); TableRef tableRef = model.findTable(alias); checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity())); col.fixTableRef(tableRef); }
public static Set<String> collectCubeMetadata(CubeInstance cube) { // cube, model_desc, cube_desc, table Set<String> dumpList = new LinkedHashSet<>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); dumpList.add(cube.getProjectInstance().getResourcePath()); for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = tableRef.getTableDesc(); dumpList.add(table.getResourcePath()); dumpList.addAll(SourceManager.getMRDependentResources(table)); } return dumpList; }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); for (TableRef table : cubeDesc.getModel().getAllTables()) { metaResource.add(table.getTableDesc().getResourcePath()); } for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
private void addTables(DataModelDesc modelDesc) throws IOException { if (modelDesc != null) { //fixme should get all tbls in prj not only in cubes when back up by prj. for (TableRef tableRef : modelDesc.getAllTables()) { addRequired(tableRef.getTableDesc().getResourcePath()); addOptional(TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()) // .getTableExt(tableRef.getTableDesc()) // .getResourcePath()); // } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); } }
TableDesc factTable = cube.getModel().findFirstTable(fullTableName).getTableDesc(); List<String> violateColumns = checkAllColumnsInCube(cube, factTable, newTableDesc); if (!violateColumns.isEmpty()) { TableDesc lookupTable = cube.getModel().findFirstTable(fullTableName).getTableDesc(); if (!checkAllColumnsInTableDesc(lookupTable, newTableDesc)) { issues.add(format(Locale.ROOT, "Table '%s' is used as Lookup Table in cube[%s] and model[%s], but "
Set<String> TABLE_NAMES = new HashSet<String>(); for (TableRef tr : tables) { if (!tr.getTableDesc().isView()) { String tableName = tr.getTableName(); String schema = tr.getTableDesc().getDatabase(); String identity = String.format(Locale.ROOT, "%s.%s", schema, tableName); TABLE_NAMES.add(identity); ISampleDataDeployer sampleDataDeployer = SourceManager.getSource(model.getRootFactTable().getTableDesc()) .getSampleDataDeployer();
public TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) { TupleInfo info = new TupleInfo(); int idx = 0; for (TblColRef col : groups) { info.setField(col.getName(), col, idx++); } TableRef sourceTable = groups.get(0).getTableRef(); for (FunctionDesc func : aggregations) { ColumnDesc colDesc = func.newFakeRewriteColumn(sourceTable.getTableDesc()); TblColRef col = sourceTable.makeFakeColumn(colDesc); info.setField(col.getName(), col, idx++); } return info; }
long maxCardinality = 0; for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) { TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc()); List<TableExtDesc.ColumnStats> columnStatses = tableExtDesc.getColumnStats(); if (!columnStatses.isEmpty()) {
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
TblColRef splitColRef = determineSplitColumn(); splitTable = splitColRef.getTableRef().getTableName(); splitTable = splitColRef.getTableRef().getTableDesc().getName(); splitTableAlias = splitColRef.getTableAlias();
public CubeInstanceResponse(CubeInstance cube, String project) { this.project = project; if (cube == null) return; setUuid(cube.getUuid()); setVersion(cube.getVersion()); setName(cube.getName()); setOwner(cube.getOwner()); setDescName(cube.getDescName()); setCost(cube.getCost()); setStatus(cube.getStatus()); setSegments(cube.getSegments()); setCreateTimeUTC(cube.getCreateTimeUTC()); setLastModified(cube.getDescriptor().getLastModified()); setCuboidLastOptimized(cube.getCuboidLastOptimized()); this.model = cube.getDescriptor().getModelName(); this.partitionDateStart = cube.getDescriptor().getPartitionDateStart(); // cuz model doesn't have a state to label a model is broken, // so in some case the model can not be loaded due to some check failed, // but the cube in this model can still be loaded. if (cube.getModel() != null) { this.partitionDateColumn = cube.getModel().getPartitionDesc().getPartitionDateColumn(); this.isStandardPartitioned = cube.getModel().isStandardPartitionedDateColumn(); this.isStreaming = cube.getModel().getRootFactTable().getTableDesc() .getSourceType() == ISourceAware.ID_STREAMING; } initSizeKB(); initInputRecordCount(); initInputRecordSizeBytes(); }
public boolean containsTable(TableDesc table) { for (TableRef t : allTableRefs) { if (t.getTableIdentity().equals(table.getIdentity()) && StringUtil.equals(t.getTableDesc().getProject(), table.getProject())) return true; } return false; }