private static TableDesc getTableDesc(String tableName, String prj) { return TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(tableName, prj); }
if (scardi.length() > 0) { scardi = scardi.substring(0, scardi.length() - 1); TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.setCardinality(scardi); metaMgr.saveTableExt(tableExt, prj); } else { TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.resetCardinality(); metaMgr.saveTableExt(tableExt, prj);
private ExtTableSnapshotInfo load(String resourcePath) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, SNAPSHOT_SERIALIZER); return snapshot; }
/** * Get table extended info. Keys are defined in {@link MetadataConstants} * * @param tableName * @return */ public TableExtDesc getTableExt(String tableName, String prj) { TableDesc t = getTableDesc(tableName, prj); if (t == null) return null; return getTableExt(t); }
private Map<String, TableDesc> getAllTablesMap(String prj) { return TableMetadataManager.getInstance(config).getAllTablesMap(prj); }
public void removeSourceTable(String tableIdentity, String prj) throws IOException { try (AutoLock lock = srcTableMapLock.lockForWrite()) { TableDesc t = getTableDesc(tableIdentity, prj); if (t == null) return; srcTableCrud.delete(t); } }
private void loadH2Table(String tableName) throws SQLException { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); TableDesc tableDesc = metaMgr.getTableDesc(tableName.toUpperCase(Locale.ROOT), project); File tempFile = null; try { tempFile = File.createTempFile("tmp_h2", ".csv"); FileOutputStream tempFileStream = new FileOutputStream(tempFile); String path = path(tableDesc); InputStream csvStream = metaMgr.getStore().getResource(path).content(); IOUtils.copy(csvStream, tempFileStream); csvStream.close(); tempFileStream.close(); } catch (IOException e) { e.printStackTrace(); } String cvsFilePath = tempFile.getPath(); try (Statement stmt = h2Connection.createStatement()) { String createDBSql = "CREATE SCHEMA IF NOT EXISTS DEFAULT;\nCREATE SCHEMA IF NOT EXISTS EDW;\nSET SCHEMA DEFAULT;\n"; stmt.executeUpdate(createDBSql); String sql = generateCreateH2TableSql(tableDesc, cvsFilePath); stmt.executeUpdate(sql); List<String> createIndexStatements = generateCreateH2IndexSql(tableDesc); for (String indexSql : createIndexStatements) { stmt.executeUpdate(indexSql); } } tempFile.delete(); }
private void addTables(DataModelDesc modelDesc) throws IOException { if (modelDesc != null) { //fixme should get all tbls in prj not only in cubes when back up by prj. for (TableRef tableRef : modelDesc.getAllTables()) { addRequired(tableRef.getTableDesc().getResourcePath()); addOptional(TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()) // .getTableExt(tableRef.getTableDesc()) // .getResourcePath()); // } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); } }
private TableMetadataManager getMetadataManager() { return TableMetadataManager.getInstance(config); }
TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName); if (existing != null && !existing.equals(tableDesc)) { logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
@Test public void testGetInstance() throws Exception { Assert.assertNotNull(getInstance(getTestConfig())); Assert.assertNotNull(getInstance(getTestConfig()).listAllTables(null)); Assert.assertTrue(getInstance(getTestConfig()).listAllTables(null).size() > 0); }
@Test public void testTableSample() throws IOException { TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc); List<TableExtDesc.ColumnStats> columnStatsList = new ArrayList<>(); TableExtDesc.ColumnStats columnStats = new TableExtDesc.ColumnStats(); columnStats.setColumnSamples("Max", "Min", "dfadsfdsfdsafds", "d"); columnStatsList.add(columnStats); tableExtDesc.setColumnStats(columnStatsList); getInstance(getTestConfig()).saveTableExt(tableExtDesc, "default"); TableExtDesc tableExtDesc1 = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc1); List<TableExtDesc.ColumnStats> columnStatsList1 = tableExtDesc1.getColumnStats(); Assert.assertEquals(1, columnStatsList1.size()); getInstance(getTestConfig()).removeTableExt("DEFAULT.WIDE_TABLE", "default"); }
@Test public void testTableExtCompatibility() throws IOException { String tableName = "DEFAULT.WIDE_TABLE"; Map<String, String> oldTableExt = new HashMap<>(); oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4"); mockUpOldTableExtJson(tableName, oldTableExt); TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default"); Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality()); getInstance(getTestConfig()).removeTableExt(tableName, "default"); }
TableExtDesc extDesc = pair.getSecond(); TableDesc origTable = metaMgr.getTableDesc(tableDesc.getIdentity(), project); if (origTable == null || origTable.getProject() == null) { tableDesc.setUuid(RandomUtil.randomUUID().toString()); tableDesc.setLastModified(origTable.getLastModified()); metaMgr.saveSourceTable(tableDesc, project); TableExtDesc origExt = metaMgr.getTableExt(tableDesc.getIdentity(), project); if (origExt == null || origExt.getProject() == null) { extDesc.setUuid(UUID.randomUUID().toString()); metaMgr.saveTableExt(extDesc, project);
TableDesc table = getTableManager().getTableDesc(tableName, prj); final TableExtDesc tableExt = getTableManager().getTableExt(tableName, prj); if (table == null) { BadRequestException e = new BadRequestException( job.addTask(step2); tableExt.setJodID(job.getId()); getTableManager().saveTableExt(tableExt, prj);
@Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { //clean up the current project's table desc TableMetadataManager.getInstance(config).resetProjectSpecificTableDesc(project); try (AutoLock lock = modelMapLock.lockForWrite()) { for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { crud.reloadQuietly(model); } } }
@Override public void addChild(TupleFilter child) { if (child instanceof ColumnTupleFilter) { super.addChild(child); ColumnTupleFilter columnFilter = (ColumnTupleFilter) child; if (this.column != null) { throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName()); } this.column = columnFilter.getColumn(); } else if (child instanceof ConstantTupleFilter) { // super.addChild(child) is omitted because the filter table name is useless at storage side, // we'll extract the useful filterTableResourceIdentifier,filterTableType etc and save it at the MassInTupleFilter itself if (filterTableName == null) { filterTableName = (String) child.getValues().iterator().next(); ExternalFilterDesc externalFilterDesc = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getExtFilterDesc(filterTableName); if (externalFilterDesc == null) { throw new IllegalArgumentException("External filter named " + filterTableName + " is not found"); } filterTableType = externalFilterDesc.getFilterTableType(); filterTableResourceIdentifier = externalFilterDesc.getFilterResourceIdentifier(); } } else { throw new IllegalStateException("MassInTupleFilter only has two children: one ColumnTupleFilter and one ConstantTupleFilter"); } }
TableDesc desc = getTableManager().getTableDesc(tableName, project); metaMgr.removeTableExt(tableName, project); metaMgr.removeSourceTable(tableName, project);
TableDesc tableDesc = metaMgr.getTableDesc(tableName, project); if (tableDesc != null) { projectCache.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc)); ExternalFilterDesc filterDesc = metaMgr.getExtFilterDesc(extFilterName); if (filterDesc != null) { projectCache.extFilters.put(extFilterName, filterDesc);
public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { throw new IllegalArgumentException(); } // updating a legacy global table if (tableExt.getProject() == null) { if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) throw new IllegalStateException( "Updating a legacy global TableExtDesc while a project level version exists: " + tableExt.getIdentity() + ", " + prj); prj = tableExt.getProject(); } tableExt.init(prj); // what is this doing?? String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER); if (t != null && t.getIdentity() == null) store.deleteResource(path); srcExtCrud.save(tableExt); } }