/** * Get table extended info. Keys are defined in {@link MetadataConstants} * * @param tableName * @return */ public TableExtDesc getTableExt(String tableName, String prj) { TableDesc t = getTableDesc(tableName, prj); if (t == null) return null; return getTableExt(t); }
public void removeTableExt(String tableName, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { // note, here assume always delete TableExtDesc first, then TableDesc TableExtDesc t = getTableExt(tableName, prj); if (t == null) return; srcExtCrud.delete(t); } }
scardi = scardi.substring(0, scardi.length() - 1); TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.setCardinality(scardi); metaMgr.saveTableExt(tableExt, prj); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.resetCardinality(); metaMgr.saveTableExt(tableExt, prj);
private TableDescResponse cloneTableDesc(TableDesc table, String prj) { TableExtDesc tableExtDesc = getTableManager().getTableExt(table.getIdentity(), prj); // Clone TableDesc TableDescResponse rtableDesc = new TableDescResponse(table); Map<String, Long> cardinality = new HashMap<String, Long>(); Map<String, String> dataSourceProp = new HashMap<>(); String scard = tableExtDesc.getCardinality(); if (!StringUtils.isEmpty(scard)) { String[] cards = StringUtils.split(scard, ","); ColumnDesc[] cdescs = rtableDesc.getColumns(); for (int i = 0; i < cdescs.length; i++) { ColumnDesc columnDesc = cdescs[i]; if (cards.length > i) { cardinality.put(columnDesc.getName(), Long.parseLong(cards[i])); } else { logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length); break; } } rtableDesc.setCardinality(cardinality); } dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); rtableDesc.setDescExd(dataSourceProp); return rtableDesc; }
@Test public void testTableExtCompatibility() throws IOException { String tableName = "DEFAULT.WIDE_TABLE"; Map<String, String> oldTableExt = new HashMap<>(); oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4"); mockUpOldTableExtJson(tableName, oldTableExt); TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default"); Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality()); getInstance(getTestConfig()).removeTableExt(tableName, "default"); }
public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { throw new IllegalArgumentException(); } // updating a legacy global table if (tableExt.getProject() == null) { if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) throw new IllegalStateException( "Updating a legacy global TableExtDesc while a project level version exists: " + tableExt.getIdentity() + ", " + prj); prj = tableExt.getProject(); } tableExt.init(prj); // what is this doing?? String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER); if (t != null && t.getIdentity() == null) store.deleteResource(path); srcExtCrud.save(tableExt); } }
public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception { // calculate cardinality for Hive source ProjectInstance projectInstance = getProjectManager().getProject(prj); if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE){ return; } TableMetadataManager metaMgr = getTableManager(); ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig()); for (String table : tables) { TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj); String jobID = tableExtDesc.getJodID(); if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) { calculateCardinality(table, submitter, prj); } } }
private void addTables(DataModelDesc modelDesc) throws IOException { if (modelDesc != null) { //fixme should get all tbls in prj not only in cubes when back up by prj. for (TableRef tableRef : modelDesc.getAllTables()) { addRequired(tableRef.getTableDesc().getResourcePath()); addOptional(TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()) // .getTableExt(tableRef.getTableDesc()) // .getResourcePath()); // } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); } }
TableExtDesc origExt = metaMgr.getTableExt(tableDesc.getIdentity(), project); if (origExt == null || origExt.getProject() == null) { extDesc.setUuid(UUID.randomUUID().toString());
@Test public void testTableSample() throws IOException { TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc); List<TableExtDesc.ColumnStats> columnStatsList = new ArrayList<>(); TableExtDesc.ColumnStats columnStats = new TableExtDesc.ColumnStats(); columnStats.setColumnSamples("Max", "Min", "dfadsfdsfdsafds", "d"); columnStatsList.add(columnStats); tableExtDesc.setColumnStats(columnStatsList); getInstance(getTestConfig()).saveTableExt(tableExtDesc, "default"); TableExtDesc tableExtDesc1 = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc1); List<TableExtDesc.ColumnStats> columnStatsList1 = tableExtDesc1.getColumnStats(); Assert.assertEquals(1, columnStatsList1.size()); getInstance(getTestConfig()).removeTableExt("DEFAULT.WIDE_TABLE", "default"); }
long maxCardinality = 0; for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) { TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc()); List<TableExtDesc.ColumnStats> columnStatses = tableExtDesc.getColumnStats(); if (!columnStatses.isEmpty()) {
final TableExtDesc tableExt = getTableManager().getTableExt(tableName, prj); if (table == null) { BadRequestException e = new BadRequestException(
/** * Get table extended info. Keys are defined in {@link MetadataConstants} * * @param tableName * @return */ public TableExtDesc getTableExt(String tableName, String prj) { TableDesc t = getTableDesc(tableName, prj); if (t == null) return null; return getTableExt(t); }
public void removeTableExt(String tableName, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { // note, here assume always delete TableExtDesc first, then TableDesc TableExtDesc t = getTableExt(tableName, prj); if (t == null) return; srcExtCrud.delete(t); } }
scardi = scardi.substring(0, scardi.length() - 1); TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.setCardinality(scardi); metaMgr.saveTableExt(tableExt, prj); TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj); tableExt.resetCardinality(); metaMgr.saveTableExt(tableExt, prj);
public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { throw new IllegalArgumentException(); } // updating a legacy global table if (tableExt.getProject() == null) { if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) throw new IllegalStateException( "Updating a legacy global TableExtDesc while a project level version exists: " + tableExt.getIdentity() + ", " + prj); prj = tableExt.getProject(); } tableExt.init(prj); // what is this doing?? String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER); if (t != null && t.getIdentity() == null) store.deleteResource(path); srcExtCrud.save(tableExt); } }
private void addTables(DataModelDesc modelDesc) throws IOException { if (modelDesc != null) { //fixme should get all tbls in prj not only in cubes when back up by prj. for (TableRef tableRef : modelDesc.getAllTables()) { addRequired(tableRef.getTableDesc().getResourcePath()); addOptional(TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()) // .getTableExt(tableRef.getTableDesc()) // .getResourcePath()); // } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); } }