TableExtDesc tableExtDesc = new TableExtDesc(); tableExtDesc.setIdentity(tableDesc.getIdentity()); tableExtDesc.setUuid(RandomUtil.randomUUID().toString()); tableExtDesc.setLastModified(0); tableExtDesc.init(prj); tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation); tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner); tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime)); tableExtDesc.addDataSourceProp("partition_column", partitionColumnString.toString()); tableExtDesc.addDataSourceProp("total_file_size", String.valueOf(hiveTableMeta.fileSize)); tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum)); tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat); tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat); tableExtDesc.addDataSourceProp("skip_header_line_count", String.valueOf(hiveTableMeta.skipHeaderLineCount));
public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException { try (AutoLock lock = srcExtMapLock.lockForWrite()) { if (tableExt.getUuid() == null || tableExt.getIdentity() == null) { throw new IllegalArgumentException(); } // updating a legacy global table if (tableExt.getProject() == null) { if (getTableExt(tableExt.getIdentity(), prj).getProject() != null) throw new IllegalStateException( "Updating a legacy global TableExtDesc while a project level version exists: " + tableExt.getIdentity() + ", " + prj); prj = tableExt.getProject(); } tableExt.init(prj); // what is this doing?? String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj); ResourceStore store = getStore(); TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER); if (t != null && t.getIdentity() == null) store.deleteResource(path); srcExtCrud.save(tableExt); } }
private TableDescResponse cloneTableDesc(TableDesc table, String prj) { TableExtDesc tableExtDesc = getTableManager().getTableExt(table.getIdentity(), prj); // Clone TableDesc TableDescResponse rtableDesc = new TableDescResponse(table); Map<String, Long> cardinality = new HashMap<String, Long>(); Map<String, String> dataSourceProp = new HashMap<>(); String scard = tableExtDesc.getCardinality(); if (!StringUtils.isEmpty(scard)) { String[] cards = StringUtils.split(scard, ","); ColumnDesc[] cdescs = rtableDesc.getColumns(); for (int i = 0; i < cdescs.length; i++) { ColumnDesc columnDesc = cdescs[i]; if (cards.length > i) { cardinality.put(columnDesc.getName(), Long.parseLong(cards[i])); } else { logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length); break; } } rtableDesc.setCardinality(cardinality); } dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); rtableDesc.setDescExd(dataSourceProp); return rtableDesc; }
public String getResourcePath() { return concatResourcePath(getIdentity(), getProject()); }
@Override public String resourceName() { return TableDesc.makeResourceName(getIdentity(), getProject()); }
@Override protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) { // convert old tableExt json to new one if (t.getIdentity() == null) { t = convertOldTableExtToNewer(resourceName); } String prj = TableDesc.parseResourcePath(resourceName).getSecond(); t.init(prj); return t; } };
if (origExt == null || origExt.getProject() == null) { extDesc.setUuid(UUID.randomUUID().toString()); extDesc.setLastModified(0); } else { extDesc.setUuid(origExt.getUuid()); extDesc.setLastModified(origExt.getLastModified()); extDesc.init(project); metaMgr.saveTableExt(extDesc, project);
private TableExtDesc convertOldTableExtToNewer(String resourceName) { ResourceStore store = getStore(); Map<String, String> attrs = Maps.newHashMap(); try { RawResource res = store.getResource( ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + resourceName + MetadataConstants.FILE_SURFIX); InputStream is = res.content(); try { attrs.putAll(JsonUtil.readValue(is, HashMap.class)); } finally { if (is != null) is.close(); } } catch (IOException ex) { throw new RuntimeException(ex); } String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); // parse table identity from file name String tableIdentity = TableDesc.parseResourcePath(resourceName).getFirst(); TableExtDesc result = new TableExtDesc(); result.setIdentity(tableIdentity); result.setUuid(RandomUtil.randomUUID().toString()); result.setLastModified(0); result.setCardinality(cardinality); return result; }
private String renameTableWithinProject(String srcItem) { if (dstProject != null && srcItem.contains(ResourceStore.TABLE_RESOURCE_ROOT)) { String tableIdentity = TableDesc.parseResourcePath(srcItem).getFirst(); if (srcItem.contains(ResourceStore.TABLE_EXD_RESOURCE_ROOT)) return TableExtDesc.concatResourcePath(tableIdentity, dstProject); else return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + "--" + dstProject + ".json"; } return srcItem; }
@Override public int hashCode() { return getIdentity().hashCode(); }
@Test public void testTableSample() throws IOException { TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc); List<TableExtDesc.ColumnStats> columnStatsList = new ArrayList<>(); TableExtDesc.ColumnStats columnStats = new TableExtDesc.ColumnStats(); columnStats.setColumnSamples("Max", "Min", "dfadsfdsfdsafds", "d"); columnStatsList.add(columnStats); tableExtDesc.setColumnStats(columnStatsList); getInstance(getTestConfig()).saveTableExt(tableExtDesc, "default"); TableExtDesc tableExtDesc1 = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default"); Assert.assertNotNull(tableExtDesc1); List<TableExtDesc.ColumnStats> columnStatsList1 = tableExtDesc1.getColumnStats(); Assert.assertEquals(1, columnStatsList1.size()); getInstance(getTestConfig()).removeTableExt("DEFAULT.WIDE_TABLE", "default"); }
@Test public void testTableExtCompatibility() throws IOException { String tableName = "DEFAULT.WIDE_TABLE"; Map<String, String> oldTableExt = new HashMap<>(); oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4"); mockUpOldTableExtJson(tableName, oldTableExt); TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default"); Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality()); getInstance(getTestConfig()).removeTableExt(tableName, "default"); }
for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) { TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc()); List<TableExtDesc.ColumnStats> columnStatses = tableExtDesc.getColumnStats(); if (!columnStatses.isEmpty()) { for (TblColRef colRef : tableRef.getColumns()) {
private TableExtDesc convertOldTableExtToNewer(String resourceName) { ResourceStore store = getStore(); Map<String, String> attrs = Maps.newHashMap(); try { RawResource res = store.getResource( ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + resourceName + MetadataConstants.FILE_SURFIX); InputStream is = res.content(); try { attrs.putAll(JsonUtil.readValue(is, HashMap.class)); } finally { if (is != null) is.close(); } } catch (IOException ex) { throw new RuntimeException(ex); } String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY); // parse table identity from file name String tableIdentity = TableDesc.parseResourcePath(resourceName).getFirst(); TableExtDesc result = new TableExtDesc(); result.setIdentity(tableIdentity); result.setUuid(RandomUtil.randomUUID().toString()); result.setLastModified(0); result.setCardinality(cardinality); return result; }
public String getResourcePath() { return concatResourcePath(getIdentity(), getProject()); }
@Test public void testLoadTableMetadata() throws Exception { Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT"); Assert.assertNotNull(pair.getFirst()); Assert.assertNotNull(pair.getSecond()); TableDesc tblDesc = pair.getFirst(); TableExtDesc tblExtDesc = pair.getSecond(); Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName()); Assert.assertEquals("TABLE", tblDesc.getTableType()); Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity()); Assert.assertEquals("DEFAULT", tblDesc.getDatabase()); Assert.assertEquals("DEFAULT", tblDesc.getProject()); Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity()); Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject()); ColumnDesc[] columnDescs = tblDesc.getColumns(); Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length); Assert.assertNotNull(columnDescs[0].getName()); Assert.assertNotNull(columnDescs[0].getDatatype()); Assert.assertNotNull(columnDescs[0].getType()); Assert.assertNotNull(columnDescs[0].getId()); }
@Override protected TableExtDesc initEntityAfterReload(TableExtDesc t, String resourceName) { // convert old tableExt json to new one if (t.getIdentity() == null) { t = convertOldTableExtToNewer(resourceName); } String prj = TableDesc.parseResourcePath(resourceName).getSecond(); t.init(prj); return t; } };
public void reloadTableExtQuietly(String table, String project) { try (AutoLock lock = srcExtMapLock.lockForWrite()) { srcExtCrud.reloadQuietly(TableExtDesc.concatResourcePath(table, project)); } }
public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); db2tables.put(parts[0], parts[1]); } // load all tables first List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList(); ProjectInstance projectInstance = getProjectManager().getProject(project); ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer(); for (Map.Entry<String, String> entry : db2tables.entries()) { Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getIdentity() .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT))); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); } return allMeta; }
public TableExtDesc getTableExt(TableDesc t) { try (AutoLock lock = srcExtMapLock.lockForRead()) { TableExtDesc result = srcExtMap.get(mapKey(t.getIdentity(), t.getProject())); if (null == result) { //TODO: notice the table ext is not project-specific, seems not necessary at all result = srcExtMap.get(mapKey(t.getIdentity(), null)); } // avoid returning null, since the TableDesc exists if (null == result) { result = new TableExtDesc(); result.setIdentity(t.getIdentity()); result.setUuid(RandomUtil.randomUUID().toString()); result.setLastModified(0); result.init(t.getProject()); srcExtMap.putLocal(mapKey(t.getIdentity(), t.getProject()), result); } return result; } }