public List<ColumnDesc> listExposedColumns(String project, TableDesc tableDesc, boolean exposeMore) { Set<ColumnDesc> exposedColumns = l2Cache.listExposedColumns(project, tableDesc.getIdentity()); if (exposeMore) { Set<ColumnDesc> dedup = Sets.newHashSet(tableDesc.getColumns()); dedup.addAll(exposedColumns); return Lists.newArrayList(dedup); } else { return Lists.newArrayList(exposedColumns); } }
public JdbcTable(JdbcConnector dataSource, TableDesc tableDesc) { this.dataSource = dataSource; this.database = tableDesc.getDatabase(); this.tableName = tableDesc.getName(); }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
public String getResourcePath() { if (isBorrowedFromGlobal()) { return concatResourcePath(getIdentity(), null); } return concatResourcePath(getIdentity(), project); }
private TableDesc deserializeTableDesc(StreamingRequest streamingRequest) { TableDesc desc = null; try { logger.debug("Saving TableDesc " + streamingRequest.getTableData()); desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class); updateRequest(streamingRequest, true, null); } catch (JsonParseException e) { logger.error("The TableDesc definition is invalid.", e); updateRequest(streamingRequest, false, e.getMessage()); } catch (JsonMappingException e) { logger.error("The data TableDesc definition is invalid.", e); updateRequest(streamingRequest, false, e.getMessage()); } catch (IOException e) { logger.error("Failed to deal with the request.", e); throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e); } if (null != desc) { String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName()); desc.setName(dbTable[1]); desc.setDatabase(dbTable[0]); desc.getIdentity(); } return desc; }
@Test public void testLoadTableMetadata() throws Exception { Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT"); Assert.assertNotNull(pair.getFirst()); Assert.assertNotNull(pair.getSecond()); TableDesc tblDesc = pair.getFirst(); TableExtDesc tblExtDesc = pair.getSecond(); Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName()); Assert.assertEquals("TABLE", tblDesc.getTableType()); Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity()); Assert.assertEquals("DEFAULT", tblDesc.getDatabase()); Assert.assertEquals("DEFAULT", tblDesc.getProject()); Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity()); Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject()); ColumnDesc[] columnDescs = tblDesc.getColumns(); Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length); Assert.assertNotNull(columnDescs[0].getName()); Assert.assertNotNull(columnDescs[0].getDatatype()); Assert.assertNotNull(columnDescs[0].getType()); Assert.assertNotNull(columnDescs[0].getId()); }
/** * @param table */ public TableDescResponse(TableDesc table) { this.setColumns(table.getColumns()); this.setDatabase(table.getDatabase()); this.setName(table.getName()); this.setSourceType(table.getSourceType()); this.setUuid(table.getUuid()); this.setTableType(table.getTableType()); }
@Override public String toString() { return "TableDesc{" + "name='" + name + '\'' + ", columns=" + Arrays.toString(columns) + ", sourceType=" + sourceType + ", tableType='" + tableType + '\'' + ", database=" + database + ", identity='" + getIdentity() + '\'' + '}'; }
private List<String> generateCreateH2IndexSql(TableDesc tableDesc) { List<String> result = Lists.newArrayList(); int x = 0; for (ColumnDesc col : tableDesc.getColumns()) { if ("T".equalsIgnoreCase(col.getIndex())) { StringBuilder ddl = new StringBuilder(); ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "(" + col.getName() + ")"); ddl.append("\n"); result.add(ddl.toString()); x++; } } return result; }
@Override public IReadableTable createReadableTable(TableDesc tableDesc, String uuid) { // hive view must have been materialized already // ref HiveMRInput.createLookupHiveViewMaterializationStep() if (tableDesc.isView()) { KylinConfig config = KylinConfig.getInstanceFromEnv(); String tableName = tableDesc.getMaterializedName(uuid); tableDesc = new TableDesc(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); tableDesc.setName(tableName); } return new HiveTable(tableDesc); }
private Set<String> getAllColumns(String project, String table) throws IOException { List<TableDesc> tableDescByProject = tableService.getTableDescByProject(project, true); Set<String> cols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (TableDesc tableDesc : tableDescByProject) { String tbl = tableDesc.getDatabase() + "." + tableDesc.getName(); if (tbl.equalsIgnoreCase(table)) { for (ColumnDesc column : tableDesc.getColumns()) { cols.add(column.getName()); } break; } } return cols; } }
private void addAlias(TableRef ref) { String alias = ref.getAlias(); if (aliasMap.containsKey(alias)) throw new IllegalStateException("Alias '" + alias + "' ref to multiple tables: " + ref.getTableIdentity() + ", " + aliasMap.get(alias).getTableIdentity()); aliasMap.put(alias, ref); TableDesc table = ref.getTableDesc(); addTableName(table.getName(), ref); addTableName(table.getIdentity(), ref); }
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) continue; out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n"); out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n"); for (int i = 0; i < t.getColumns().length; i++) { ColumnDesc col = t.getColumns()[i]; out.print(" "); if (i > 0) { out.print(","); } out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n"); } out.print(")" + "\n"); out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); out.print("STORED AS TEXTFILE" + ";\n"); out.print("\n"); } }
public void saveSourceTable(TableDesc srcTable) throws IOException { if (srcTable.getUuid() == null || srcTable.getIdentity() == null) { throw new IllegalArgumentException(); } srcTable.init(); String path = srcTable.getResourcePath(); getStore().putResource(path, srcTable, TABLE_SERIALIZER); srcTableMap.put(srcTable.getIdentity(), srcTable); }
TableExtDesc extDesc = pair.getSecond(); TableDesc origTable = metaMgr.getTableDesc(tableDesc.getIdentity(), project); if (origTable == null || origTable.getProject() == null) { tableDesc.setUuid(RandomUtil.randomUUID().toString()); tableDesc.setLastModified(0); } else { tableDesc.setUuid(origTable.getUuid()); tableDesc.setLastModified(origTable.getLastModified()); TableExtDesc origExt = metaMgr.getTableExt(tableDesc.getIdentity(), project); if (origExt == null || origExt.getProject() == null) { extDesc.setUuid(UUID.randomUUID().toString()); saved.add(tableDesc.getIdentity());
protected static String getTableNameForHCat(TableDesc table, String uuid) { String tableName = (table.isView()) ? table.getMaterializedName(uuid) : table.getName(); String database = (table.isView()) ? KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() : table.getDatabase(); return String.format(Locale.ROOT, "%s.%s", database, tableName).toUpperCase(Locale.ROOT); }
public List<T> scan(String col, List<T> values, String returnCol) { ArrayList<T> result = new ArrayList<T>(); int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex(); int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex(); for (T[] row : data.values()) { if (values.contains(row[colIdx])) result.add(row[returnIdx]); } return result; }
TableDesc ret = makeCopy ? new TableDesc(this) : this; ColumnDesc[] existingColumns = ret.columns; List<ColumnDesc> newColumns = Lists.newArrayList(); String errorMsg = String.format(Locale.ROOT, "There is already a column named %s on table %s, please change your computed column name", computedColumns[j].getName(), this.getIdentity()); throw new IllegalArgumentException(errorMsg); } else {