/** create a mockup table for unit test */ public static TableDesc mockup(String tableName) { TableDesc mockup = new TableDesc(); mockup.setName(tableName); return mockup; }
private void markInnerColumn(InnerDataTypeEnum dataType) { this.column.setDatatype(dataType.getDataType()); this.column.getTable().setName(INNER_TABLE_NAME); this.column.getTable().setDatabase("DEFAULT"); }
private TableDesc deserializeTableDesc(StreamingRequest streamingRequest) { TableDesc desc = null; try { logger.debug("Saving TableDesc " + streamingRequest.getTableData()); desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class); updateRequest(streamingRequest, true, null); } catch (JsonParseException e) { logger.error("The TableDesc definition is invalid.", e); updateRequest(streamingRequest, false, e.getMessage()); } catch (JsonMappingException e) { logger.error("The data TableDesc definition is invalid.", e); updateRequest(streamingRequest, false, e.getMessage()); } catch (IOException e) { logger.error("Failed to deal with the request.", e); throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e); } if (null != desc) { String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName()); desc.setName(dbTable[1]); desc.setDatabase(dbTable[0]); desc.getIdentity(); } return desc; }
TableDesc tableDesc = new TableDesc(); tableDesc.setDatabase(database.toUpperCase(Locale.ROOT)); tableDesc.setName(table.toUpperCase(Locale.ROOT)); tableDesc.setUuid(RandomUtil.randomUUID().toString()); tableDesc.setLastModified(0);
tableDesc = new TableDesc(); tableDesc.setDatabase(database.toUpperCase(Locale.ROOT)); tableDesc.setName(tableName.toUpperCase(Locale.ROOT)); tableDesc.setUuid(RandomUtil.randomUUID().toString()); tableDesc.setLastModified(0);
@Override public IReadableTable createReadableTable(TableDesc tableDesc, String uuid) { // hive view must have been materialized already // ref HiveMRInput.createLookupHiveViewMaterializationStep() if (tableDesc.isView()) { KylinConfig config = KylinConfig.getInstanceFromEnv(); String tableName = tableDesc.getMaterializedName(uuid); tableDesc = new TableDesc(); tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable()); tableDesc.setName(tableName); } return new HiveTable(tableDesc); }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
TableDesc tableDesc = new TableDesc(); tableDesc.setDatabase(database.toUpperCase(Locale.ROOT)); tableDesc.setName(table.toUpperCase(Locale.ROOT)); tableDesc.setUuid(UUID.randomUUID().toString()); tableDesc.setLastModified(0);
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
@Test public void testReadTableDesc() throws IOException { String requestTableData = "{\"name\":\"my_table_name\",\"source_type\":1,\"columns\":[{\"id\":1,\"name\":" + "\"amount\",\"datatype\":\"decimal\"},{\"id\":2,\"name\":\"category\",\"datatype\":\"varchar(256)\"}," + "{\"id\":3,\"name\":\"order_time\",\"datatype\":\"timestamp\"},{\"id\":4,\"name\":\"device\"," + "\"datatype\":\"varchar(256)\"},{\"id\":5,\"name\":\"qty\",\"datatype\":\"int\"},{\"id\":6,\"name\":" + "\"user_id\",\"datatype\":\"varchar(256)\"},{\"id\":7,\"name\":\"user_age\",\"datatype\":\"int\"}," + "{\"id\":8,\"name\":\"user_gender\",\"datatype\":\"varchar(256)\"},{\"id\":9,\"name\":\"currency\"," + "\"datatype\":\"varchar(256)\"},{\"id\":10,\"name\":\"country\",\"datatype\":\"varchar(256)\"}," + "{\"id\":11,\"name\":\"year_start\",\"datatype\":\"date\"},{\"id\":12,\"name\":\"quarter_start\"," + "\"datatype\":\"date\"},{\"id\":13,\"name\":\"month_start\",\"datatype\":\"date\"},{\"id\":14," + "\"name\":\"week_start\",\"datatype\":\"date\"},{\"id\":15,\"name\":\"day_start\",\"datatype\":" + "\"date\"},{\"id\":16,\"name\":\"hour_start\",\"datatype\":\"timestamp\"},{\"id\":17,\"name\":" + "\"minute_start\",\"datatype\":\"timestamp\"}],\"database\":\"my_database_name\"}"; TableDesc desc = JsonUtil.readValue(requestTableData, TableDesc.class); String[] dbTable = HadoopUtil.parseHiveTableName(desc.getIdentity()); desc.setName(dbTable[1]); desc.setDatabase(dbTable[0]); Assert.assertEquals("my_table_name".toUpperCase(Locale.ROOT), desc.getName()); Assert.assertEquals("my_database_name".toUpperCase(Locale.ROOT), desc.getDatabase()); } }
@Override public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { String tableName = BytesUtil.readUTFString(buffer); if (_QUALIFIED_.equals(tableName)) { // qualified column (from model) String model = BytesUtil.readUTFString(buffer); String alias = BytesUtil.readUTFString(buffer); String col = BytesUtil.readUTFString(buffer); KylinConfig config = KylinConfig.getInstanceFromEnv(); DataModelDesc modelDesc = DataModelManager.getInstance(config).getDataModelDesc(model); this.columnRef = modelDesc.findColumn(alias, col); } else { // un-qualified column TableDesc tableDesc = null; if (tableName != null) { tableDesc = new TableDesc(); tableDesc.setName(tableName); } ColumnDesc column = new ColumnDesc(); column.setId(BytesUtil.readUTFString(buffer)); column.setName(BytesUtil.readUTFString(buffer)); column.setDatatype(BytesUtil.readUTFString(buffer)); column.init(tableDesc); this.columnRef = column.getRef(); } }
tableDesc.setName(tableName);
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); int bodyOffset = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); String datatype = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); col.setDatatype(datatype); col.init(table); cols[i] = col.getRef(); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes, bodyOffset); } };
public void markInnerColumn(InnerDataTypeEnum dataType) { this.column.setDatatype(dataType.getDataType()); this.column.getTable().setName(INNER_TABLE_NAME); this.column.getTable().setDatabase("DEFAULT"); }
@Override public void deserialize(byte[] bytes) { ColumnDesc column = new ColumnDesc(); ByteBuffer buffer = ByteBuffer.wrap(bytes); String tableName = BytesUtil.readUTFString(buffer); if (tableName != null) { TableDesc table = new TableDesc(); table.setName(tableName); column.setTable(table); } column.setName(BytesUtil.readUTFString(buffer)); column.setDatatype(BytesUtil.readUTFString(buffer)); this.columnRef = new TblColRef(column); } }
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = new TblColRef(column); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); TupleFilter filter = buildCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes); compareFilter(filter, newFilter); }
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); cols[i] = new TblColRef(col); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes); } }
private List<TblColRef> buildGroups() { List<TblColRef> groups = new ArrayList<TblColRef>(); TableDesc t1 = new TableDesc(); t1.setName("TEST_KYLIN_FACT"); t1.setDatabase("DEFAULT"); ColumnDesc c1 = new ColumnDesc(); c1.setName("CAL_DT"); c1.setTable(t1); c1.setDatatype("string"); TblColRef cf1 = new TblColRef(c1); groups.add(cf1); TableDesc t2 = new TableDesc(); t2.setName("TEST_CATEGORY_GROUPINGS"); t2.setDatabase("DEFAULT"); ColumnDesc c2 = new ColumnDesc(); c2.setName("META_CATEG_NAME"); c2.setTable(t2); c2.setDatatype("string"); TblColRef cf2 = new TblColRef(c2); groups.add(cf2); return groups; }
protected List<TblColRef> buildGroups() { List<TblColRef> groups = new ArrayList<TblColRef>(); TableDesc t1 = new TableDesc(); t1.setName("TEST_KYLIN_FACT"); t1.setDatabase("DEFAULT"); ColumnDesc c1 = new ColumnDesc(); c1.setName("CAL_DT"); c1.setDatatype("String"); c1.setTable(t1); TblColRef cf1 = new TblColRef(c1); groups.add(cf1); TableDesc t2 = new TableDesc(); t2.setName("TEST_CATEGORY_GROUPINGS"); t2.setDatabase("DEFAULT"); ColumnDesc c2 = new ColumnDesc(); c2.setName("META_CATEG_NAME"); c1.setDatatype("String"); c2.setTable(t2); TblColRef cf2 = new TblColRef(c2); groups.add(cf2); return groups; }
private CoprocessorRowType newRowType() { TableDesc t = new TableDesc(); t.setName("TABLE"); t.setDatabase("DEFAULT"); TblColRef[] cols = new TblColRef[] { newCol("A", t), newCol("B", t), newCol("C", t), newCol("D", t) }; int[] sizes = new int[] { 1, 1, 1, 1 }; return new CoprocessorRowType(cols, sizes); }