@Deprecated public TblColRef makeFakeColumn(String name) { ColumnDesc colDesc = new ColumnDesc(); colDesc.setName(name); colDesc.setTable(table); return new TblColRef(this, colDesc); }
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) { int columnNumber = hiveTableMeta.allColumns.size(); List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber); for (int i = 0; i < columnNumber; i++) { HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i); // skip unsupported fields, e.g. map<string, int> if (DataType.isKylinSupported(field.dataType)) { ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(field.name.toUpperCase(Locale.ROOT)); // use "double" in kylin for "float" if ("float".equalsIgnoreCase(field.dataType)) { cdesc.setDatatype("double"); } else { cdesc.setDatatype(field.dataType); } cdesc.setId(String.valueOf(i + 1)); cdesc.setComment(field.comment); columns.add(cdesc); } else { logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name); } } return columns.toArray(new ColumnDesc[0]); } }
private ColumnDesc[] extractColumnFromMeta(ResultSet meta) throws SQLException { List<ColumnDesc> columns = new ArrayList<>(); while (meta.next()) { String cname = meta.getString("COLUMN_NAME"); int type = meta.getInt("DATA_TYPE"); int csize = meta.getInt("COLUMN_SIZE"); int digits = meta.getInt("DECIMAL_DIGITS"); int pos = meta.getInt("ORDINAL_POSITION"); String remarks = meta.getString("REMARKS"); ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase(Locale.ROOT)); String kylinType = SqlUtil.jdbcTypeToKylinDataType(type); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1; cdesc.setDatatype(new DataType(kylinType, precision, scale).toString()); cdesc.setId(String.valueOf(pos)); cdesc.setComment(remarks); columns.add(cdesc); } return columns.toArray(new ColumnDesc[columns.size()]); } }
public TableDesc(TableDesc other) { this.uuid = other.uuid; this.lastModified = other.lastModified; this.name = other.name; this.sourceType = other.sourceType; this.tableType = other.tableType; this.dataGen = other.dataGen; this.columns = new ColumnDesc[other.columns.length]; for (int i = 0; i < other.columns.length; i++) { this.columns[i] = new ColumnDesc(other.columns[i]); this.columns[i].init(this); } this.project = other.project; this.config = other.config; this.database.setName(other.getDatabase()); this.identity = other.identity; }
public static ColumnDesc mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype) { ColumnDesc desc = new ColumnDesc(); String id = "" + oneBasedColumnIndex; desc.setId(id); desc.setName(name); desc.setDatatype(datatype); desc.init(table); return desc; }
@Override public ColumnDesc[] evalQueryMetadata(String query) { if (StringUtils.isEmpty(query)) { throw new RuntimeException("Evaluate query shall not be empty."); } try (Connection conn = dataSource.getConnection(); Statement state = conn.createStatement(); ResultSet rs = state.executeQuery(dataSource.convertSql(query))) { ResultSetMetaData rsMeta = rs.getMetaData(); ColumnDesc[] columnDescs = new ColumnDesc[rsMeta.getColumnCount()]; for (int i = 0; i < columnDescs.length; i++) { columnDescs[i] = new ColumnDesc(); columnDescs[i].setName(rsMeta.getColumnName(i + 1).toUpperCase(Locale.ROOT)); String kylinType = dataSource.toKylinTypeName(rsMeta.getColumnType(i + 1)); int precision = (SqlUtil.isPrecisionApplicable(kylinType) && rsMeta.getPrecision(i + 1) > 0) ? rsMeta.getPrecision(i + 1) : -1; int scale = (SqlUtil.isScaleApplicable(kylinType) && rsMeta.getScale(i + 1) > 0) ? rsMeta.getScale(i + 1) : -1; columnDescs[i].setDatatype(new DataType(kylinType, precision, scale).toString()); columnDescs[i].setId(String.valueOf(i + 1)); } return columnDescs; } catch (Exception e) { throw new RuntimeException("Cannot evaluate metadata of query: " + query, e); } }
public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType, String parserDescription) { ColumnDesc column = new ColumnDesc(); column.setName(columnName); TableDesc table = new TableDesc(); column.setTable(table); TblColRef colRef = new TblColRef(column); colRef.markInnerColumn(dataType); colRef.parserDescription = parserDescription; return colRef; }
@Test public void testSerialize05() { ColumnDesc column = new ColumnDesc(); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype, String comment) { ColumnDesc desc = new ColumnDesc(); String id = "" + oneBasedColumnIndex; desc.setId(id); desc.setName(name); desc.setDatatype(datatype); desc.init(table); desc.setComment(comment); return new TblColRef(desc); }
@Test public void testSerialize06() { ColumnDesc column = new ColumnDesc(); column.setName("META_CATEG_NAME"); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
public ColumnDesc newFakeRewriteColumn(TableDesc sourceTable) { ColumnDesc fakeCol = new ColumnDesc(); fakeCol.setName(getRewriteFieldName()); fakeCol.setDatatype(getRewriteFieldType().toString()); if (isCount()) fakeCol.setNullable(false); fakeCol.init(sourceTable); return fakeCol; }
@Test public void testSerialize08() { TableDesc table = new TableDesc(); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
String remarks = columns.getString("REMARKS"); ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(cname.toUpperCase(Locale.ROOT));
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
@Override public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { String tableName = BytesUtil.readUTFString(buffer); if (_QUALIFIED_.equals(tableName)) { // qualified column (from model) String model = BytesUtil.readUTFString(buffer); String alias = BytesUtil.readUTFString(buffer); String col = BytesUtil.readUTFString(buffer); KylinConfig config = KylinConfig.getInstanceFromEnv(); DataModelDesc modelDesc = DataModelManager.getInstance(config).getDataModelDesc(model); this.columnRef = modelDesc.findColumn(alias, col); } else { // un-qualified column TableDesc tableDesc = null; if (tableName != null) { tableDesc = new TableDesc(); tableDesc.setName(tableName); } ColumnDesc column = new ColumnDesc(); column.setId(BytesUtil.readUTFString(buffer)); column.setName(BytesUtil.readUTFString(buffer)); column.setDatatype(BytesUtil.readUTFString(buffer)); column.init(tableDesc); this.columnRef = column.getRef(); } }
ColumnDesc column = new ColumnDesc(); column.setId(BytesUtil.readUTFString(buffer)); column.setName(BytesUtil.readUTFString(buffer));
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); int bodyOffset = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); String datatype = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); col.setDatatype(datatype); col.init(table); cols[i] = col.getRef(); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes, bodyOffset); } };
public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType) { ColumnDesc column = new ColumnDesc(); column.setName(columnName); TableDesc table = new TableDesc(); column.setTable(table); TblColRef colRef = new TblColRef(column); colRef.markInnerColumn(dataType); return colRef; }
@Test public void testSerialize06() { ColumnDesc column = new ColumnDesc(); column.setName("META_CATEG_NAME"); TblColRef colRef = new TblColRef(column); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); TupleFilter filter = buildCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes); compareFilter(filter, newFilter); }