public int getIndexByName(String columnName) { for (int i = 0; i < columns.size(); i++) { if (columns.get(i).getName().equals(columnName)) { return i; } } return -1; }
public String getIdentity() { if (identity == null) identity = getTableAlias() + "." + getName(); return identity; }
public String getCanonicalName() { return getTable() + "." + getName(); }
private void outputDimRangeInfo() throws IOException, InterruptedException { if (col != null && minValue != null) { // output written to baseDir/colName/colName.dci-r-00000 (etc) String dimRangeFileName = col.getIdentity() + "/" + col.getName() + DIMENSION_COL_INFO_FILE_POSTFIX; mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new Text(minValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName); mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new Text(maxValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName); logger.info("write dimension range info for col : " + col.getName() + " minValue:" + minValue + " maxValue:" + maxValue); } }
private void outputDimRangeInfo(List<Tuple2<String, Tuple3<Writable, Writable, String>>> result) { if (col != null && minValue != null) { // output written to baseDir/colName/colName.dci-r-00000 (etc) String dimRangeFileName = col.getIdentity() + "/" + col.getName() + DIMENSION_COL_INFO_FILE_POSTFIX; result.add(new Tuple2<String, Tuple3<Writable, Writable, String>>(BatchConstants.CFG_OUTPUT_PARTITION, new Tuple3<Writable, Writable, String>(NullWritable.get(), new Text(minValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName))); result.add(new Tuple2<String, Tuple3<Writable, Writable, String>>(BatchConstants.CFG_OUTPUT_PARTITION, new Tuple3<Writable, Writable, String>(NullWritable.get(), new Text(maxValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName))); logger.info("write dimension range info for col : {} minValue:{} maxValue:{}", col.getName(), minValue, maxValue); } }
public String[] getEmbeddedPropertyNames(TblColRef column) { final String colName = column.getName().toLowerCase(Locale.ROOT); String[] names = nameMap.get(colName); if (names == null) { String comment = column.getColumnDesc().getComment(); // use comment to parse the structure if (!StringUtils.isEmpty(comment) && comment.contains(EMBEDDED_PROPERTY_SEPARATOR)) { names = comment.toLowerCase(Locale.ROOT).split("\\" + EMBEDDED_PROPERTY_SEPARATOR); nameMap.put(colName, names); } else if (colName.contains(separator)) { // deprecated, just be compitable for old version names = colName.toLowerCase(Locale.ROOT).split(separator); nameMap.put(colName, names); } } return names; }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } }
private static Map<String, Map<String, String>> buildTableToColumnsMap(IJoinedFlatTableDesc flatDesc) { Map<String, Map<String, String>> map = Maps.newHashMap(); List<TblColRef> colRefs = flatDesc.getAllColumns(); for (TblColRef colRef : colRefs) { String colName = colRef.getName(); String tableName = colRef.getTableRef().getTableName(); String colAlias = colRef.getTableAlias() + "_" + colRef.getName(); if (map.containsKey(tableName)) { map.get(tableName).put(colName, colAlias); } else { Map<String, String> colToAliasMap = Maps.newHashMap(); colToAliasMap.put(colName, colAlias); map.put(tableName, colToAliasMap); } } return map; }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } mos.close(); } }
@SuppressWarnings("unused") MeasureDesc findMeasure(TblColRef col) { for (MeasureDesc measure : this.context.realization.getMeasures()) { if (col.getName().equals(measure.getFunction().getRewriteFieldName())) { return measure; } } return null; }
private void outputDict(TblColRef col, Dictionary<String> dict, List<Tuple2<String, Tuple3<Writable, Writable, String>>> result) throws IOException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos)) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); result.add(new Tuple2<String, Tuple3<Writable, Writable, String>>(BatchConstants.CFG_OUTPUT_DICT, new Tuple3<Writable, Writable, String>(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName))); } }
public String getDictResPath(TblColRef col) { String r; String dictKey = col.getIdentity(); r = getDictionaries().get(dictKey); // try Kylin v1.x dict key as well if (r == null) { String v1DictKey = col.getTable() + "/" + col.getName(); r = getDictionaries().get(v1DictKey); } return r; }
/** * Get the encoding name and version for the given col from Measure FunctionDesc * @param functionDesc * @param tblColRef * @return a pair of the encoding name and encoding version */ public static final Pair<String, String> getEncoding(FunctionDesc functionDesc, TblColRef tblColRef) { String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getIdentity()); String encodingVersion = functionDesc.getConfiguration() .get(CONFIG_ENCODING_VERSION_PREFIX + tblColRef.getIdentity()); if (StringUtils.isEmpty(encoding)) { // for backward compatibility encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX + tblColRef.getName()); encodingVersion = functionDesc.getConfiguration().get(CONFIG_ENCODING_VERSION_PREFIX + tblColRef.getName()); } return new Pair<>(encoding, encodingVersion); }
private TupleExpression getCountColumnExpression(TblColRef colRef) { List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(1); TupleFilter whenFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.ISNULL); whenFilter.addChild(new ColumnTupleFilter(colRef)); whenList.add(new Pair<TupleFilter, TupleExpression>(whenFilter, new NumberTupleExpression(0))); TupleExpression elseExpr = new ColumnTupleExpression(SumDynamicFunctionDesc.mockCntCol); TupleExpression ret = new CaseTupleExpression(whenList, elseExpr); ret.setDigest("_KY_COUNT(" + colRef.getName() + ")"); return ret; } }
void init(DataModelDesc model) { table = table.toUpperCase(Locale.ROOT); if (columns != null) { StringUtil.toUpperCaseArray(columns, columns); } if (model != null) { table = model.findTable(table).getAlias(); if (columns != null) { for (int i = 0; i < columns.length; i++) { TblColRef column = model.findColumn(table, columns[i]); if (column.getColumnDesc().isComputedColumn() && !model.isFactTable(column.getTableRef())) { throw new RuntimeException("Computed Column on lookup table is not allowed"); } columns[i] = column.getName(); } } } }
public TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) { TupleInfo info = new TupleInfo(); int idx = 0; for (TblColRef col : groups) { info.setField(col.getName(), col, idx++); } TableRef sourceTable = groups.get(0).getTableRef(); for (FunctionDesc func : aggregations) { ColumnDesc colDesc = func.newFakeRewriteColumn(sourceTable.getTableDesc()); TblColRef col = sourceTable.makeFakeColumn(colDesc); info.setField(col.getName(), col, idx++); } return info; }
@Override public void serialize(CoprocessorRowType o, ByteBuffer out) { int n = o.columns.length; BytesUtil.writeVInt(o.columns.length, out); BytesUtil.writeVInt(o.bodyOffset, out); for (int i = 0; i < n; i++) { BytesUtil.writeAsciiString(o.columns[i].getTable(), out); BytesUtil.writeAsciiString(o.columns[i].getName(), out); BytesUtil.writeAsciiString(o.columns[i].getDatatype(), out); BytesUtil.writeVInt(o.columnSizes[i], out); } }
public static String getQuotedColExpressionInSourceDB(IJoinedFlatTableDesc flatDesc, TblColRef col) { if (!col.getColumnDesc().isComputedColumn()) { return quoteIdentifier(col.getTableAlias()) + "." + quoteIdentifier(col.getName()); } else { String computeExpr = col.getColumnDesc().getComputedColumnExpr(); return quoteIdentifierInSqlExpr(flatDesc, computeExpr, "`"); } } }