private void outputDimRangeInfo() throws IOException, InterruptedException { if (col != null && minValue != null) { // output written to baseDir/colName/colName.dci-r-00000 (etc) String dimRangeFileName = col.getIdentity() + "/" + col.getName() + DIMENSION_COL_INFO_FILE_POSTFIX; mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new Text(minValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName); mos.write(BatchConstants.CFG_OUTPUT_PARTITION, NullWritable.get(), new Text(maxValue.getBytes(StandardCharsets.UTF_8)), dimRangeFileName); logger.info("write dimension range info for col : " + col.getName() + " minValue:" + minValue + " maxValue:" + maxValue); } }
public static String getQuotedColExpressionInSourceDB(IJoinedFlatTableDesc flatDesc, TblColRef col) { if (!col.getColumnDesc().isComputedColumn()) { return quoteIdentifier(col.getTableAlias()) + "." + quoteIdentifier(col.getName()); } else { String computeExpr = col.getColumnDesc().getComputedColumnExpr(); return quoteIdentifierInSqlExpr(flatDesc, computeExpr, "`"); } } }
public boolean isInnerColumn() { return InnerDataTypeEnum.contains(getDatatype()); }
public String getIdentity() { if (identity == null) identity = getTableAlias() + "." + getName(); return identity; }
@Override public IReadableTable getDistinctValuesFor(TblColRef col) { return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getIdentity(), col.getType()); } }, new DictionaryProvider() {
@Override public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { TableRef tableRef = columnRef.getTableRef(); if (tableRef == null) { // un-qualified column String table = columnRef.getTable(); BytesUtil.writeUTFString(table, buffer); String columnId = columnRef.getColumnDesc().getId(); BytesUtil.writeUTFString(columnId, buffer); String columnName = columnRef.getName(); BytesUtil.writeUTFString(columnName, buffer); String dataType = columnRef.getDatatype(); BytesUtil.writeUTFString(dataType, buffer); } else { // qualified column (from model) BytesUtil.writeUTFString(_QUALIFIED_, buffer); String model = tableRef.getModel().getName(); BytesUtil.writeUTFString(model, buffer); String alias = tableRef.getAlias(); BytesUtil.writeUTFString(alias, buffer); String col = columnRef.getName(); BytesUtil.writeUTFString(col, buffer); } }
@Override public void serialize(CoprocessorRowType o, ByteBuffer out) { int n = o.columns.length; BytesUtil.writeVInt(o.columns.length, out); BytesUtil.writeVInt(o.bodyOffset, out); for (int i = 0; i < n; i++) { BytesUtil.writeAsciiString(o.columns[i].getTable(), out); BytesUtil.writeAsciiString(o.columns[i].getName(), out); BytesUtil.writeAsciiString(o.columns[i].getDatatype(), out); BytesUtil.writeVInt(o.columnSizes[i], out); } }
TableDesc table = metaMgr.getTableDesc(col.getTable(), prjCache.project); if (table == null) { logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but its table is not found by MetadataManager"); return false; if (!col.getColumnDesc().isComputedColumn()) { ColumnDesc foundCol = table.findColumnByName(col.getName()); if (col.getColumnDesc().equals(foundCol) == false) { logger.error("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "', but it is not equal to '" + foundCol + "' according to MetadataManager"); return false; logger.warn("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
col = findColumn(pks[i]); if (col == null || col.getTableRef().equals(dimTable) == false) { throw new IllegalStateException("Can't find PK column " + pks[i] + " in table " + dimTable); pks[i] = col.getIdentity(); pkCols[i] = col; throw new IllegalStateException("Can't find FK column " + fks[i]); fks[i] = col.getIdentity(); fkCols[i] = col; TableRef fkTable = fkCols[0].getTableRef(); if (pkCols.length == 0 || fkCols.length == 0) throw new IllegalStateException("Missing join columns on table " + dimTable); if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns(); int taskId = context.getTaskAttemptID().getTaskID().getId(); col = uhcColumns.get(taskId); logger.info("column name: " + col.getIdentity()); if (cube.getDescriptor().getShardByColumns().contains(col)) { //for ShardByColumns builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0, null); } else { //for GlobalDictionaryColumns String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR); DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype()); String builderClass = cubeDesc.getDictionaryBuilderClass(col); builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); builder.init(dictionaryInfo, 0, hdfsDir); } }
public String getDictResPath(TblColRef col) { String r; String dictKey = col.getIdentity(); r = getDictionaries().get(dictKey); // try Kylin v1.x dict key as well if (r == null) { String v1DictKey = col.getTable() + "/" + col.getName(); r = getDictionaries().get(v1DictKey); } return r; }
void init(DataModelDesc model) { table = table.toUpperCase(Locale.ROOT); if (columns != null) { StringUtil.toUpperCaseArray(columns, columns); } if (model != null) { table = model.findTable(table).getAlias(); if (columns != null) { for (int i = 0; i < columns.length; i++) { TblColRef column = model.findColumn(table, columns[i]); if (column.getColumnDesc().isComputedColumn() && !model.isFactTable(column.getTableRef())) { throw new RuntimeException("Computed Column on lookup table is not allowed"); } columns[i] = column.getName(); } } } }
private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) { for (TblColRef col : allColumns) { if (allColumnDescs.contains(col.getColumnDesc()) == false) return false; } return true; }
private static Map<String, Map<String, String>> buildTableToColumnsMap(IJoinedFlatTableDesc flatDesc) { Map<String, Map<String, String>> map = Maps.newHashMap(); List<TblColRef> colRefs = flatDesc.getAllColumns(); for (TblColRef colRef : colRefs) { String colName = colRef.getName(); String tableName = colRef.getTableRef().getTableName(); String colAlias = colRef.getTableAlias() + "_" + colRef.getName(); if (map.containsKey(tableName)) { map.get(tableName).put(colName, colAlias); } else { Map<String, String> colToAliasMap = Maps.newHashMap(); colToAliasMap.put(colName, colAlias); map.put(tableName, colToAliasMap); } } return map; }
public boolean partitionColumnIsTimeMillis() { if (partitionDateColumnRef == null) return false; DataType type = partitionDateColumnRef.getType(); return type.isBigInt() && !DateFormat.isDatePattern(partitionDateFormat); }
private DictionaryInfo createDictionaryInfo(TblColRef col, IReadableTable inpTable) throws IOException { TableSignature inputSig = inpTable.getSignature(); if (inputSig == null) // table does not exists throw new IllegalStateException("Input table does not exist: " + inpTable); DictionaryInfo dictInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype(), inputSig); return dictInfo; }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
private void markExposedTablesAndColumns(ProjectCache prjCache, IRealization realization) { if (!realization.isReady()) { return; } for (TblColRef col : realization.getAllColumns()) { TableCache tableCache = prjCache.tables.get(col.getTable()); prjCache.exposedTables.add(tableCache.tableDesc); tableCache.exposed = true; tableCache.exposedColumns.add(col.getColumnDesc()); } }