: cubeDesc.listDimensionColumnsExcludingDerived(true); Set<TblColRef> filterCols = Sets.newHashSet(dynFunc.getFilterColumnSet()); filterCols.removeAll(definedCols);
@Override protected void doCleanup(Context context) throws IOException, InterruptedException { if (isStatistics) { //output the hll info; List<Long> allCuboids = Lists.newArrayList(); allCuboids.addAll(cuboidHLLMap.keySet()); Collections.sort(allCuboids); logMapperAndCuboidStatistics(allCuboids); // for human check outputStatistics(allCuboids); } else { //dimension col if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) { outputDimRangeInfo(); } // dic col if (buildDictInReducer) { Dictionary<String> dict = builder.build(); outputDict(col, dict); } } mos.close(); }
if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col) || isSelectAll) { sqlDigest.allColumns.add(col); if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) {
public static Map<TblColRef, Dictionary<String>> buildDictionary(final CubeInstance cubeInstance, Iterable<List<String>> recordList) throws IOException { final List<TblColRef> columnsNeedToBuildDictionary = cubeInstance.getDescriptor() .listDimensionColumnsExcludingDerived(true); final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap(); int index = 0; for (TblColRef column : columnsNeedToBuildDictionary) { tblColRefMap.put(index++, column); } HashMap<TblColRef, Dictionary<String>> result = Maps.newHashMap(); HashMultimap<TblColRef, String> valueMap = HashMultimap.create(); for (List<String> row : recordList) { for (int i = 0; i < row.size(); i++) { String cell = row.get(i); if (tblColRefMap.containsKey(i)) { valueMap.put(tblColRefMap.get(i), cell); } } } for (TblColRef tblColRef : valueMap.keySet()) { Set<String> values = valueMap.get(tblColRef); Dictionary<String> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(values)); result.put(tblColRef, dict); } return result; }
Set<TblColRef> allCols = cube.getAllColumns(); Set<TblColRef> dictCols = desc.getAllColumnsNeedDictionaryBuilt(); List<TblColRef> dimCols = desc.listDimensionColumnsExcludingDerived(true); for (TblColRef colRef : allCols) { if (dictCols.contains(colRef)) {
List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(true); checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%s) doesn't match dimensions columns count (%s)",
logAFewRows(value); if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col) && col.getType().needCompare()) { if (minValue == null || col.getType().compare(minValue, value) > 0) { minValue = value; } else { if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) { outputDimRangeInfo(result);
Set<TblColRef> realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true)); for (JoinTableDesc joinTable : model.getJoinTables()) { JoinDesc join = joinTable.getJoin();
logAFewRows(value); if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col) && col.getType().needCompare()) { if (minValue == null || col.getType().compare(minValue, value) > 0) { minValue = value;
private void updateSegment(CubeSegment segment) throws IOException { final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef(); for (TblColRef dimColRef : segment.getCubeDesc().listDimensionColumnsExcludingDerived(true)) { if (!dimColRef.getType().needCompare()) continue;
for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived(false)) { initAddColumn(col);
public Map<String, TblColRef> buildColumnNameAbbreviation() { Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>(); for (TblColRef col : listDimensionColumnsExcludingDerived()) { r.put(col.getName(), col); } return r; }
for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived()) { dimensionIndexMap.put(colName(col.getCanonicalName()), columnIndex); columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
public void init(KylinConfig config, Map<String, TableDesc> tables) { this.errors.clear(); this.config = config; if (this.modelName == null || this.modelName.length() == 0) { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); } for (DimensionDesc dim : dimensions) { dim.init(this, tables); } sortDimAndMeasure(); initDimensionColumns(tables); initMeasureColumns(tables); rowkey.init(this); if (hbaseMapping != null) { hbaseMapping.init(this); } initMeasureReferenceToColumnFamily(); // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); if (rowkey.getRowKeyColumns().length != dimCols.size()) { addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); } }
public Map<String, TblColRef> buildColumnNameAbbreviation() { Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>(); for (TblColRef col : listDimensionColumnsExcludingDerived()) { r.put(col.getName(), col); } return r; }
: cubeDesc.listDimensionColumnsExcludingDerived(true); Set<TblColRef> filterCols = Sets.newHashSet(dynFunc.getFilterColumnSet()); filterCols.removeAll(definedCols);
public static Map<TblColRef, Dictionary<String>> buildDictionary(final CubeInstance cubeInstance, Iterable<List<String>> recordList) throws IOException { final List<TblColRef> columnsNeedToBuildDictionary = cubeInstance.getDescriptor() .listDimensionColumnsExcludingDerived(true); final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap(); int index = 0; for (TblColRef column : columnsNeedToBuildDictionary) { tblColRefMap.put(index++, column); } HashMap<TblColRef, Dictionary<String>> result = Maps.newHashMap(); HashMultimap<TblColRef, String> valueMap = HashMultimap.create(); for (List<String> row : recordList) { for (int i = 0; i < row.size(); i++) { String cell = row.get(i); if (tblColRefMap.containsKey(i)) { valueMap.put(tblColRefMap.get(i), cell); } } } for (TblColRef tblColRef : valueMap.keySet()) { Set<String> values = valueMap.get(tblColRef); Dictionary<String> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(values)); result.put(tblColRef, dict); } return result; }
@Override protected void doCleanup(Context context) throws IOException, InterruptedException { if (isStatistics) { //output the hll info; List<Long> allCuboids = Lists.newArrayList(); allCuboids.addAll(cuboidHLLMap.keySet()); Collections.sort(allCuboids); logMapperAndCuboidStatistics(allCuboids); // for human check outputStatistics(allCuboids); } else { //dimension col if (cubeDesc.listDimensionColumnsExcludingDerived(true).contains(col)) { outputDimRangeInfo(); } // dic col if (buildDictInReducer) { Dictionary<String> dict = builder.build(); outputDict(col, dict); } } mos.close(); }
public void init(KylinConfig config, Map<String, TableDesc> tables) { this.errors.clear(); this.config = config; if (this.modelName == null || this.modelName.length() == 0) { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); } for (DimensionDesc dim : dimensions) { dim.init(this, tables); } sortDimAndMeasure(); initDimensionColumns(tables); initMeasureColumns(tables); rowkey.init(this); if (hbaseMapping != null) { hbaseMapping.init(this); } initMeasureReferenceToColumnFamily(); // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); if (rowkey.getRowKeyColumns().length != dimCols.size()) { addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "); rowkeys are " + Arrays.toString(rowkey.getRowKeyColumns()) + ", dimension columns are " + dimCols.toString()); } }
for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived(false)) { initAddColumn(col);