.put(queryDimensions[i].getDimension(), buffer); mergedComplexDimensionDataMap.put(complexParentOrdinal, childColumnByteBuffer); } else if (!queryDimensions[i].getDimension().isComplex()) {
if (filterDimension.isComplex()) { continue;
public static void getComplexNoDictionaryMapping(List<CarbonDimension> carbonDimensions, List<Integer> complexNoDictionary) { for (CarbonDimension carbonDimension : carbonDimensions) { if (carbonDimension.isComplex()) { getComplexNoDictionaryMapping(carbonDimension.getListOfChildDimensions(), complexNoDictionary); } else { // This is primitive type. Check the encoding for NoDictionary. if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) { complexNoDictionary.add(carbonDimension.getOrdinal()); } } } }
public TableSpec(CarbonTable carbonTable) { this.carbonTable = carbonTable; List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getTableName()); List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getTableName()); // first calculate total number of columnar field considering column group and complex column numSimpleDimensions = 0; for (CarbonDimension dimension : dimensions) { if (!dimension.isComplex()) { numSimpleDimensions++; } } dimensionSpec = new DimensionSpec[dimensions.size()]; measureSpec = new MeasureSpec[measures.size()]; noDictionaryDimensionSpec = new ArrayList<>(); addDimensions(dimensions); addMeasures(measures); }
public void init(FilterResolverIntf filterExp) { if (filterExp != null) { // cache all columns List<CarbonColumn> minMaxCacheColumns = new ArrayList<>(); for (CarbonDimension dimension : carbonTable.getDimensions()) { if (!dimension.isComplex()) { minMaxCacheColumns.add(dimension); } } minMaxCacheColumns.addAll(carbonTable.getMeasures()); // prepare cardinality of all dimensions List<ColumnSchema> listOfColumns = carbonTable.getTableInfo().getFactTable().getListOfColumns(); int[] columnCardinality = new int[listOfColumns.size()]; for (int index = 0; index < columnCardinality.length; index++) { columnCardinality[index] = Integer.MAX_VALUE; } // initial filter executor SegmentProperties segmentProperties = new SegmentProperties(listOfColumns, columnCardinality); filterExecuter = FilterUtil.getFilterExecuterTree( filterExp, segmentProperties, null, minMaxCacheColumns); } }
private void addDimensions(List<CarbonDimension> dimensions) { int dimIndex = 0; for (int i = 0; i < dimensions.size(); i++) { CarbonDimension dimension = dimensions.get(i); if (dimension.isComplex()) { DimensionSpec spec = new DimensionSpec(ColumnType.COMPLEX, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension .isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else if (dimension.isGlobalDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.GLOBAL_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } } }
if (null != dimensionFromCurrentBlock) { if (dimensionFromCurrentBlock.isComplex()) { complexDimensions.add(dimensionFromCurrentBlock); continue;
private int[] getComplexDimensionParentBlockIndexes(List<ProjectionDimension> queryDimensions) { List<Integer> parentBlockIndexList = new ArrayList<Integer>(); for (ProjectionDimension queryDimension : queryDimensions) { if (queryDimension.getDimension().getDataType().isComplexType()) { if (null != queryDimension.getDimension().getComplexParentDimension()) { if (queryDimension.getDimension().isComplex()) { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } else { parentBlockIndexList.add(queryDimension.getParentDimension().getOrdinal()); } } else { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } } } return ArrayUtils .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()])); }
} else { if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) {
boolean hasComplex = false; for (ProjectionDimension projectionDimension : queryModel.getProjectionDimensions()) { if (projectionDimension.getDimension().isComplex()) { hasComplex = true; break;
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (queryDimensions[i].getDimension().isComplex()) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); complexList.add(columnVectorInfo);
if (dimension.getDimension().isComplex()) { complexDimension = dimension.getDimension();
int noDicAndComp = 0; for (CarbonDimension dim : allDimensions) { if (!dim.isComplex() && !dim.hasEncoding(Encoding.DICTIONARY) && dim.getDataType() == DataTypes.VARCHAR) {
CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null); } else if (dim.getDimension().isComplex()) { fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,