/** * This method will give storage order column list */ public List<CarbonColumn> getStreamStorageOrderColumn(String tableName) { List<CarbonDimension> dimensions = tableDimensionsMap.get(tableName); List<CarbonMeasure> measures = tableMeasuresMap.get(tableName); List<CarbonColumn> columnList = new ArrayList<>(dimensions.size() + measures.size()); List<CarbonColumn> complexDimensionList = new ArrayList<>(dimensions.size()); for (CarbonColumn column : dimensions) { if (column.isComplex()) { complexDimensionList.add(column); } else { columnList.add(column); } } columnList.addAll(complexDimensionList); for (CarbonColumn column : measures) { if (!(column.getColName().equals("default_dummy_measure"))) { columnList.add(column); } } return columnList; }
public static boolean[] getNoDictionaryMapping(CarbonColumn[] carbonColumns) { List<Boolean> noDictionaryMapping = new ArrayList<Boolean>(); for (CarbonColumn column : carbonColumns) { // for complex type need to break the loop if (column.isComplex()) { break; } if (!column.hasEncoding(Encoding.DICTIONARY) && column.isDimension()) { noDictionaryMapping.add(true); } else if (column.isDimension()) { noDictionaryMapping.add(false); } } return ArrayUtils .toPrimitive(noDictionaryMapping.toArray(new Boolean[noDictionaryMapping.size()])); }
/** * Preparing the boolean [] to map whether the dimension is no Dictionary or not. */ public static boolean[] getNoDictionaryMapping(DataField[] fields) { List<Boolean> noDictionaryMapping = new ArrayList<Boolean>(); for (DataField field : fields) { // for complex type need to break the loop if (field.getColumn().isComplex()) { break; } if (!field.hasDictionaryEncoding() && field.getColumn().isDimension()) { noDictionaryMapping.add(true); } else if (field.getColumn().isDimension()) { noDictionaryMapping.add(false); } } return ArrayUtils .toPrimitive(noDictionaryMapping.toArray(new Boolean[noDictionaryMapping.size()])); }
/** * Preparing the boolean [] to map whether the dimension is varchar data type or not. */ public static boolean[] getIsVarcharColumnMapping(DataField[] fields) { List<Boolean> isVarcharColumnMapping = new ArrayList<Boolean>(); for (DataField field : fields) { // for complex type need to break the loop if (field.getColumn().isComplex()) { break; } if (field.getColumn().isDimension()) { isVarcharColumnMapping.add( field.getColumn().getColumnSchema().getDataType() == DataTypes.VARCHAR); } } return ArrayUtils.toPrimitive( isVarcharColumnMapping.toArray(new Boolean[isVarcharColumnMapping.size()])); }
public static void getComplexNoDictionaryMapping(DataField[] dataFields, List<Integer> complexNoDictionary) { // save the Ordinal Number in the List. for (DataField field : dataFields) { if (field.getColumn().isComplex()) { // get the childs. getComplexNoDictionaryMapping( ((CarbonDimension) field.getColumn()).getListOfChildDimensions(), complexNoDictionary); } } }
/** * Get the data types of the no dictionary and the complex dimensions of the table * * @return */ public CarbonColumn[] getNoDictAndComplexDimensions() { List<Integer> noDicOrCompIndexes = new ArrayList<>(dataFields.length); int noDicCount = 0; for (int i = 0; i < dataFields.length; i++) { if (dataFields[i].getColumn().isDimension() && ( !(dataFields[i].getColumn().hasEncoding(Encoding.DICTIONARY)) || dataFields[i].getColumn() .isComplex())) { noDicOrCompIndexes.add(i); noDicCount++; } } CarbonColumn[] dims = new CarbonColumn[noDicCount]; for (int i = 0; i < dims.length; i++) { dims[i] = dataFields[noDicOrCompIndexes.get(i)].getColumn(); } return dims; }
private void convertComplexDataType(Map<Integer, GenericDataType> dataFieldsWithComplexDataType) { DataField[] srcDataField = configuration.getDataFields(); FieldEncoderFactory fieldConverterFactory = FieldEncoderFactory.getInstance(); String nullFormat = configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT) .toString(); boolean isEmptyBadRecord = Boolean.parseBoolean( configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD) .toString()); for (int i = 0; i < srcDataField.length; i++) { if (srcDataField[i].getColumn().isComplex()) { // create a ComplexDataType dataFieldsWithComplexDataType.put(srcDataField[i].getColumn().getOrdinal(), fieldConverterFactory .createComplexDataType(srcDataField[i], configuration.getTableIdentifier(), null, false, null, i, nullFormat, isEmptyBadRecord)); } } }
public void setDataFields(DataField[] dataFields) { this.dataFields = dataFields; // set counts for each column category for (DataField dataField : dataFields) { CarbonColumn column = dataField.getColumn(); if (column.isDimension()) { dimensionCount++; if (column.isComplex()) { if (!dataField.hasDictionaryEncoding()) { complexNonDictionaryColumnCount++; } else { complexDictionaryColumnCount++; } } else if (!dataField.hasDictionaryEncoding()) { noDictionaryCount++; } } if (column.isMeasure()) { measureCount++; } } }
public static GenericQueryType[] getComplexDimensions(CarbonTable carbontable, CarbonColumn[] carbonColumns, Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache) throws IOException { GenericQueryType[] queryTypes = new GenericQueryType[carbonColumns.length]; for (int i = 0; i < carbonColumns.length; i++) { if (carbonColumns[i].isComplex()) { if (DataTypes.isArrayType(carbonColumns[i].getDataType())) { queryTypes[i] = new ArrayQueryType(carbonColumns[i].getColName(), carbonColumns[i].getColName(), i); } else if (DataTypes.isStructType(carbonColumns[i].getDataType())) { queryTypes[i] = new StructQueryType(carbonColumns[i].getColName(), carbonColumns[i].getColName(), i); } else { throw new UnsupportedOperationException( carbonColumns[i].getDataType().getName() + " is not supported"); } fillChildren(carbontable, queryTypes[i], (CarbonDimension) carbonColumns[i], i, cache); } } return queryTypes; }
/** * This initialization is done inside executor task * for column dictionary involved in decoding. * * @param carbonColumns column list * @param carbonTable table identifier */ @Override public void initialize(CarbonColumn[] carbonColumns, CarbonTable carbonTable) throws IOException { this.carbonColumns = carbonColumns; dictionaries = new Dictionary[carbonColumns.length]; dataTypes = new DataType[carbonColumns.length]; for (int i = 0; i < carbonColumns.length; i++) { if (carbonColumns[i].hasEncoding(Encoding.DICTIONARY) && !carbonColumns[i] .hasEncoding(Encoding.DIRECT_DICTIONARY) && !carbonColumns[i].isComplex()) { CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider .createCache(CacheType.FORWARD_DICTIONARY); dataTypes[i] = carbonColumns[i].getDataType(); String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() .get(CarbonCommonConstants.DICTIONARY_PATH); dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier( carbonTable.getAbsoluteTableIdentifier(), carbonColumns[i].getColumnIdentifier(), dataTypes[i], dictionaryPath)); } else { dataTypes[i] = carbonColumns[i].getDataType(); } } }
CarbonColumn column = dataField.getColumn(); if (!dataField.hasDictionaryEncoding()) { if (!column.isComplex() && column.getDataType() == DataTypes.VARCHAR) { varcharDimIdxInNoDict.add(noDictionayDimensionIndex);
!dataField.getColumn().isComplex()) { return new DirectDictionaryFieldConverterImpl(dataField, nullFormat, index, isEmptyBadRecord); } else if (dataField.getColumn().hasEncoding(Encoding.DICTIONARY) && !dataField.getColumn().isComplex()) { DictionaryColumnUniqueIdentifier identifier = null; nullFormat, index, null, false, null, isEmptyBadRecord, identifier); } else if (dataField.getColumn().isComplex()) { return new ComplexFieldConverterImpl( createComplexDataType(dataField, absoluteTableIdentifier,
column.setTimestampFormat(loadModel.getTimestampformat()); if (column.isComplex()) { complexDataFields.add(dataField); List<CarbonDimension> childDimensions =
condExpression = (ConditionalExpression) expression; column = condExpression.getColumnList().get(0).getCarbonColumn(); if (condExpression.isSingleColumn() && ! column.isComplex()) { condExpression = (ConditionalExpression) expression; if ((condExpression.getColumnList().get(0).getCarbonColumn()
fields[i] = new StructField(carbonColumn.getColName(), convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null); } else if (carbonColumn.isComplex()) { fields[i] = new StructField(carbonColumn.getColName(), convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);