public boolean hasDictionaryEncoding() { return column.hasEncoding(Encoding.DICTIONARY); }
public static boolean[] getNoDictionaryMapping(CarbonColumn[] carbonColumns) { List<Boolean> noDictionaryMapping = new ArrayList<Boolean>(); for (CarbonColumn column : carbonColumns) { // for complex type need to break the loop if (column.isComplex()) { break; } if (!column.hasEncoding(Encoding.DICTIONARY) && column.isDimension()) { noDictionaryMapping.add(true); } else if (column.isDimension()) { noDictionaryMapping.add(false); } } return ArrayUtils .toPrimitive(noDictionaryMapping.toArray(new Boolean[noDictionaryMapping.size()])); }
protected void addValue2BloomIndex(int indexColIdx, Object value) { byte[] indexValue; // convert measure to bytes // convert non-dict dimensions to simple bytes without length // convert internal-dict dimensions to simple bytes without any encode if (indexColumns.get(indexColIdx).isMeasure()) { // NULL value of all measures are already processed in `ColumnPage.getData` // or `RawBytesReadSupport.readRow` with actual data type // Carbon stores boolean as byte. Here we convert it for `getValueAsBytes` if (indexColumns.get(indexColIdx).getDataType().equals(DataTypes.BOOLEAN)) { value = BooleanConvert.boolean2Byte((Boolean)value); } indexValue = CarbonUtil.getValueAsBytes(indexColumns.get(indexColIdx).getDataType(), value); } else { if (indexColumns.get(indexColIdx).hasEncoding(Encoding.DICTIONARY) || indexColumns.get(indexColIdx).hasEncoding(Encoding.DIRECT_DICTIONARY)) { indexValue = convertDictionaryValue(indexColIdx, value); } else { indexValue = convertNonDictionaryValue(indexColIdx, value); } } if (indexValue.length == 0) { indexValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } indexBloomFilters.get(indexColIdx).add(new Key(indexValue)); }
/** * Get the data types of the no dictionary and the complex dimensions of the table * * @return */ public CarbonColumn[] getNoDictAndComplexDimensions() { List<Integer> noDicOrCompIndexes = new ArrayList<>(dataFields.length); int noDicCount = 0; for (int i = 0; i < dataFields.length; i++) { if (dataFields[i].getColumn().isDimension() && ( !(dataFields[i].getColumn().hasEncoding(Encoding.DICTIONARY)) || dataFields[i].getColumn() .isComplex())) { noDicOrCompIndexes.add(i); noDicCount++; } } CarbonColumn[] dims = new CarbonColumn[noDicCount]; for (int i = 0; i < dims.length; i++) { dims[i] = dataFields[noDicOrCompIndexes.get(i)].getColumn(); } return dims; }
else if (carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
/** * This initialization is done inside executor task * for column dictionary involved in decoding. * * @param carbonColumns column list * @param carbonTable table identifier */ @Override public void initialize(CarbonColumn[] carbonColumns, CarbonTable carbonTable) throws IOException { this.carbonColumns = carbonColumns; dictionaries = new Dictionary[carbonColumns.length]; dataTypes = new DataType[carbonColumns.length]; for (int i = 0; i < carbonColumns.length; i++) { if (carbonColumns[i].hasEncoding(Encoding.DICTIONARY) && !carbonColumns[i] .hasEncoding(Encoding.DIRECT_DICTIONARY) && !carbonColumns[i].isComplex()) { CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider .createCache(CacheType.FORWARD_DICTIONARY); dataTypes[i] = carbonColumns[i].getDataType(); String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties() .get(CarbonCommonConstants.DICTIONARY_PATH); dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier( carbonTable.getAbsoluteTableIdentifier(), carbonColumns[i].getColumnIdentifier(), dataTypes[i], dictionaryPath)); } else { dataTypes[i] = carbonColumns[i].getDataType(); } } }
directDictionaryGenerators = new DirectDictionaryGenerator[storageColumns.length]; for (int i = 0; i < storageColumns.length; i++) { if (storageColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) { directDictionaryGenerators[i] = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(storageColumns[i].getDataType());
} else if (carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY) || carbonColumn.hasEncoding(Encoding.DICTIONARY)) {
@Override public void initialize() throws IOException { super.initialize(); // if logger is enabled then raw data will be required. RowConverterImpl rowConverter = new RowConverterImpl(configuration.getDataFields(), configuration, null); rowConverter.initialize(); configuration.setCardinalityFinder(rowConverter); noDictionaryMapping = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()); dataFieldsWithComplexDataType = new HashMap<>(); convertComplexDataType(dataFieldsWithComplexDataType); dataTypes = new DataType[configuration.getDataFields().length]; for (int i = 0; i < dataTypes.length; i++) { if (configuration.getDataFields()[i].getColumn().hasEncoding(Encoding.DICTIONARY)) { dataTypes[i] = DataTypes.INT; } else { dataTypes[i] = configuration.getDataFields()[i].getColumn().getDataType(); } } orderOfData = arrangeData(configuration.getDataFields(), configuration.getHeader()); }
carbonColumn.hasEncoding(Encoding.DICTIONARY)); for (CarbonDimension dimension : listOfChildDimensions) { arrayDataType.addChildren( carbonColumn.hasEncoding(Encoding.DICTIONARY)); for (CarbonDimension dimension : dimensions) { structDataType.addChildren(
CarbonColumn carbonColumn = currentCondExpression.getColumnList().get(0).getCarbonColumn(); if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) { return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()); .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0) .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) { if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft()) && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || ( .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0) .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) { if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft()) && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || ( condExpression = (ConditionalExpression) expression; if ((condExpression.getColumnList().get(0).getCarbonColumn() .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0) .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) || (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) { return new ConditionalFilterResolverImpl(expression, true, true,
CarbonColumn carbonColumn = carbonColumns[i]; if (carbonColumn.isDimension()) { if (carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY)) { DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(carbonColumn.getDataType()); fields[i] = new StructField(carbonColumn.getColName(), convertCarbonToSparkDataType(generator.getReturnType()), true, null); } else if (!carbonColumn.hasEncoding(Encoding.DICTIONARY)) { fields[i] = new StructField(carbonColumn.getColName(), convertCarbonToSparkDataType(carbonColumn.getDataType()), true, null);
if (dataField.getColumn().hasEncoding(Encoding.DIRECT_DICTIONARY) && !dataField.getColumn().isComplex()) { return new DirectDictionaryFieldConverterImpl(dataField, nullFormat, index, isEmptyBadRecord); } else if (dataField.getColumn().hasEncoding(Encoding.DICTIONARY) && !dataField.getColumn().isComplex()) { DictionaryColumnUniqueIdentifier identifier = null;