/** * below method is to check whether data type is present in the data type array * * @param dataType data type to be searched * @param dataTypes all data types * @return if data type is present */ public static boolean hasDataType(DataType dataType, DataType[] dataTypes) { for (int i = 0; i < dataTypes.length; i++) { if (dataType.equals(dataTypes[i])) { return true; } } return false; }
/** * Overridden equals method for columnSchema */ @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof ColumnSchema)) { return false; } ColumnSchema other = (ColumnSchema) obj; if (columnName == null) { if (other.columnName != null) { return false; } } else if (!columnName.equalsIgnoreCase(other.columnName)) { return false; } if (dataType == null) { if (other.dataType != null) { return false; } } else if (!dataType.equals(other.dataType)) { return false; } return true; }
@Override public void encode(int rowId, double value) { if (targetDataType.equals(DataTypes.BYTE)) { encodedPage.putByte(rowId, (byte) (max - Math.round(value * factor))); } else if (targetDataType.equals(DataTypes.SHORT)) { encodedPage.putShort(rowId, (short) (max - Math.round(value * factor))); } else if (targetDataType.equals(DataTypes.SHORT_INT)) { encodedPage.putShortInt(rowId, (int) (max - Math.round(value * factor))); } else if (targetDataType.equals(DataTypes.INT)) { encodedPage.putInt(rowId, (int) (max - Math.round(value * factor))); } else if (targetDataType.equals(DataTypes.DOUBLE)) { encodedPage.putDouble(rowId, value); } else { throw new RuntimeException("internal error: " + debugInfo()); } }
@Override public void encode(int rowId, float value) { if (targetDataType.equals(DataTypes.BYTE)) { encodedPage.putByte(rowId, (byte) (max - (value * factor))); } else if (targetDataType.equals(DataTypes.SHORT)) { encodedPage.putShort(rowId, (short) (max - (value * factor))); } else if (targetDataType.equals(DataTypes.SHORT_INT)) { encodedPage.putShortInt(rowId, (int) (max - (value * factor))); } else if (targetDataType.equals(DataTypes.INT)) { encodedPage.putInt(rowId, (int) (max - (value * factor))); } else { throw new RuntimeException("internal error: " + debugInfo()); } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } StructField other = (StructField) obj; if (!this.fieldName.equalsIgnoreCase(other.fieldName)) { return false; } if (!this.dataType.equals(other.dataType)) { return false; } if (children == null) { if (other.children != null) { return false; } } else if (other.children == null) { return false; } else if (!children.equals(other.children)) { return false; } return true; } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } MapType other = (MapType) obj; if (!this.getName().equalsIgnoreCase(other.getName())) { return false; } if (!this.keyType.equals(other.keyType)) { return false; } if (!this.valueType.equals(other.valueType)) { return false; } return true; }
/** * Below method will be used to get the format for * date or timestamp data type from property. This * is added to avoid the code duplication * * @param dataType * @return format */ public static String getFormatFromProperty(DataType dataType) { if (dataType.equals(DataTypes.DATE)) { return CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT); } else if (dataType.equals(DataTypes.TIMESTAMP)) { return CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT); } else { return null; } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } ArrayType other = (ArrayType) obj; if (!this.getName().equalsIgnoreCase(other.getName())) { return false; } if (!this.getElementType().equals(other.getElementType())) { return false; } return true; }
/** * This method will fill the direct dictionary column data * * @param vector * @param columnVectorInfo * @param defaultValue */ private void fillDirectDictionaryData(CarbonColumnVector vector, ColumnVectorInfo columnVectorInfo, Object defaultValue) { if (null != defaultValue) { if (columnVectorInfo.directDictionaryGenerator.getReturnType().equals(DataTypes.INT)) { vector.putInts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (int) defaultValue); } else { vector.putLongs(columnVectorInfo.vectorOffset, columnVectorInfo.size, (long) defaultValue); } } else { vector.putNulls(columnVectorInfo.vectorOffset, columnVectorInfo.size); } }
/** * traverse through the columns of complex column specified in local dictionary include, * and set local dictionary for all the string and varchar child columns * @param allColumns * @param dimensionOrdinal * @param childColumnCount * @return */ private static int setLocalDictForComplexColumns(List<ColumnSchema> allColumns, int dimensionOrdinal, int childColumnCount) { for (int i = 0; i < childColumnCount; i++) { ColumnSchema column = allColumns.get(dimensionOrdinal); if (column.getNumberOfChild() > 0) { dimensionOrdinal++; setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild()); } else { if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)) { column.setLocalDictColumn(true); } } dimensionOrdinal++; } return dimensionOrdinal; }
@Override public boolean equals(Object obj) { if (obj instanceof Field) { Field field = (Field) obj; if ((!this.getDataType().equals(field.getDataType())) || (!this.getFieldName().equals(field.getFieldName())) || (!(this.getSchemaOrdinal() == (field.getSchemaOrdinal()))) ) { return false; } } else { return false; } return true; } }
protected void addValue2BloomIndex(int indexColIdx, Object value) { byte[] indexValue; // convert measure to bytes // convert non-dict dimensions to simple bytes without length // convert internal-dict dimensions to simple bytes without any encode if (indexColumns.get(indexColIdx).isMeasure()) { // NULL value of all measures are already processed in `ColumnPage.getData` // or `RawBytesReadSupport.readRow` with actual data type // Carbon stores boolean as byte. Here we convert it for `getValueAsBytes` if (indexColumns.get(indexColIdx).getDataType().equals(DataTypes.BOOLEAN)) { value = BooleanConvert.boolean2Byte((Boolean)value); } indexValue = CarbonUtil.getValueAsBytes(indexColumns.get(indexColIdx).getDataType(), value); } else { if (indexColumns.get(indexColIdx).hasEncoding(Encoding.DICTIONARY) || indexColumns.get(indexColIdx).hasEncoding(Encoding.DIRECT_DICTIONARY)) { indexValue = convertDictionaryValue(indexColIdx, value); } else { indexValue = convertNonDictionaryValue(indexColIdx, value); } } if (indexValue.length == 0) { indexValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } indexBloomFilters.get(indexColIdx).add(new Key(indexValue)); }
private static ColumnPage createPage(ColumnPageEncoderMeta columnPageEncoderMeta, int pageSize) { if (DataTypes.isDecimal(columnPageEncoderMeta.getStoreDataType())) { return createDecimalPage(columnPageEncoderMeta, pageSize); } else if (columnPageEncoderMeta.getStoreDataType().equals(BYTE_ARRAY)) { return createVarLengthPage(columnPageEncoderMeta, pageSize); } else { return createFixLengthPage(columnPageEncoderMeta, pageSize); } }
/** * create column vector for newly added dimension columns */ private void createVectorForNewlyAddedDimensions() { for (int i = 0; i < queryDimensions.length; i++) { if (!dimensionInfo.getDimensionExists()[i]) { // add a dummy column vector result collector object ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); columnVectorInfo.dimension = queryDimensions[i]; if (queryDimensions[i].getDimension().getDataType().equals(DataTypes.TIMESTAMP) || queryDimensions[i].getDimension().getDataType().equals(DataTypes.DATE)) { columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType()); } allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } } }
if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column .hasEncoding(Encoding.DICTIONARY)) { column.setLocalDictColumn(true); if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column .hasEncoding(Encoding.DICTIONARY)) { if (!Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName())) { continue; if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY) && localDictIncludeColumns.toLowerCase()
if (carbonColumn.getDataType().equals(DataTypes.BOOLEAN)) { convertedValue = BooleanConvert.boolean2Byte((Boolean)convertedValue);