private TableSpec.ColumnSpec getColumnSpec(int columnPageIndex, LocalDictionaryGenerator localDictionaryGenerator) { if ((localDictionaryGenerator == null) && isColumnPageBasedOnDataType(columnPageIndex)) { return TableSpec.ColumnSpec .newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(), complexColumnInfoList.get(columnPageIndex).getColumnDataTypes(), complexColumnInfoList.get(columnPageIndex).getComplexColumnType()); } else { return TableSpec.ColumnSpec .newInstance(complexColumnInfoList.get(columnPageIndex).getColumnNames(), DataTypes.BYTE_ARRAY, complexColumnInfoList.get(columnPageIndex).getComplexColumnType()); } }
TableSpec.ColumnSpec.newInstance(columnName, DataTypes.BYTE_ARRAY, columnType); ColumnPage dictionaryColumnPage = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor),
VarLengthColumnPageBase(ColumnPageEncoderMeta columnPageEncoderMeta, int pageSize) { super(columnPageEncoderMeta, pageSize); TableSpec.ColumnSpec spec = TableSpec.ColumnSpec.newInstance( columnPageEncoderMeta.getColumnSpec().getFieldName(), DataTypes.INT, ColumnType.MEASURE); try { rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, columnPageEncoderMeta.getCompressorName()), pageSize); } catch (MemoryException e) { throw new RuntimeException(e); } totalLength = 0; }
String compressorName = meta.getCompressorName(); TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName),
private static ColumnPage getLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
private static ColumnPage getComplexLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }