public static ColumnPage newDecimalPage(ColumnPageEncoderMeta columnPageEncoderMeta, int pageSize) throws MemoryException { return newPage(columnPageEncoderMeta, pageSize); }
@Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { DataType dataType = columnSpec.getSchemaDataType(); DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length)); ColumnPage resultPage = ColumnPage.newPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), pageSize); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { decodeBytePage(in, resultPage); } else if (dataType == DataTypes.SHORT) { decodeShortPage(in, resultPage); } else if (dataType == DataTypes.INT) { decodeIntPage(in, resultPage); } else if (dataType == DataTypes.LONG) { decodeLongPage(in, resultPage); } else { throw new RuntimeException("unsupported datatype:" + dataType); } return resultPage; }
VarLengthColumnPageBase(ColumnPageEncoderMeta columnPageEncoderMeta, int pageSize) { super(columnPageEncoderMeta, pageSize); TableSpec.ColumnSpec spec = TableSpec.ColumnSpec.newInstance( columnPageEncoderMeta.getColumnSpec().getFieldName(), DataTypes.INT, ColumnType.MEASURE); try { rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, columnPageEncoderMeta.getCompressorName()), pageSize); } catch (MemoryException e) { throw new RuntimeException(e); } totalLength = 0; }
this.columnPages[i] = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, dataType, columnCompressor), pageSize); this.columnPages[i].setStatsCollector(PrimitivePageStatsCollector.newInstance(dataType)); } else { this.columnPages[i] = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize); this.columnPages[i].setStatsCollector(new DummyStatsCollector());
ColumnPage dictionaryColumnPage = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), usedDictionaryValues.cardinality());
private static ColumnPage getComplexLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
private static ColumnPage getLVBytesColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, String compressorName) throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset); rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; counter++; } rowOffset.putInt(counter, offset); return getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, offset, compressorName); }
/** * Get the new column page based on the sorted data * * @param input * @return * @throws MemoryException */ public ColumnPage getSortedColumnPageIfRequired(ColumnPage input) throws MemoryException { if (null != indexStorage) { Object[] dataPage = indexStorage.getDataPage(); ColumnPageEncoderMeta columnPageEncoderMeta = new ColumnPageEncoderMeta(input.getColumnSpec(), input.getDataType(), input.getColumnPageEncoderMeta().getCompressorName()); ColumnPage columnPage = ColumnPage.newPage(columnPageEncoderMeta, input.getPageSize()); putDataToPage(columnPage, dataPage); return columnPage; } else { return input; } }
public byte[] encodeAndCompressPage(ColumnPage input, ColumnPageValueConverter converter, Compressor compressor) throws MemoryException, IOException { encodedPage = ColumnPage.newPage( new ColumnPageEncoderMeta(input.getColumnPageEncoderMeta().getColumnSpec(), targetDataType, input.getColumnPageEncoderMeta().getCompressorName()), input.getPageSize()); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForNoDictionary(getPageBasedOnDataType(input), input.getDataType(), isInvertedIndex); } ColumnPage columnPage = getSortedColumnPageIfRequired(input); columnPage.convertValue(converter); byte[] result = encodedPage.compress(compressor); return result; }
if (ColumnType.GLOBAL_DICTIONARY == columnType || ColumnType.DIRECT_DICTIONARY == columnType) { page = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize); page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY)); page = ColumnPage.newDecimalPage(columnPageEncoderMeta, pageSize); } else { page = ColumnPage.newPage(columnPageEncoderMeta, pageSize); page = ColumnPage.newPage(columnPageEncoderMeta, pageSize); page = ColumnPage.newDecimalPage(columnPageEncoderMeta, pageSize); } else { page = ColumnPage.newPage(columnPageEncoderMeta, pageSize);
TableSpec.ColumnSpec spec = TableSpec.ColumnSpec .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); ColumnPage rowOffset = ColumnPage.newPage( new ColumnPageEncoderMeta(spec, DataTypes.INT, compressorName), CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
ColumnPage.newPage(encodedColumnPage.getActualPage().getColumnPageEncoderMeta(), encodedColumnPage.getActualPage().getPageSize());