private LazyColumnPage(ColumnPage columnPage, ColumnPageValueConverter converter) { super(columnPage.getColumnPageEncoderMeta(), columnPage.getPageSize()); this.columnPage = columnPage; this.converter = converter; }
/** * Pick corresponding column pages and add to all registered datamap * * @param pageId sequence number of page, start from 0 * @param tablePage page data */ public void onPageAdded(int blockletId, int pageId, TablePage tablePage) throws IOException { Set<Map.Entry<List<CarbonColumn>, List<DataMapWriter>>> entries = registry.entrySet(); for (Map.Entry<List<CarbonColumn>, List<DataMapWriter>> entry : entries) { List<CarbonColumn> indexedColumns = entry.getKey(); ColumnPage[] pages = new ColumnPage[indexedColumns.size()]; for (int i = 0; i < indexedColumns.size(); i++) { pages[i] = tablePage.getColumnPage(indexedColumns.get(i).getColName()); } List<DataMapWriter> writers = entry.getValue(); int pageSize = pages[0].getPageSize(); for (DataMapWriter writer : writers) { writer.onPageAdded(blockletId, pageId, pageSize, pages); } } }
private void fillBasicFields(ColumnPage inputPage, DataChunk2 dataChunk) throws IOException { dataChunk.setChunk_meta(CarbonMetadataUtil.getChunkCompressorMeta(inputPage, dataChunk.getData_page_length())); dataChunk.setNumberOfRowsInpage(inputPage.getPageSize()); dataChunk.setRowMajor(false); }
@Override protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) { return new RLEEncoderMeta(inputPage.getColumnSpec(), inputPage.getDataType(), inputPage.getPageSize(), inputPage.getStatistics(), inputPage.getColumnCompressorName()); }
/** * Get the new column page based on the sorted data * * @param input * @return * @throws MemoryException */ public ColumnPage getSortedColumnPageIfRequired(ColumnPage input) throws MemoryException { if (null != indexStorage) { Object[] dataPage = indexStorage.getDataPage(); ColumnPageEncoderMeta columnPageEncoderMeta = new ColumnPageEncoderMeta(input.getColumnSpec(), input.getDataType(), input.getColumnPageEncoderMeta().getCompressorName()); ColumnPage columnPage = ColumnPage.newPage(columnPageEncoderMeta, input.getPageSize()); putDataToPage(columnPage, dataPage); return columnPage; } else { return input; } }
/** * Create a new column page with input data type and page size. */ protected LocalDictColumnPage(ColumnPage actualDataColumnPage, ColumnPage encodedColumnpage, LocalDictionaryGenerator localDictionaryGenerator, boolean isComplexTypePrimitive, boolean isDecoderBasedFallBackEnabled) { super(actualDataColumnPage.getColumnPageEncoderMeta(), actualDataColumnPage.getPageSize()); // if threshold is not reached then create page level dictionary // for encoding with local dictionary if (!localDictionaryGenerator.isThresholdReached()) { pageLevelDictionary = new PageLevelDictionary(localDictionaryGenerator, actualDataColumnPage.getColumnSpec().getFieldName(), actualDataColumnPage.getDataType(), isComplexTypePrimitive, actualDataColumnPage.getColumnCompressorName()); this.encodedDataColumnPage = encodedColumnpage; this.keyGenerator = KeyGeneratorFactory .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX + 1 }); this.dummyKey = new int[1]; } else { // else free the encoded column page memory as its of no use encodedColumnpage.freeMemory(); } this.isDecoderBasedFallBackEnabled = isDecoderBasedFallBackEnabled; this.actualDataColumnPage = actualDataColumnPage; }
public byte[] encodeAndCompressPage(ColumnPage input, ColumnPageValueConverter converter, Compressor compressor) throws MemoryException, IOException { encodedPage = ColumnPage.newPage( new ColumnPageEncoderMeta(input.getColumnPageEncoderMeta().getColumnSpec(), targetDataType, input.getColumnPageEncoderMeta().getCompressorName()), input.getPageSize()); if (isInvertedIndex) { indexStorage = new BlockIndexerStorageForNoDictionary(getPageBasedOnDataType(input), input.getDataType(), isInvertedIndex); } ColumnPage columnPage = getSortedColumnPageIfRequired(input); columnPage.convertValue(converter); byte[] result = encodedPage.compress(compressor); return result; }
@Override public FallbackEncodedColumnPage call() throws Exception { int pageSize = encodedColumnPage.getActualPage().getPageSize(); int offset = 0; int[] reverseInvertedIndex = new int[pageSize]; encodedColumnPage.getActualPage().getPageSize());