/** * free memory as needed */ public void freeMemory() { if (null != rowOffset) { rowOffset.freeMemory(); rowOffset = null; } }
/** * to free the used memory */ public void freeMemory() { for (int i = 0; i < complexColumnIndex; i++) { columnPages[i].freeMemory(); } }
public void freeMemoryForce() { if (!isActualPageMemoryFreed) { actualDataColumnPage.freeMemory(); isActualPageMemoryFreed = true; } }
@Override public void freeMemory() { if (null != columnPage) { columnPage.freeMemory(); columnPage = null; } }
private void freeEncodedColumnPage() { if (null != encodedDataColumnPage) { encodedDataColumnPage.freeMemory(); encodedDataColumnPage = null; } }
@Override public void freeMemory() { columnPage.freeMemory(); }
@Override public void freeMemory() { // free the encoded column page as data is already encoded and it is of no use, during fallback // if goes to actual databased fallback, we need actual data and decoder based fallback we need // just the encoded data to form a new page if (null != encodedDataColumnPage) { encodedDataColumnPage.freeMemory(); } if (isDecoderBasedFallBackEnabled) { actualDataColumnPage.freeMemory(); isActualPageMemoryFreed = true; } else if (null == pageLevelDictionary) { actualDataColumnPage.freeMemory(); isActualPageMemoryFreed = true; } }
void freeMemory() { for (ColumnPage page : dictDimensionPages) { page.freeMemory(); } for (ColumnPage page : noDictDimensionPages) { page.freeMemory(); } for (ColumnPage page : measurePages) { page.freeMemory(); } for (ComplexColumnPage page : complexDimensionPages) { if (null != page) { page.freeMemory(); } } }
@Override public void freeMemory() { super.freeMemory(); if (null != columnPages) { for (int i = 0; i < columnPages.length; i++) { if (columnPages[i] != null) { columnPages[i].freeMemory(); columnPages[i] = null; } } } rawData = null; }
private void freeDataChunkMemory() { for (int i = 0; i < dimensionColumnPages.length; i++) { if (pageCounter > 0 && dimensionColumnPages[i][pageCounter - 1] != null) { dimensionColumnPages[i][pageCounter - 1].freeMemory(); dimensionColumnPages[i][pageCounter - 1] = null; } } for (int i = 0; i < measureColumnPages.length; i++) { if (pageCounter > 0 && measureColumnPages[i][pageCounter - 1] != null) { measureColumnPages[i][pageCounter - 1].freeMemory(); measureColumnPages[i][pageCounter - 1] = null; } } clearValidRowIdList(); }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor( input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
@Override protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException { if (encodedPage != null) { throw new IllegalStateException("already encoded"); } Compressor compressor = CompressorFactory.getInstance().getCompressor(input.getColumnCompressorName()); result = encodeAndCompressPage(input, converter, compressor); byte[] bytes = writeInvertedIndexIfRequired(result); encodedPage.freeMemory(); if (bytes.length != 0) { return bytes; } return result; }
for (int j = 0; j < measureColumnPages[i].length; j++) { if (null != measureColumnPages[i][j]) { measureColumnPages[i][j].freeMemory(); measureColumnPages[i][j] = null;
/** * Below method will be used to add column data to page * * @param rowId row number * @param bytes actual data */ @Override public void putBytes(int rowId, byte[] bytes) { if (null != pageLevelDictionary) { try { actualDataColumnPage.putBytes(rowId, bytes); dummyKey[0] = pageLevelDictionary.getDictionaryValue(bytes); encodedDataColumnPage.putBytes(rowId, keyGenerator.generateKey(dummyKey)); } catch (DictionaryThresholdReachedException e) { LOGGER.warn("Local Dictionary threshold reached for the column: " + actualDataColumnPage .getColumnSpec().getFieldName() + ", " + e.getMessage()); pageLevelDictionary = null; encodedDataColumnPage.freeMemory(); encodedDataColumnPage = null; } catch (KeyGenException e) { LOGGER.error("Unable to generate key for: " + actualDataColumnPage .getColumnSpec().getFieldName(), e); throw new RuntimeException(e); } } else { actualDataColumnPage.putBytes(rowId, bytes); } }
/** * Create a new column page with input data type and page size. */ protected LocalDictColumnPage(ColumnPage actualDataColumnPage, ColumnPage encodedColumnpage, LocalDictionaryGenerator localDictionaryGenerator, boolean isComplexTypePrimitive, boolean isDecoderBasedFallBackEnabled) { super(actualDataColumnPage.getColumnPageEncoderMeta(), actualDataColumnPage.getPageSize()); // if threshold is not reached then create page level dictionary // for encoding with local dictionary if (!localDictionaryGenerator.isThresholdReached()) { pageLevelDictionary = new PageLevelDictionary(localDictionaryGenerator, actualDataColumnPage.getColumnSpec().getFieldName(), actualDataColumnPage.getDataType(), isComplexTypePrimitive, actualDataColumnPage.getColumnCompressorName()); this.encodedDataColumnPage = encodedColumnpage; this.keyGenerator = KeyGeneratorFactory .getKeyGenerator(new int[] { CarbonCommonConstants.LOCAL_DICTIONARY_MAX + 1 }); this.dummyKey = new int[1]; } else { // else free the encoded column page memory as its of no use encodedColumnpage.freeMemory(); } this.isDecoderBasedFallBackEnabled = isDecoderBasedFallBackEnabled; this.actualDataColumnPage = actualDataColumnPage; }
private static VarLengthColumnPageBase getVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, ColumnPage rowOffset, int offset, String compressorName) throws MemoryException { int lvEncodedOffset; int length; int numRows = rowId; VarLengthColumnPageBase page; int inputDataLength = offset; if (unsafe) { page = new UnsafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), numRows, inputDataLength); } else { page = new SafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), numRows); } // set total length and rowOffset in page page.totalLength = offset; page.rowOffset.freeMemory(); page.rowOffset = rowOffset; // set data in page lvEncodedOffset = 0; for (int i = 0; i < numRows; i++) { length = rowOffset.getInt(i + 1) - rowOffset.getInt(i); page.putBytes(i, lvEncodedBytes, lvEncodedOffset + lvLength, length); lvEncodedOffset += lvLength + length; } return page; }
page.rowOffset.freeMemory(); page.rowOffset = rowOffset; for (int i = 0; i < rowId; i++) {