/** * @return actual memory used */ public long allocatedSize() { return repetitionLevelColumn.getAllocatedSize() + definitionLevelColumn.getAllocatedSize() + dataColumn.getAllocatedSize() + pageWriter.allocatedSize(); }
@Override public long getBufferedSizeInMemory() { return repetitionLevelColumn.getBufferedSize() + definitionLevelColumn.getBufferedSize() + dataColumn.getBufferedSize() + pageWriter.getMemSize(); }
public String memUsageString(String indent) { StringBuilder b = new StringBuilder(indent).append(path).append(" {\n"); b.append(repetitionLevelColumn.memUsageString(indent + " r:")).append("\n"); b.append(definitionLevelColumn.memUsageString(indent + " d:")).append("\n"); b.append(dataColumn.memUsageString(indent + " data:")).append("\n"); b.append(pageWriter.memUsageString(indent + " pages:")).append("\n"); b.append(indent).append(String.format(" total: %,d/%,d", getBufferedSizeInMemory(), allocatedSize())).append("\n"); b.append(indent).append("}\n"); return b.toString(); } }
/** * Finalizes the Column chunk. Possibly adding extra pages if needed (dictionary, ...) * Is called right after writePage */ public void finalizeColumnChunk() { final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose(); if (dictionaryPage != null) { if (DEBUG) LOG.debug("write dictionary"); try { pageWriter.writeDictionaryPage(dictionaryPage); } catch (IOException e) { throw new ParquetEncodingException("could not write dictionary page for " + path, e); } dataColumn.resetDictionary(); } }
@Override void writePage(int rowCount, int valueCount, Statistics<?> statistics, ValuesWriter repetitionLevels, ValuesWriter definitionLevels, ValuesWriter values) throws IOException { pageWriter.writePage( concat(repetitionLevels.getBytes(), definitionLevels.getBytes(), values.getBytes()), valueCount, rowCount, statistics, repetitionLevels.getEncoding(), definitionLevels.getEncoding(), values.getEncoding()); } }
@Override void writePage(int rowCount, int valueCount, Statistics<?> statistics, ValuesWriter repetitionLevels, ValuesWriter definitionLevels, ValuesWriter values) throws IOException { // TODO: rework this API. The bytes shall be retrieved before the encoding (encoding might be different otherwise) BytesInput bytes = values.getBytes(); Encoding encoding = values.getEncoding(); pageWriter.writePageV2( rowCount, Ints.checkedCast(statistics.getNumNulls()), valueCount, repetitionLevels.getBytes(), definitionLevels.getBytes(), encoding, bytes, statistics); } }
/** * Finalizes the Column chunk. Possibly adding extra pages if needed (dictionary, ...) * Is called right after writePage */ void finalizeColumnChunk() { final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose(); if (dictionaryPage != null) { if (DEBUG) LOG.debug("write dictionary"); try { pageWriter.writeDictionaryPage(dictionaryPage); } catch (IOException e) { throw new ParquetEncodingException("could not write dictionary page for " + path, e); } dataColumn.resetDictionary(); } }
private void writePage() { if (DEBUG) LOG.debug("write page"); try { pageWriter.writePage( concat(repetitionLevelColumn.getBytes(), definitionLevelColumn.getBytes(), dataColumn.getBytes()), valueCount, statistics, repetitionLevelColumn.getEncoding(), definitionLevelColumn.getEncoding(), dataColumn.getEncoding()); } catch (IOException e) { throw new ParquetEncodingException("could not write page for " + path, e); } repetitionLevelColumn.reset(); definitionLevelColumn.reset(); dataColumn.reset(); valueCount = 0; resetStatistics(); }
pageWriter.writePageV2( pageRowCount, Ints.checkedCast(statistics.getNumNulls()),
/** * Used to decide when to write a page or row group * * @return the number of bytes of memory used to buffer the current data and the previously written pages */ long getTotalBufferedSize() { return repetitionLevelColumn.getBufferedSize() + definitionLevelColumn.getBufferedSize() + dataColumn.getBufferedSize() + pageWriter.getMemSize(); }
public long allocatedSize() { return repetitionLevelColumn.getAllocatedSize() + definitionLevelColumn.getAllocatedSize() + dataColumn.getAllocatedSize() + pageWriter.allocatedSize(); }
/** * @param indent a prefix to format lines * @return a formatted string showing how memory is used */ public String memUsageString(String indent) { StringBuilder b = new StringBuilder(indent).append(path).append(" {\n"); b.append(indent).append(" r:").append(repetitionLevelColumn.getAllocatedSize()).append(" bytes\n"); b.append(indent).append(" d:").append(definitionLevelColumn.getAllocatedSize()).append(" bytes\n"); b.append(dataColumn.memUsageString(indent + " data:")).append("\n"); b.append(pageWriter.memUsageString(indent + " pages:")).append("\n"); b.append(indent).append(String.format(" total: %,d/%,d", getTotalBufferedSize(), allocatedSize())).append("\n"); b.append(indent).append("}\n"); return b.toString(); }
public void flush() { if (valueCount > 0) { writePage(); } final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose(); if (dictionaryPage != null) { if (DEBUG) LOG.debug("write dictionary"); try { pageWriter.writeDictionaryPage(dictionaryPage); } catch (IOException e) { throw new ParquetEncodingException("could not write dictionary page for " + path, e); } dataColumn.resetDictionary(); } }
@Override public long getBufferedSizeInMemory() { return repetitionLevelColumn.getBufferedSize() + definitionLevelColumn.getBufferedSize() + dataColumn.getBufferedSize() + pageWriter.getMemSize(); }
/** * @return actual memory used */ long allocatedSize() { return repetitionLevelColumn.getAllocatedSize() + definitionLevelColumn.getAllocatedSize() + dataColumn.getAllocatedSize() + pageWriter.allocatedSize(); }
/** * @param indent * a prefix to format lines * @return a formatted string showing how memory is used */ String memUsageString(String indent) { StringBuilder b = new StringBuilder(indent).append(path).append(" {\n"); b.append(indent).append(" r:").append(repetitionLevelColumn.getAllocatedSize()).append(" bytes\n"); b.append(indent).append(" d:").append(definitionLevelColumn.getAllocatedSize()).append(" bytes\n"); b.append(dataColumn.memUsageString(indent + " data:")).append("\n"); b.append(pageWriter.memUsageString(indent + " pages:")).append("\n"); b.append(indent).append(String.format(" total: %,d/%,d", getTotalBufferedSize(), allocatedSize())).append("\n"); b.append(indent).append("}\n"); return b.toString(); }
/** * used to decide when to write a page or row group * @return the number of bytes of memory used to buffer the current data and the previously written pages */ public long getTotalBufferedSize() { return repetitionLevelColumn.getBufferedSize() + definitionLevelColumn.getBufferedSize() + dataColumn.getBufferedSize() + pageWriter.getMemSize(); }
@Override public long getBufferedSizeInMemory() { return repetitionLevelColumn.getBufferedSize() + definitionLevelColumn.getBufferedSize() + dataColumn.getBufferedSize() + pageWriter.getMemSize(); }