@SuppressWarnings("unused") @Override public void writeAllTo(OutputStream out) throws IOException { for (BytesInput input : inputs) { LOG.debug("write {} bytes to out", input.size()); if (input instanceof SequenceBytesIn) LOG.debug("{"); input.writeAllTo(out); if (input instanceof SequenceBytesIn) LOG.debug("}"); } }
@SuppressWarnings("unused") @Override public void writeAllTo(OutputStream out) throws IOException { for (BytesInput input : inputs) { LOG.debug("write {} bytes to out", input.size()); if (input instanceof SequenceBytesIn) LOG.debug("{"); input.writeAllTo(out); if (input instanceof SequenceBytesIn) LOG.debug("}"); } }
/** * * @return a new byte array materializing the contents of this input * @throws IOException if there is an exception reading */ public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); LOG.debug("converted {} to byteArray of {} bytes", size() , baos.size()); return baos.getBuf(); }
/** * * @return a new byte array materializing the contents of this input * @throws IOException if there is an exception reading */ public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); LOG.debug("converted {} to byteArray of {} bytes", size() , baos.size()); return baos.getBuf(); }
/** * writes a number of pages at once * @param bytes bytes to be written including page headers * @param uncompressedTotalPageSize total uncompressed size (without page headers) * @param compressedTotalPageSize total compressed size (without page headers) * @throws IOException if there is an error while writing */ void writeDataPages(BytesInput bytes, long uncompressedTotalPageSize, long compressedTotalPageSize, Statistics totalStats, Set<Encoding> rlEncodings, Set<Encoding> dlEncodings, List<Encoding> dataEncodings) throws IOException { state = state.write(); LOG.debug("{}: write data pages", out.getPos()); long headersSize = bytes.size() - compressedTotalPageSize; this.uncompressedLength += uncompressedTotalPageSize + headersSize; this.compressedLength += compressedTotalPageSize + headersSize; LOG.debug("{}: write data pages content", out.getPos()); bytes.writeAllTo(out); encodingStatsBuilder.addDataEncodings(dataEncodings); if (rlEncodings.isEmpty()) { encodingStatsBuilder.withV2Pages(); } currentEncodings.addAll(rlEncodings); currentEncodings.addAll(dlEncodings); currentEncodings.addAll(dataEncodings); currentStatistics = totalStats; }
@Override public BytesInput compress(BytesInput bytes) throws IOException { final BytesInput compressedBytes; if (codec == null) { compressedBytes = bytes; } else { compressedOutBuffer.reset(); if (compressor != null) { // null compressor for non-native gzip compressor.reset(); } CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor); bytes.writeAllTo(cos); cos.finish(); cos.close(); compressedBytes = BytesInput.from(compressedOutBuffer); } return compressedBytes; }
this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write data page content {}", out.getPos(), compressedPageSize); bytes.writeAllTo(out); encodingStatsBuilder.addDataEncoding(valuesEncoding); currentEncodings.add(rlEncoding);
@Override public BytesInput compress(BytesInput bytes) throws IOException { final BytesInput compressedBytes; if (codec == null) { compressedBytes = bytes; } else { compressedOutBuffer.reset(); if (compressor != null) { // null compressor for non-native gzip compressor.reset(); } CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor); bytes.writeAllTo(cos); cos.finish(); cos.close(); compressedBytes = BytesInput.from(compressedOutBuffer); } return compressedBytes; }
this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write data page content {}", out.getPos(), compressedPageSize); bytes.writeAllTo(out); encodingStatsBuilder.addDataEncoding(valuesEncoding); currentEncodings.add(rlEncoding);
this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write data page content {}", out.getPos(), compressedPageSize); bytes.writeAllTo(out);
this.pageCount += 1; this.totalStatistics.mergeStatistics(statistics); compressedBytes.writeAllTo(buf); rlEncodings.add(rlEncoding); dlEncodings.add(dlEncoding);
this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write data page content {}", out.getPos(), compressedPageSize); bytes.writeAllTo(out);
LOG.debug("{}: write data pages content", out.getPos()); firstPageOffset = out.getPos(); bytes.writeAllTo(out); encodingStatsBuilder.addDataEncodings(dataEncodings); if (rlEncodings.isEmpty()) {
/** * writes a dictionary page page * @param dictionaryPage the dictionary page * @throws IOException if there is an error while writing */ public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException { state = state.write(); LOG.debug("{}: write dictionary page: {} values", out.getPos(), dictionaryPage.getDictionarySize()); currentChunkDictionaryPageOffset = out.getPos(); int uncompressedSize = dictionaryPage.getUncompressedSize(); int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts metadataConverter.writeDictionaryPageHeader( uncompressedSize, compressedPageSize, dictionaryPage.getDictionarySize(), dictionaryPage.getEncoding(), out); long headerSize = out.getPos() - currentChunkDictionaryPageOffset; this.uncompressedLength += uncompressedSize + headerSize; this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write dictionary page content {}", out.getPos(), compressedPageSize); dictionaryPage.getBytes().writeAllTo(out); encodingStatsBuilder.addDictEncoding(dictionaryPage.getEncoding()); currentEncodings.add(dictionaryPage.getEncoding()); }
/** * writes a dictionary page page * @param dictionaryPage the dictionary page * @throws IOException if there is an error while writing */ public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException { state = state.write(); LOG.debug("{}: write dictionary page: {} values", out.getPos(), dictionaryPage.getDictionarySize()); currentChunkDictionaryPageOffset = out.getPos(); int uncompressedSize = dictionaryPage.getUncompressedSize(); int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts metadataConverter.writeDictionaryPageHeader( uncompressedSize, compressedPageSize, dictionaryPage.getDictionarySize(), dictionaryPage.getEncoding(), out); long headerSize = out.getPos() - currentChunkDictionaryPageOffset; this.uncompressedLength += uncompressedSize + headerSize; this.compressedLength += compressedPageSize + headerSize; LOG.debug("{}: write dictionary page content {}", out.getPos(), compressedPageSize); dictionaryPage.getBytes().writeAllTo(out); encodingStatsBuilder.addDictEncoding(dictionaryPage.getEncoding()); currentEncodings.add(dictionaryPage.getEncoding()); }
this.totalStatistics.mergeStatistics(statistics); definitionLevels.writeAllTo(buf); compressedData.writeAllTo(buf);