/** * @param size the size of the page * @return the page * @throws IOException */ public BytesInput readAsBytesInput(int size) throws IOException { final BytesInput r = BytesInput.from(this.buf, this.pos, size); this.pos += size; return r; }
@Override public BytesInput getBytes() { serializeCurrentValue(); byte[] buf = bitWriter.finish(); if (Log.DEBUG) LOG.debug("writing a buffer of size " + buf.length + " + 4 bytes"); // We serialize the length so that on deserialization we can // deserialize as we go, instead of having to load everything // into memory return BytesInput.fromSequence(BytesInput.fromInt(buf.length), BytesInput.from(buf)); }
public BytesInput getBytesInput(int size) throws IOException { BytesInput bytesInput = BytesInput.from(buf, pos, size); pos += size; return bytesInput; }
/** * copies the input into a new byte array * @param bytesInput * @return * @throws IOException */ public static BytesInput copy(BytesInput bytesInput) throws IOException { return from(bytesInput.toByteArray()); }
/** * copies the input into a new byte array * @param bytesInput * @return * @throws IOException */ public static BytesInput copy(BytesInput bytesInput) throws IOException { return from(bytesInput.toByteArray()); }
/** * copies the input into a new byte array * @param bytesInput * @return * @throws IOException */ public static BytesInput copy(BytesInput bytesInput) throws IOException { return from(bytesInput.toByteArray()); }
public BytesInput finish() { if (!finished) { if (currentBytePosition > 0) { baos.write(currentByte); if (DEBUG) LOG.debug("to buffer: " + toBinary(currentByte)); } } finished = true; return BytesInput.from(baos); }
/** * {@inheritDoc} * @see parquet.column.values.ValuesWriter#getBytes() */ @Override public BytesInput getBytes() { try { this.bitPackingWriter.finish(); return BytesInput.from(out); } catch (IOException e) { throw new ParquetEncodingException(e); } }
/** * * {@inheritDoc} * @see parquet.column.primitive.PrimitiveColumnWriter#getBytes() */ @Override public BytesInput getBytes() { try { this.bitPackingWriter.finish(); return BytesInput.from(out); } catch (IOException e) { throw new ParquetEncodingException(e); } }
public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException { final BytesInput decompressed; if (codec != null) { decompressor.reset(); InputStream is = codec.createInputStream(new ByteArrayInputStream(bytes.toByteArray()), decompressor); decompressed = BytesInput.from(is, uncompressedSize); } else { decompressed = bytes; } return decompressed; }
public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException { final BytesInput decompressed; if (codec != null) { decompressor.reset(); InputStream is = codec.createInputStream(new ByteArrayInputStream(bytes.toByteArray()), decompressor); decompressed = BytesInput.from(is, uncompressedSize); } else { decompressed = bytes; } return decompressed; }
public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException { BytesInput decompressed; if (codec != null) { decompressor.reset(); InputStream inputStream = codec.createInputStream(new ByteArrayInputStream(bytes.toByteArray()), decompressor); decompressed = BytesInput.from(inputStream, uncompressedSize); } else { decompressed = bytes; } return decompressed; }
@Override public BytesInput getBytes() { try { bitPackingWriter.finish(); } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } if (Log.DEBUG) LOG.debug("writing a buffer of size " + out.size()); return BytesInput.from(out); }
@Override public BytesInput getBytes() { try { out.flush(); } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size()); return BytesInput.from(arrayOut); }
/** * @return the bytes representing the packed values * @throws IOException */ public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes"); if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; } pack(); } return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); }
@Override public BytesInput getBytes() { try { out.flush(); } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size()); return BytesInput.from(arrayOut); }
/** * @return the bytes representing the packed values * @throws IOException */ public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes"); if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; } pack(); } return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); }
@Override public BytesInput getBytes() { try { out.flush(); } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size()); return BytesInput.from(arrayOut); }
public BytesInput readAsBytesInput(int size) throws IOException { if (pos + size > count) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing int l1 = count - pos; int l2 = size - l1; LOG.info("completed the column chunk with " + l2 + " bytes"); return BytesInput.concat(super.readAsBytesInput(l1), BytesInput.copy(BytesInput.from(f, l2))); } return super.readAsBytesInput(size); }
@Override public BytesInput getBytes() { try { out.flush(); } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size()); return BytesInput.concat(lengthWriter.getBytes(), BytesInput.from(arrayOut)); }