void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
private void startBlock(int block) throws IOException { this.block = block; this.row = column.firstRows[block]; in.seek(column.blockStarts[block]); int end = column.blocks[block].compressedSize; byte[] raw = new byte[end+checksum.size()]; in.readFully(raw); ByteBuffer data = codec.decompress(ByteBuffer.wrap(raw, 0, end)); if (!checksum.compute(data).equals (ByteBuffer.wrap(raw, end, checksum.size()))) throw new IOException("Checksums mismatch."); values = new InputBuffer(new InputBytes(data)); }
public ColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { this.writer = writer; this.meta = meta; this.codec = Codec.get(meta); this.checksum = Checksum.get(meta); this.buffer = new OutputBuffer(); this.blockDescriptors = new ArrayList<>(); this.blockData = new ArrayList<>(); if (meta.hasIndexValues()) this.firstValues = new ArrayList<>(); }
Codec codec = Codec.get(meta); ByteBuffer compressedBuffer = codec.compress(inputByteBuffer); ByteBuffer decompressedBuffer = codec.decompress(compressedBuffer);
ColumnValues(ColumnDescriptor column) throws IOException { this.column = column; this.type = column.metaData.getType(); this.codec = Codec.get(column.metaData); this.checksum = Checksum.get(column.metaData); this.in = new InputBuffer(column.file); column.ensureBlocksRead(); }
private void startBlock(int block) throws IOException { this.block = block; this.row = column.firstRows[block]; in.seek(column.blockStarts[block]); int end = column.blocks[block].compressedSize; byte[] raw = new byte[end+checksum.size()]; in.readFully(raw); ByteBuffer data = codec.decompress(ByteBuffer.wrap(raw, 0, end)); if (!checksum.compute(data).equals (ByteBuffer.wrap(raw, end, checksum.size()))) throw new IOException("Checksums mismatch."); values = new InputBuffer(new InputBytes(data)); }
private void flushBuffer() throws IOException { if (rowCount == 0) return; ByteBuffer raw = buffer.asByteBuffer(); ByteBuffer c = codec.compress(raw); blockDescriptors.add(new BlockDescriptor(rowCount, raw.remaining(), c.remaining())); ByteBuffer data = ByteBuffer.allocate(c.remaining() + checksum.size()); data.put(c); data.put(checksum.compute(raw)); blockData.add(data.array()); int sizeIncrement = (4*3) // descriptor + (firstValues != null // firstValue ? firstValues.get(firstValues.size()-1).length : 0) + data.position(); // data writer.incrementSize(sizeIncrement); size += sizeIncrement; buffer = new OutputBuffer(); rowCount = 0; }
public ColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { this.writer = writer; this.meta = meta; this.codec = Codec.get(meta); this.checksum = Checksum.get(meta); this.buffer = new OutputBuffer(); this.blockDescriptors = new ArrayList<BlockDescriptor>(); this.blockData = new ArrayList<byte[]>(); if (meta.hasIndexValues()) this.firstValues = new ArrayList<byte[]>(); }
ColumnValues(ColumnDescriptor column) throws IOException { this.column = column; this.type = column.metaData.getType(); this.codec = Codec.get(column.metaData); this.checksum = Checksum.get(column.metaData); this.in = new InputBuffer(column.file); column.ensureBlocksRead(); }