private void newBuffer() { this.buffer = allocator.allocate(PageFormat.PAGE_HEADER_SIZE + fixedRecordSize); this.bufferSlice = Slices.wrappedBuffer(buffer.array(), buffer.offset(), buffer.capacity()); this.count = 0; this.position = PageFormat.PAGE_HEADER_SIZE; this.stringReferences = new ArrayList<>(); this.valueReferences = new ArrayList<>(); this.referenceSize = 0; }
public void addRecord() { // record row.write(this); // record header bufferSlice.setInt(position, nextVariableLengthDataOffset); // nextVariableLengthDataOffset means record size bufferSlice.setBytes(position + 4, nullBitSet); count++; this.position += nextVariableLengthDataOffset; this.nextVariableLengthDataOffset = fixedRecordSize; Arrays.fill(nullBitSet, (byte) -1); // flush if next record will not fit in this buffer if (buffer.capacity() < position + nextVariableLengthDataOffset + referenceSize) { flush(); } }
@Override public void write(int b) { buffer.array()[buffer.offset() + pos] = (byte) b; pos++; if (pos >= buffer.capacity()) { flush(); } }
public Buffer poll() { if (current == null || current.getInputStream() == null) { throw new IllegalStateException("nextFile() must be called before poll()"); } Buffer buffer = allocator.allocate(); try { int n = current.getInputStream().read(buffer.array(), buffer.offset(), buffer.capacity()); if (n < 0) { return null; } buffer.limit(n); Buffer b = buffer; buffer = null; return b; } catch (IOException ex) { throw new RuntimeException(ex); } finally { if (buffer != null) { buffer.release(); buffer = null; } } }
@Override public void write(byte[] b, int off, int len) { while (true) { int available = buffer.capacity() - pos; if (available < len) { buffer.setBytes(pos, b, off, available); pos += available; len -= available; off += available; flush(); } else { buffer.setBytes(pos, b, off, len); pos += len; if (available <= len) { flush(); } break; } } }
public static Buffer readSample(FileInput fileInput, Buffer sample, int offset, int sampleBufferBytes) { if (!fileInput.nextFile()) { // no input files return sample; } try { for (Buffer buffer : each(fileInput)) { int size = Math.min(buffer.limit(), sample.capacity() - offset); sample.setBytes(offset, buffer, 0, size); offset += size; buffer.release(); if (offset >= sampleBufferBytes) { break; } } } finally { sample.limit(offset); } return sample; } }