public ColumnEncodeOutput(SliceOutput lengthOutput, SliceOutput valueOutput) { this.lengthOutput = lengthOutput; this.valueOutput = valueOutput; this.previousOffset = valueOutput.size(); this.previousLength = -1; }
@Override public long getSizeInBytes() { long arraysSizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positions; return sliceOutput.size() + arraysSizeInBytes; }
@Override public String toString() { StringBuilder sb = new StringBuilder("VariableWidthBlockBuilder{"); sb.append("positionCount=").append(positions); sb.append(", size=").append(sliceOutput.size()); sb.append('}'); return sb.toString(); } }
@Override public String toString() { StringBuilder sb = new StringBuilder("FixedWidthBlockBuilder{"); sb.append("positionCount=").append(positionCount); sb.append(", fixedSize=").append(fixedSize); sb.append(", size=").append(sliceOutput.size()); sb.append('}'); return sb.toString(); } }
public int getBufferedSize() { return lengthOutput.size() + output.size(); }
@Override public void closeEntry() { int offset = valueOutput.size(); int length = offset - previousOffset; previousOffset = offset; if (length == previousLength) { runLength++; } else { if (runLength > 0) { int value = ~runLength; writeVInt(lengthOutput, value); } writeVInt(lengthOutput, length); previousLength = length; runLength = 0; } }
public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }
private void entryAdded(int bytesWritten, boolean isNull) { if (!initialized) { initializeCapacity(); } if (valueIsNull.length <= positions) { growCapacity(); } valueIsNull[positions] = isNull; offsets[positions + 1] = sliceOutput.size(); positions++; if (blockBuilderStatus != null) { blockBuilderStatus.addBytes(SIZE_OF_BYTE + SIZE_OF_INT + bytesWritten); } }
private static Segment writeSegment(SliceOutput output, List<List<Integer>> rowGroups) { int offset = output.size(); // if we are at the beginning of the file write a file header, otherwise write a sync if (offset == 0) { writeFileHeader(output); } else { writeSync(output); } ImmutableList.Builder<Integer> rowGroupOffsets = ImmutableList.builder(); for (List<Integer> rowGroup : rowGroups) { rowGroupOffsets.add(output.size() - offset); writeRowGroup(output, rowGroup); } int length = output.size() - offset; return new Segment( rowGroups.stream() .flatMap(List::stream) .collect(toList()), offset, length, rowGroupOffsets.build()); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int finalLength = 0; for (int i = offset; i < offset + length; i++) { finalLength += getSliceLength(positions[i]); } SliceOutput newSlice = Slices.allocate(finalLength).getOutput(); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = null; if (valueIsNull != null) { newValueIsNull = new boolean[length]; } for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (!isEntryNull(position)) { newSlice.writeBytes(slice, getPositionOffset(position), getSliceLength(position)); } else if (newValueIsNull != null) { newValueIsNull[i] = true; } newOffsets[i + 1] = newSlice.size(); } return new VariableWidthBlock(0, length, newSlice.slice(), newOffsets, newValueIsNull); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int finalLength = 0; for (int i = offset; i < offset + length; i++) { finalLength += getSliceLength(positions[i]); } SliceOutput newSlice = Slices.allocate(finalLength).getOutput(); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = null; if (hasNullValue) { newValueIsNull = new boolean[length]; } for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (isEntryNull(position)) { newValueIsNull[i] = true; } else { newSlice.writeBytes(sliceOutput.getUnderlyingSlice(), getPositionOffset(position), getSliceLength(position)); } newOffsets[i + 1] = newSlice.size(); } return new VariableWidthBlock(0, length, newSlice.slice(), newOffsets, newValueIsNull); }
@SuppressWarnings("AssignmentToForLoopParameter") private static ColumnData unescape(ColumnData columnData, byte escapeByte) { Slice slice = columnData.getSlice(); // does slice contain escape byte if (slice.indexOfByte(escapeByte) < 0) { return columnData; } Slice newSlice = Slices.allocate(slice.length()); SliceOutput output = newSlice.getOutput(); int[] newOffsets = new int[columnData.rowCount() + 1]; for (int row = 0; row < columnData.rowCount(); row++) { int offset = columnData.getOffset(row); int length = columnData.getLength(row); for (int i = 0; i < length; i++) { byte value = slice.getByte(offset + i); if (value == escapeByte && i + 1 < length) { // read byte after escape i++; value = slice.getByte(offset + i); } output.write(value); } newOffsets[row + 1] = output.size(); } return new ColumnData(newOffsets, output.slice()); }
SliceOutput actualSliceOutput = new DynamicSliceOutput(100); writeBlock(blockEncodingSerde, actualSliceOutput, (Block) type.getObject(block, position)); SliceOutput expectedSliceOutput = new DynamicSliceOutput(actualSliceOutput.size()); writeBlock(blockEncodingSerde, expectedSliceOutput, (Block) expectedStackValue); assertEquals(actualSliceOutput.slice(), expectedSliceOutput.slice());
public ColumnEncodeOutput(SliceOutput lengthOutput, SliceOutput valueOutput) { this.lengthOutput = lengthOutput; this.valueOutput = valueOutput; this.previousOffset = valueOutput.size(); this.previousLength = -1; }
@Override public long getSizeInBytes() { long arraysSizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positions; return sliceOutput.size() + arraysSizeInBytes; }
@Override public long getSizeInBytes() { long arraysSizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positions; return sliceOutput.size() + arraysSizeInBytes; }
@Override public String toString() { StringBuilder sb = new StringBuilder("FixedWidthBlockBuilder{"); sb.append("positionCount=").append(positionCount); sb.append(", fixedSize=").append(fixedSize); sb.append(", size=").append(sliceOutput.size()); sb.append('}'); return sb.toString(); } }
@Override public String toString() { StringBuilder sb = new StringBuilder("VariableWidthBlockBuilder{"); sb.append("positionCount=").append(positions); sb.append(", size=").append(sliceOutput.size()); sb.append('}'); return sb.toString(); } }