public static Slice unscaledDecimal(Slice decimal) { return Slices.copyOf(decimal); }
public List<Slice> getSlices() { return ImmutableList.<Slice>builder() .addAll(closedSlices) .add(Slices.copyOf(slice, 0, bufferPosition)) .build(); }
public List<Slice> getSlices() { return ImmutableList.<Slice>builder() .addAll(closedSlices) .add(Slices.copyOf(slice, 0, bufferPosition)) .build(); }
public static Slice multiply(Slice decimal, int multiplier) { Slice result = Slices.copyOf(decimal); multiplyDestructive(result, multiplier); return result; }
static Slice shiftLeft(Slice decimal, int leftShifts) { Slice result = Slices.copyOf(decimal); shiftLeftDestructive(result, leftShifts); return result; }
@VisibleForTesting public static Slice minStringTruncateToValidRange(Slice value, HiveWriterVersion version) { if (value == null) { return null; } if (version != ORIGINAL) { return value; } int index = findStringStatisticTruncationPositionForOriginalOrcWriter(value); if (index == value.length()) { return value; } return Slices.copyOf(value, 0, index); }
/** * Returns a slice containing values in the specified range of the specified slice. * If the range matches the entire slice, the input slice will be returned. * Otherwise, a copy will be returned. */ static Slice compactSlice(Slice slice, int index, int length) { if (slice.isCompact() && index == 0 && length == slice.length()) { return slice; } return Slices.copyOf(slice, index, length); }
private Slice dropStringMinMaxIfNecessary(Slice minOrMax) { if (minOrMax == null || minOrMax.length() > stringStatisticsLimitInBytes) { return null; } // Do not hold the entire slice where the actual stats could be small if (minOrMax.isCompact()) { return minOrMax; } return Slices.copyOf(minOrMax); } }
@Override public Block copyRegion(int positionOffset, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, positionOffset, length); Slice newSlice = Slices.copyOf(sliceOutput.getUnderlyingSlice(), positionOffset * fixedSize, length * fixedSize); Slice newValueIsNull = null; if (hasNullValue) { newValueIsNull = Slices.copyOf(valueIsNull.getUnderlyingSlice(), positionOffset, length); } return new FixedWidthBlock(fixedSize, length, newSlice, newValueIsNull); }
protected static Slice[] createExpectedUniqueValues(int positionCount) { Slice[] expectedValues = new Slice[positionCount]; for (int position = 0; position < positionCount; position++) { expectedValues[position] = Slices.copyOf(createExpectedValue(position)); } return expectedValues; }
@VisibleForTesting public static Slice maxStringTruncateToValidRange(Slice value, HiveWriterVersion version) { if (value == null) { return null; } if (version != ORIGINAL) { return value; } int index = findStringStatisticTruncationPositionForOriginalOrcWriter(value); if (index == value.length()) { return value; } // Append 0xFF so that it is larger than value Slice newValue = Slices.copyOf(value, 0, index + 1); newValue.setByte(index, 0xFF); return newValue; }
@Setup public void setup() { Slice whitespace = createRandomUtf8Slice(ascii ? ASCII_WHITESPACE : ALL_WHITESPACE, length + 1); leftWhitespace = Slices.copyOf(whitespace); leftWhitespace.setByte(leftWhitespace.length() - 1, 'X'); rightWhitespace = Slices.copyOf(whitespace); rightWhitespace.setByte(0, 'X'); bothWhitespace = Slices.copyOf(whitespace); bothWhitespace.setByte(length / 2, 'X'); }
@Override public Block getSingleValueBlock(int position) { checkReadablePosition(position); Slice copy = Slices.copyOf(getRawSlice(), valueOffset(position), fixedSize); Slice valueIsNull = null; if (isNull(position)) { valueIsNull = Slices.wrappedBooleanArray(true); } return new FixedWidthBlock(fixedSize, 1, copy, valueIsNull); }
@Override public Block getSingleValueBlock(int position) { if (isNull(position)) { return new VariableWidthBlock(0, 1, EMPTY_SLICE, new int[] {0, 0}, new boolean[] {true}); } int offset = getPositionOffset(position); int entrySize = getSliceLength(position); Slice copy = Slices.copyOf(getRawSlice(position), offset, entrySize); return new VariableWidthBlock(0, 1, copy, new int[] {0, copy.length()}, null); }
@Test public void testCompactBlock() { Slice compactSlice = Slices.copyOf(createExpectedValue(16)); Slice incompactSlice = Slices.copyOf(createExpectedValue(20)).slice(0, 16); int[] offsets = {0, 1, 1, 2, 4, 8, 16}; boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(new VariableWidthBlock(0, EMPTY_SLICE, new int[1], Optional.empty())); testCompactBlock(new VariableWidthBlock(valueIsNull.length, compactSlice, offsets, Optional.of(valueIsNull))); testIncompactBlock(new VariableWidthBlock(valueIsNull.length - 1, compactSlice, offsets, Optional.of(valueIsNull))); // underlying slice is not compact testIncompactBlock(new VariableWidthBlock(valueIsNull.length, incompactSlice, offsets, Optional.of(valueIsNull))); }
@Override public void deserialize(Block block, int index, LongDecimalWithOverflowState state) { if (!block.isNull(index)) { SliceInput slice = VARBINARY.getSlice(block, index).getInput(); state.setOverflow(slice.readLong()); state.setLongDecimal(Slices.copyOf(slice.readSlice(slice.available()))); } } }
private static Slice writeVintOld(SliceOutput output, long value) throws IOException { output.reset(); WritableUtils.writeVLong(output, value); Slice vLongOld = Slices.copyOf(output.slice()); output.reset(); RcFileDecoderUtils.writeVLong(output, value); Slice vLongNew = Slices.copyOf(output.slice()); assertEquals(vLongNew, vLongOld); if (value == (int) value) { output.reset(); WritableUtils.writeVInt(output, (int) value); Slice vIntOld = Slices.copyOf(output.slice()); assertEquals(vIntOld, vLongOld); output.reset(); RcFileDecoderUtils.writeVInt(output, (int) value); Slice vIntNew = Slices.copyOf(output.slice()); assertEquals(vIntNew, vLongOld); } return vLongOld; } }
@Test public void testCompactBlock() { Slice compactSlice = Slices.copyOf(createExpectedValue(24)); Slice incompactSlice = Slices.copyOf(createExpectedValue(30)).slice(0, 24); boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(new FixedWidthBlock(4, 0, EMPTY_SLICE, Optional.empty())); testCompactBlock(new FixedWidthBlock(4, valueIsNull.length, compactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); testIncompactBlock(new FixedWidthBlock(4, valueIsNull.length - 1, compactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); // underlying slice is not compact testIncompactBlock(new FixedWidthBlock(4, valueIsNull.length, incompactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); }
@Override public void deserialize(Block block, int index, LongDecimalWithOverflowAndLongState state) { if (!block.isNull(index)) { SliceInput slice = VARBINARY.getSlice(block, index).getInput(); state.setLong(slice.readLong()); state.setOverflow(slice.readLong()); state.setLongDecimal(Slices.copyOf(slice.readSlice(slice.available()))); } } }
public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact((page.getSizeInBytes() + Integer.BYTES))); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); if (!compressor.isPresent()) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } int maxCompressedLength = maxCompressedLength(serializationBuffer.size()); byte[] compressionBuffer = new byte[maxCompressedLength]; int actualCompressedLength = compressor.get().compress(serializationBuffer.slice().getBytes(), 0, serializationBuffer.size(), compressionBuffer, 0, maxCompressedLength); if (((1.0 * actualCompressedLength) / serializationBuffer.size()) > MINIMUM_COMPRESSION_RATIO) { return new SerializedPage(serializationBuffer.slice(), UNCOMPRESSED, page.getPositionCount(), serializationBuffer.size()); } return new SerializedPage( Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, actualCompressedLength)), COMPRESSED, page.getPositionCount(), serializationBuffer.size()); }