@Override public Block getRegion(int positionOffset, int length) { checkValidRegion(positionCount, positionOffset, length); Slice newSlice = slice.slice(positionOffset * fixedSize, length * fixedSize); Slice newValueIsNull = null; if (valueIsNull != null) { newValueIsNull = valueIsNull.slice(positionOffset, length); } return new FixedWidthBlock(fixedSize, length, newSlice, newValueIsNull); }
public static Slice truncateToLength(Slice slice, int maxLength) { requireNonNull(slice, "slice is null"); if (maxLength < 0) { throw new IllegalArgumentException("Max length must be greater or equal than zero"); } if (maxLength == 0) { return Slices.EMPTY_SLICE; } return slice.slice(0, byteCount(slice, 0, slice.length(), maxLength)); }
public static Slice trimTrailingSpaces(Slice slice) { requireNonNull(slice, "slice is null"); return slice.slice(0, byteCountWithoutTrailingSpace(slice, 0, slice.length())); }
private void closeChunk() { // add trimmed view of slice to closed slices closedSlices.add(slice.slice(0, bufferPosition)); // create a new buffer // double size until we hit the max chunk size buffer = chunkSupplier.get(); slice = Slices.wrappedBuffer(buffer); streamOffset += bufferPosition; bufferPosition = 0; }
private void closeChunk() { // add trimmed view of slice to closed slices closedSlices.add(slice.slice(0, bufferPosition)); closedSlicesRetainedSize += slice.getRetainedSize(); // create a new buffer // double size until we hit the max chunk size buffer = chunkSupplier.get(); slice = Slices.wrappedBuffer(buffer); streamOffset += bufferPosition; bufferPosition = 0; }
@Override public Block getRegion(int positionOffset, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, positionOffset, length); Slice newSlice = sliceOutput.slice().slice(positionOffset * fixedSize, length * fixedSize); Slice newValueIsNull = null; if (hasNullValue) { newValueIsNull = valueIsNull.slice().slice(positionOffset, length); } return new FixedWidthBlock(fixedSize, length, newSlice, newValueIsNull); }
@Override public Slice getSlice(int position, int offset, int length) { checkReadablePosition(position); return getRawSlice().slice(valueOffset(position) + offset, length); }
@Override public Slice getSlice(int position, int offset, int length) { checkReadablePosition(position); return getRawSlice(position).slice(getPositionOffset(position) + offset, length); }
public static String truncateIfNecessaryForErrorMessage(Slice json) { if (json.length() <= MAX_JSON_LENGTH_IN_ERROR_MESSAGE) { return json.toStringUtf8(); } else { return json.slice(0, MAX_JSON_LENGTH_IN_ERROR_MESSAGE).toStringUtf8() + "...(truncated)"; } }
public Slice loadNestedDiskRangeBuffer(DiskRange nestedDiskRange) { load(); checkArgument(diskRange.contains(nestedDiskRange)); int offset = toIntExact(nestedDiskRange.getOffset() - diskRange.getOffset()); return bufferSlice.slice(offset, nestedDiskRange.getLength()); }
public Block split(Slice source) { Matcher matcher = re2jPattern.matcher(source); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 32); int lastEnd = 0; while (matcher.find()) { Slice slice = source.slice(lastEnd, matcher.start() - lastEnd); lastEnd = matcher.end(); VARCHAR.writeSlice(blockBuilder, slice); } VARCHAR.writeSlice(blockBuilder, source.slice(lastEnd, source.length() - lastEnd)); return blockBuilder.build(); }
@Test public void testCompactBlock() { Slice compactSlice = Slices.copyOf(createExpectedValue(16)); Slice incompactSlice = Slices.copyOf(createExpectedValue(20)).slice(0, 16); int[] offsets = {0, 1, 1, 2, 4, 8, 16}; boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(new VariableWidthBlock(0, EMPTY_SLICE, new int[1], Optional.empty())); testCompactBlock(new VariableWidthBlock(valueIsNull.length, compactSlice, offsets, Optional.of(valueIsNull))); testIncompactBlock(new VariableWidthBlock(valueIsNull.length - 1, compactSlice, offsets, Optional.of(valueIsNull))); // underlying slice is not compact testIncompactBlock(new VariableWidthBlock(valueIsNull.length, incompactSlice, offsets, Optional.of(valueIsNull))); }
@Description("suffix starting at given index") @ScalarFunction @LiteralParameters("x") @SqlType("varchar(x)") public static Slice substr(@SqlType("varchar(x)") Slice utf8, @SqlType(StandardTypes.BIGINT) long start) { if ((start == 0) || utf8.length() == 0) { return Slices.EMPTY_SLICE; } int startCodePoint = Ints.saturatedCast(start); if (startCodePoint > 0) { int indexStart = offsetOfCodePoint(utf8, startCodePoint - 1); if (indexStart < 0) { // before beginning of string return Slices.EMPTY_SLICE; } int indexEnd = utf8.length(); return utf8.slice(indexStart, indexEnd - indexStart); } // negative start is relative to end of string int codePoints = countCodePoints(utf8); startCodePoint += codePoints; // before beginning of string if (startCodePoint < 0) { return Slices.EMPTY_SLICE; } int indexStart = offsetOfCodePoint(utf8, startCodePoint); int indexEnd = utf8.length(); return utf8.slice(indexStart, indexEnd - indexStart); }
@Test public void testCompactBlock() { Slice compactSlice = Slices.copyOf(createExpectedValue(24)); Slice incompactSlice = Slices.copyOf(createExpectedValue(30)).slice(0, 24); boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(new FixedWidthBlock(4, 0, EMPTY_SLICE, Optional.empty())); testCompactBlock(new FixedWidthBlock(4, valueIsNull.length, compactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); testIncompactBlock(new FixedWidthBlock(4, valueIsNull.length - 1, compactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); // underlying slice is not compact testIncompactBlock(new FixedWidthBlock(4, valueIsNull.length, incompactSlice, Optional.of(Slices.wrappedBooleanArray(valueIsNull)))); }
protected void assertSlicePosition(Block block, int position, Slice expectedSliceValue) { int length = block.getSliceLength(position); assertEquals(length, expectedSliceValue.length()); Block expectedBlock = toSingeValuedBlock(expectedSliceValue); for (int offset = 0; offset < length - 3; offset++) { assertEquals(block.getSlice(position, offset, 3), expectedSliceValue.slice(offset, 3)); assertTrue(block.bytesEqual(position, offset, expectedSliceValue, offset, 3)); // if your tests fail here, please change your test to not use this value assertFalse(block.bytesEqual(position, offset, Slices.utf8Slice("XXX"), 0, 3)); assertEquals(block.bytesCompare(position, offset, 3, expectedSliceValue, offset, 3), 0); assertTrue(block.bytesCompare(position, offset, 3, expectedSliceValue, offset, 2) > 0); Slice greaterSlice = createGreaterValue(expectedSliceValue, offset, 3); assertTrue(block.bytesCompare(position, offset, 3, greaterSlice, 0, greaterSlice.length()) < 0); assertTrue(block.equals(position, offset, expectedBlock, 0, offset, 3)); assertEquals(block.compareTo(position, offset, 3, expectedBlock, 0, offset, 3), 0); BlockBuilder blockBuilder = VARBINARY.createBlockBuilder(null, 1); block.writeBytesTo(position, offset, 3, blockBuilder); blockBuilder.closeEntry(); Block segment = blockBuilder.build(); assertTrue(block.equals(position, offset, segment, 0, 0, 3)); } }
@SqlNullable @Description("returns regex group of extracted string with a pattern") @ScalarFunction @LiteralParameters("x") @SqlType("varchar(x)") public static Slice regexpExtract(@SqlType("varchar(x)") Slice source, @SqlType(JoniRegexpType.NAME) Regex pattern, @SqlType(StandardTypes.BIGINT) long groupIndex) { Matcher matcher = pattern.matcher(source.getBytes()); validateGroup(groupIndex, matcher.getEagerRegion()); int group = toIntExact(groupIndex); int offset = matcher.search(0, source.length(), Option.DEFAULT); if (offset == -1) { return null; } Region region = matcher.getEagerRegion(); int beg = region.beg[group]; int end = region.end[group]; if (beg == -1) { // end == -1 must be true return null; } Slice slice = source.slice(beg, end - beg); return slice; }
@Override public Block decodeColumn(ColumnData columnData) { int size = columnData.rowCount(); BlockBuilder builder = type.createBlockBuilder(null, size); Slice slice = columnData.getSlice(); for (int i = 0; i < size; i++) { int length = columnData.getLength(i); if (length > 0) { int offset = columnData.getOffset(i); if ((length == 1) && slice.getByte(offset) == HIVE_EMPTY_STRING_BYTE) { type.writeSlice(builder, EMPTY_SLICE); } else { length = calculateTruncationLength(type, slice, offset, length); type.writeSlice(builder, slice.slice(offset, length)); } } else { builder.appendNull(); } } return builder.build(); }
@ScalarFunction @LiteralParameters("x") @Description("returns array of strings split by pattern") @SqlType("array(varchar(x))") public static Block regexpSplit(@SqlType("varchar(x)") Slice source, @SqlType(JoniRegexpType.NAME) Regex pattern) { Matcher matcher = pattern.matcher(source.getBytes()); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 32); int lastEnd = 0; int nextStart = 0; while (true) { int offset = matcher.search(nextStart, source.length(), Option.DEFAULT); if (offset == -1) { break; } if (matcher.getEnd() == matcher.getBegin()) { nextStart = matcher.getEnd() + 1; } else { nextStart = matcher.getEnd(); } Slice slice = source.slice(lastEnd, matcher.getBegin() - lastEnd); lastEnd = matcher.getEnd(); VARCHAR.writeSlice(blockBuilder, slice); } VARCHAR.writeSlice(blockBuilder, source.slice(lastEnd, source.length() - lastEnd)); return blockBuilder.build(); }
return inputSlice.slice(0, resultLength);