@Override public Block getRegion(int positionOffset, int length) { return block.getRegion(positionOffset, length); }
@Override public Block getRegion(int positionOffset, int length) { assureLoaded(); return block.getRegion(positionOffset, length); }
public Page getRegion(int positionOffset, int length) { if (positionOffset < 0 || length < 0 || positionOffset + length > positionCount) { throw new IndexOutOfBoundsException(format("Invalid position %s and length %s in page with %s positions", positionOffset, length, positionCount)); } int channelCount = getChannelCount(); Block[] slicedBlocks = new Block[channelCount]; for (int i = 0; i < channelCount; i++) { slicedBlocks[i] = blocks[i].getRegion(positionOffset, length); } return new Page(length, slicedBlocks); }
@Override public <T> T getObject(int position, Class<T> clazz) { if (clazz != Block.class) { throw new IllegalArgumentException("clazz must be Block.class"); } checkReadablePosition(position); int startValueOffset = getOffset(position); int endValueOffset = getOffset(position + 1); return clazz.cast(getRawElementBlock().getRegion(startValueOffset, endValueOffset - startValueOffset)); }
public static RunLengthEncodedBlock createTestRleBlock(Block block, int position) { return new RunLengthEncodedBlock(block.getRegion(position, 1), 10); } }
protected List<Block> splitBlock(Block block, int count) { double sizePerSplit = block.getPositionCount() * 1.0 / count; ImmutableList.Builder<Block> result = ImmutableList.builder(); for (int i = 0; i < count; i++) { int startPosition = toIntExact(Math.round(sizePerSplit * i)); int endPosition = toIntExact(Math.round(sizePerSplit * (i + 1))); result.add(block.getRegion(startPosition, endPosition - startPosition)); } return result.build(); }
public static ColumnarMap toColumnarMap(Block block) { requireNonNull(block, "block is null"); if (block instanceof DictionaryBlock) { return toColumnarMap((DictionaryBlock) block); } if (block instanceof RunLengthEncodedBlock) { return toColumnarMap((RunLengthEncodedBlock) block); } if (!(block instanceof AbstractMapBlock)) { throw new IllegalArgumentException("Invalid map block: " + block.getClass().getName()); } AbstractMapBlock mapBlock = (AbstractMapBlock) block; int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); // get the keys and values for visible region int firstEntryPosition = mapBlock.getOffset(0); int totalEntryCount = mapBlock.getOffset(block.getPositionCount()) - firstEntryPosition; Block keysBlock = mapBlock.getRawKeyBlock().getRegion(firstEntryPosition, totalEntryCount); Block valuesBlock = mapBlock.getRawValueBlock().getRegion(firstEntryPosition, totalEntryCount); return new ColumnarMap(block, offsetBase, offsets, keysBlock, valuesBlock); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getRawFieldBlock(i).getRegion(rowIndex, 1)); } }
@Override public Work<Block> project(ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { Block block = requireNonNull(page, "page is null").getBlock(0); requireNonNull(selectedPositions, "selectedPositions is null"); Block result; if (selectedPositions.isList()) { result = block.copyPositions(selectedPositions.getPositions(), selectedPositions.getOffset(), selectedPositions.size()); } else { result = block.getRegion(selectedPositions.getOffset(), selectedPositions.size()); } return new CompletedWork<>(result); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); }
@Override public void addInput(Page page) { checkState(needsInput()); if (page.getPositionCount() <= remainingLimit) { remainingLimit -= page.getPositionCount(); nextPage = page; } else { Block[] blocks = new Block[page.getChannelCount()]; for (int channel = 0; channel < page.getChannelCount(); channel++) { Block block = page.getBlock(channel); blocks[channel] = block.getRegion(0, (int) remainingLimit); } nextPage = new Page((int) remainingLimit, blocks); remainingLimit = 0; } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
protected void assertPositionEquals(Block block, int position, Object expectedStackValue, Object expectedObjectValue) { long hash = 0; if (type.isComparable()) { hash = hashPosition(type, block, position); } assertPositionValue(block, position, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getSingleValueBlock(position), 0, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(position, 1), 0, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(0, position + 1), position, expectedStackValue, hash, expectedObjectValue); assertPositionValue(block.getRegion(position, block.getPositionCount() - position), 0, expectedStackValue, hash, expectedObjectValue); BlockBuilder blockBuilder = type.createBlockBuilder(null, 1); type.appendTo(block, position, blockBuilder); assertPositionValue(blockBuilder.build(), 0, expectedStackValue, hash, expectedObjectValue); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getRawFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(fieldBlockOffsets[offsetBase + position] - startFieldBlockOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
private static void verifyBlock(Block block, Slice[][][] expectedValues) { assertBlock(block, expectedValues); assertColumnarMap(block, expectedValues); assertDictionaryBlock(block, expectedValues); assertRunLengthEncodedBlock(block, expectedValues); int offset = 1; int length = expectedValues.length - 2; Block blockRegion = block.getRegion(offset, length); Slice[][][] expectedValuesRegion = Arrays.copyOfRange(expectedValues, offset, offset + length); assertBlock(blockRegion, expectedValuesRegion); assertColumnarMap(blockRegion, expectedValuesRegion); assertDictionaryBlock(blockRegion, expectedValuesRegion); assertRunLengthEncodedBlock(blockRegion, expectedValuesRegion); }
private static <T> void verifyBlock(Block block, T[] expectedValues) { assertBlock(block, expectedValues); assertColumnarArray(block, expectedValues); assertDictionaryBlock(block, expectedValues); assertRunLengthEncodedBlock(block, expectedValues); int offset = 1; int length = expectedValues.length - 2; Block blockRegion = block.getRegion(offset, length); T[] expectedValuesRegion = Arrays.copyOfRange(expectedValues, offset, offset + length); assertBlock(blockRegion, expectedValuesRegion); assertColumnarArray(blockRegion, expectedValuesRegion); assertDictionaryBlock(blockRegion, expectedValuesRegion); assertRunLengthEncodedBlock(blockRegion, expectedValuesRegion); }
private static <T> void verifyBlock(Block block, T[] expectedValues) { assertBlock(block, expectedValues); assertColumnarRow(block, expectedValues); assertDictionaryBlock(block, expectedValues); assertRunLengthEncodedBlock(block, expectedValues); int offset = 1; int length = expectedValues.length - 2; Block blockRegion = block.getRegion(offset, length); T[] expectedValuesRegion = Arrays.copyOfRange(expectedValues, offset, offset + length); assertBlock(blockRegion, expectedValuesRegion); assertColumnarRow(blockRegion, expectedValuesRegion); assertDictionaryBlock(blockRegion, expectedValuesRegion); assertRunLengthEncodedBlock(blockRegion, expectedValuesRegion); }
private static void testProjectRange(Block block, Class<? extends Block> expectedResultType, DictionaryAwarePageProjection projection, boolean forceYield) { DriverYieldSignal yieldSignal = new DriverYieldSignal(); Work<Block> work = projection.project(null, yieldSignal, new Page(block), SelectedPositions.positionsRange(5, 10)); Block result; if (forceYield) { result = projectWithYield(work, yieldSignal); } else { assertTrue(work.process()); result = work.getResult(); } assertBlockEquals( BIGINT, result, block.getRegion(5, 10)); assertInstanceOf(result, expectedResultType); }
private static void testProjectFastReturnIgnoreYield(Block block, DictionaryAwarePageProjection projection) { DriverYieldSignal yieldSignal = new DriverYieldSignal(); Work<Block> work = projection.project(null, yieldSignal, new Page(block), SelectedPositions.positionsRange(5, 10)); yieldSignal.setWithDelay(1, executor); yieldSignal.forceYieldForTesting(); // yield signal is ignored given the block has already been loaded assertTrue(work.process()); Block result = work.getResult(); yieldSignal.reset(); assertBlockEquals( BIGINT, result, block.getRegion(5, 10)); assertInstanceOf(result, DictionaryBlock.class); }