public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { return blockEncodingSerde.readBlock(input); }
private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
public static void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput output, Block block) { blockEncodingSerde.writeBlock(output, block); } }
public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { BlockEncoding blockEncoding = blockEncodingSerde.readBlockEncoding(input); return blockEncoding.readBlock(input); }
public static void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput output, Block block) { BlockEncoding encoding = block.getEncoding(); blockEncodingSerde.writeBlockEncoding(output, encoding); encoding.writeBlock(output, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int numFields = sliceInput.readInt(); Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < fieldBlocks.length; i++) { fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } return new SingleRowBlock(0, fieldBlocks); } }
private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
@Override public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { SliceOutput output = new DynamicSliceOutput(64); blockEncodingSerde.writeBlock(output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } }
@Override public RunLengthEncodedBlock readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // read the run length int positionCount = sliceInput.readInt(); // read the value Block value = blockEncodingSerde.readBlock(sliceInput); return new RunLengthEncodedBlock(value, positionCount); } }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getRawFieldBlock(i).getRegion(rowIndex, 1)); } }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Block values = blockEncodingSerde.readBlock(sliceInput); int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(Slices.wrappedIntArray(offsets)); boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createArrayBlockInternal(0, positionCount, valueIsNull, offsets, values); } }
@Test public void testRoundTrip() { int positionCount = 40; // build dictionary BlockBuilder dictionaryBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(dictionaryBuilder, "alice"); VARCHAR.writeString(dictionaryBuilder, "bob"); VARCHAR.writeString(dictionaryBuilder, "charlie"); VARCHAR.writeString(dictionaryBuilder, "dave"); Block dictionary = dictionaryBuilder.build(); // build ids int[] ids = new int[positionCount]; for (int i = 0; i < 40; i++) { ids[i] = i % 4; } DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; assertBlockEquals(VARCHAR, actualDictionaryBlock.getDictionary(), dictionary); for (int position = 0; position < actualDictionaryBlock.getPositionCount(); position++) { assertEquals(actualDictionaryBlock.getId(position), ids[position]); } assertEquals(actualDictionaryBlock.getDictionarySourceId(), dictionaryBlock.getDictionarySourceId()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); }
@Override public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); return blockEncodingSerde.readBlock(input); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int numFields = sliceInput.readInt(); Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } int positionCount = sliceInput.readInt(); int[] fieldBlockOffsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(fieldBlockOffsets)); boolean[] rowIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Type keyType = TypeSerde.readType(typeManager, sliceInput); MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( format("Deserialized SingleMapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); } return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } }