public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { return blockEncodingSerde.readBlock(input); }
public static Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput input) { return blockEncodingSerde.readBlock(input); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int numFields = sliceInput.readInt(); Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < fieldBlocks.length; i++) { fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } return new SingleRowBlock(0, fieldBlocks); } }
@Override public RunLengthEncodedBlock readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // read the run length int positionCount = sliceInput.readInt(); // read the value Block value = blockEncodingSerde.readBlock(sliceInput); return new RunLengthEncodedBlock(value, positionCount); } }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Block values = blockEncodingSerde.readBlock(sliceInput); int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(Slices.wrappedIntArray(offsets)); boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createArrayBlockInternal(0, positionCount, valueIsNull, offsets, values); } }
@Override public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); return blockEncodingSerde.readBlock(input); } }
@Override public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); return blockEncodingSerde.readBlock(input); } }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int numFields = sliceInput.readInt(); Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } int positionCount = sliceInput.readInt(); int[] fieldBlockOffsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(fieldBlockOffsets)); boolean[] rowIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); } }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Type keyType = TypeSerde.readType(typeManager, sliceInput); MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( format("Deserialized SingleMapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); } return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Type keyType = TypeSerde.readType(typeManager, sliceInput); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( format("Deserialized MapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); } int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(offsets)); Optional<boolean[]> mapIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); return MapType.createMapBlockInternal(typeManager, keyType, 0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, hashTable); } }
private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // positionCount int positionCount = sliceInput.readInt(); // dictionary Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput); // ids int[] ids = new int[positionCount]; sliceInput.readBytes(Slices.wrappedIntArray(ids)); // instance id long mostSignificantBits = sliceInput.readLong(); long leastSignificantBits = sliceInput.readLong(); long sequenceId = sliceInput.readLong(); // We always compact the dictionary before we send it. However, dictionaryBlock comes from sliceInput, which may over-retain memory. // As a result, setting dictionaryIsCompacted to true is not appropriate here. // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory. return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); } }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
@Test public void testRoundTrip() { int positionCount = 40; // build dictionary BlockBuilder dictionaryBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(dictionaryBuilder, "alice"); VARCHAR.writeString(dictionaryBuilder, "bob"); VARCHAR.writeString(dictionaryBuilder, "charlie"); VARCHAR.writeString(dictionaryBuilder, "dave"); Block dictionary = dictionaryBuilder.build(); // build ids int[] ids = new int[positionCount]; for (int i = 0; i < 40; i++) { ids[i] = i % 4; } DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; assertBlockEquals(VARCHAR, actualDictionaryBlock.getDictionary(), dictionary); for (int position = 0; position < actualDictionaryBlock.getPositionCount(); position++) { assertEquals(actualDictionaryBlock.getId(position), ids[position]); } assertEquals(actualDictionaryBlock.getDictionarySourceId(), dictionaryBlock.getDictionarySourceId()); }
@Test public void testRoundTrip() { int positionCount = 40; // build dictionary BlockBuilder dictionaryBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(dictionaryBuilder, "alice"); VARCHAR.writeString(dictionaryBuilder, "bob"); VARCHAR.writeString(dictionaryBuilder, "charlie"); VARCHAR.writeString(dictionaryBuilder, "dave"); Block dictionary = dictionaryBuilder.build(); // build ids int[] ids = new int[positionCount]; for (int i = 0; i < 40; i++) { ids[i] = i % 4; } DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; assertBlockEquals(VARCHAR, actualDictionaryBlock.getDictionary(), dictionary); for (int position = 0; position < actualDictionaryBlock.getPositionCount(); position++) { assertEquals(actualDictionaryBlock.getId(position), ids[position]); } assertEquals(actualDictionaryBlock.getDictionarySourceId(), dictionaryBlock.getDictionarySourceId()); }