public static void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput output, Block block) { blockEncodingSerde.writeBlock(output, block); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
@Override public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { SliceOutput output = new DynamicSliceOutput(64); blockEncodingSerde.writeBlock(output, block); String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); jsonGenerator.writeString(encoded); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getRawFieldBlock(i).getRegion(rowIndex, 1)); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); }
private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getRawFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(fieldBlockOffsets[offsetBase + position] - startFieldBlockOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Test public void testRoundTrip() { BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(expectedBlockBuilder, "alice"); VARCHAR.writeString(expectedBlockBuilder, "bob"); VARCHAR.writeString(expectedBlockBuilder, "charlie"); VARCHAR.writeString(expectedBlockBuilder, "dave"); Block expectedBlock = expectedBlockBuilder.build(); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, expectedBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertBlockEquals(VARCHAR, actualBlock, expectedBlock); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }
@Test public void testRoundTrip() { int positionCount = 40; // build dictionary BlockBuilder dictionaryBuilder = VARCHAR.createBlockBuilder(null, 4); VARCHAR.writeString(dictionaryBuilder, "alice"); VARCHAR.writeString(dictionaryBuilder, "bob"); VARCHAR.writeString(dictionaryBuilder, "charlie"); VARCHAR.writeString(dictionaryBuilder, "dave"); Block dictionary = dictionaryBuilder.build(); // build ids int[] ids = new int[positionCount]; for (int i = 0; i < 40; i++) { ids[i] = i % 4; } DictionaryBlock dictionaryBlock = new DictionaryBlock(dictionary, ids); DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); blockEncodingSerde.writeBlock(sliceOutput, dictionaryBlock); Block actualBlock = blockEncodingSerde.readBlock(sliceOutput.slice().getInput()); assertTrue(actualBlock instanceof DictionaryBlock); DictionaryBlock actualDictionaryBlock = (DictionaryBlock) actualBlock; assertBlockEquals(VARCHAR, actualDictionaryBlock.getDictionary(), dictionary); for (int position = 0; position < actualDictionaryBlock.getPositionCount(); position++) { assertEquals(actualDictionaryBlock.getId(position), ids[position]); } assertEquals(actualDictionaryBlock.getDictionarySourceId(), dictionaryBlock.getDictionarySourceId()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getRawFieldBlock(i).getRegion(rowIndex, 1)); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getRawFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(fieldBlockOffsets[offsetBase + position] - startFieldBlockOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }