@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractFixedWidthBlock fixedWidthBlock = (AbstractFixedWidthBlock) block; sliceOutput.appendInt(fixedWidthBlock.getFixedSize()); sliceOutput.appendInt(fixedWidthBlock.getPositionCount()); // write null bits 8 at a time encodeNullsAsBits(sliceOutput, fixedWidthBlock); Slice slice = fixedWidthBlock.getRawSlice(); sliceOutput .appendInt(slice.length()) .writeBytes(slice); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. AbstractVariableWidthBlock variableWidthBlock = (AbstractVariableWidthBlock) block; int positionCount = variableWidthBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // offsets int totalLength = 0; for (int position = 0; position < positionCount; position++) { int length = variableWidthBlock.getSliceLength(position); totalLength += length; sliceOutput.appendInt(totalLength); } encodeNullsAsBits(sliceOutput, variableWidthBlock); sliceOutput .appendInt(totalLength) .writeBytes(variableWidthBlock.getRawSlice(0), variableWidthBlock.getPositionOffset(0), totalLength); }
public Slice serialize() { try (SliceOutput output = new DynamicSliceOutput(estimatedSerializedSize())) { output.appendByte(UNCOMPRESSED_FORMAT); Slice serializedHll = hll.serialize(); output.appendInt(serializedHll.length()); output.appendBytes(serializedHll); output.appendInt(maxHashes); output.appendInt(minhash.size()); for (long key : minhash.keySet()) { output.appendLong(key); } for (short value : minhash.values()) { output.appendShort(value); } return output.slice(); } catch (IOException e) { throw new UncheckedIOException(e); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeByte(block.getByte(position, 0)); } } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeShort(block.getShort(position, 0)); } } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeLong(block.getLong(position, 0)); } } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeInt(block.getInt(position, 0)); } } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleRowBlock singleRowBlock = (SingleRowBlock) block; int numFields = singleRowBlock.getNumFields(); int rowIndex = singleRowBlock.getRowIndex(); sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, singleRowBlock.getRawFieldBlock(i).getRegion(rowIndex, 1)); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getRawFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(fieldBlockOffsets[offsetBase + position] - startFieldBlockOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
public Slice serialize() { compact(); int requiredBytes = SizeOf.SIZE_OF_BYTE + // format SizeOf.SIZE_OF_INT + // max buckets SizeOf.SIZE_OF_INT + // entry count SizeOf.SIZE_OF_DOUBLE * nextIndex + // values SizeOf.SIZE_OF_DOUBLE * nextIndex; // weights return Slices.allocate(requiredBytes) .getOutput() .appendByte(FORMAT_TAG) .appendInt(maxBuckets) .appendInt(nextIndex) .appendBytes(Slices.wrappedDoubleArray(values, 0, nextIndex)) .appendBytes(Slices.wrappedDoubleArray(weights, 0, nextIndex)) .getUnderlyingSlice(); }
@Override public void serialize(DigestAndPercentileArrayState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { Slice digest = state.getDigest().serialize(); SliceOutput output = Slices.allocate( SIZE_OF_INT + // number of percentiles state.getPercentiles().size() * SIZE_OF_DOUBLE + // percentiles SIZE_OF_INT + // digest length digest.length()) // digest .getOutput(); // write percentiles List<Double> percentiles = state.getPercentiles(); output.appendInt(percentiles.size()); for (double percentile : percentiles) { output.appendDouble(percentile); } output.appendInt(digest.length()); output.appendBytes(digest); VARBINARY.writeSlice(out, output.slice()); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public byte[] getSerializedData() { // Serialization format is (<key:int><min:double><max:double>)* SliceOutput output = Slices.allocate((SizeOf.SIZE_OF_INT + 2 * SizeOf.SIZE_OF_DOUBLE) * mins.size()).getOutput(); for (int key : mins.keySet()) { output.appendInt(key); output.appendDouble(mins.get(key)); output.appendDouble(maxs.get(key)); } return output.slice().getBytes(); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { SingleMapBlock singleMapBlock = (SingleMapBlock) block; TypeSerde.writeType(sliceOutput, singleMapBlock.keyType); int offset = singleMapBlock.getOffset(); int positionCount = singleMapBlock.getPositionCount(); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawKeyBlock().getRegion(offset / 2, positionCount / 2)); blockEncodingSerde.writeBlock(sliceOutput, singleMapBlock.getRawValueBlock().getRegion(offset / 2, positionCount / 2)); int[] hashTable = singleMapBlock.getHashTable(); sliceOutput.appendInt(positionCount / 2 * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, offset / 2 * HASH_MULTIPLIER, positionCount / 2 * HASH_MULTIPLIER)); }
@Override public void serialize(DigestAndPercentileState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { Slice serialized = state.getDigest().serialize(); SliceOutput output = Slices.allocate(SIZE_OF_DOUBLE + SIZE_OF_INT + serialized.length()).getOutput(); output.appendDouble(state.getPercentile()); output.appendInt(serialized.length()); output.appendBytes(serialized); VARBINARY.writeSlice(out, output.slice()); } }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }
@Override public void loadValue(SliceOutput output, int valueIndex) { output.appendInt(valueIndex); }
@Override public void loadValue(SliceOutput output, int valueIndex) { output.appendInt(valueIndex); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeLong(block.getLong(position, 0)); } } }