public void setBuffers(Slice lengthsBuffer, Slice dataBuffer, int uncompressedDataSize) { this.lengthsInput = lengthsBuffer.getInput(); this.dataBuffer = dataBuffer; this.uncompressedDataSize = uncompressedDataSize; compressed = (decompressor != null); currentPosition = 0; currentOffset = 0; runLength = 0; lastValueLength = 0; }
public static JsonParser createJsonParser(JsonFactory factory, Slice json) throws IOException { // Jackson tries to detect the character encoding automatically when using InputStream // so we pass an InputStreamReader instead. return factory.createParser(new InputStreamReader(json.getInput(), UTF_8)); }
public static GeometrySerializationType deserializeType(Slice shape) { requireNonNull(shape, "shape is null"); BasicSliceInput input = shape.getInput(); verify(input.available() > 0); return GeometrySerializationType.getForCode(input.readByte()); }
@Nullable public static Envelope deserializeEnvelope(Slice shape) { requireNonNull(shape, "shape is null"); BasicSliceInput input = shape.getInput(); verify(input.available() > 0); int length = input.available() - 1; GeometrySerializationType type = GeometrySerializationType.getForCode(input.readByte()); return getEnvelope(input, type, length); }
@Setup public void setup() { byte[] bytes = new byte[256 * 64]; ThreadLocalRandom.current().nextBytes(bytes); input = Slices.wrappedBuffer(bytes).getInput(); } }
public StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); int tailLength = toIntExact(stripe.getFooterLength()); // read the footer byte[] tailBuffer = new byte[tailLength]; orcDataSource.readFully(offset, tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, systemMemoryUsage, tailLength)) { return metadataReader.readStripeFooter(types, inputStream); } }
public static FeatureUnitNormalizer deserialize(byte[] modelData) { SliceInput input = Slices.wrappedBuffer(modelData).getInput(); FeatureUnitNormalizer model = new FeatureUnitNormalizer(); while (input.isReadable()) { int key = input.readInt(); model.mins.put(key, input.readDouble()); model.maxs.put(key, input.readDouble()); } return model; }
@Override public void deserialize(Block block, int index, LongDecimalWithOverflowState state) { if (!block.isNull(index)) { SliceInput slice = VARBINARY.getSlice(block, index).getInput(); state.setOverflow(slice.readLong()); state.setLongDecimal(Slices.copyOf(slice.readSlice(slice.available()))); } } }
private static Block copyBlock(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
private static Block copyBlockViaBlockSerde(Block block) { DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024); BLOCK_ENCODING_SERDE.writeBlock(sliceOutput, block); return BLOCK_ENCODING_SERDE.readBlock(sliceOutput.slice().getInput()); }
@Override public void deserialize(Block block, int index, LongDecimalWithOverflowAndLongState state) { if (!block.isNull(index)) { SliceInput slice = VARBINARY.getSlice(block, index).getInput(); state.setLong(slice.readLong()); state.setOverflow(slice.readLong()); state.setLongDecimal(Slices.copyOf(slice.readSlice(slice.available()))); } } }
@Override protected DecimalInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new DecimalInputStream(input); }
@Override protected LongInputStreamDwrf createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new LongInputStreamDwrf(input, LONG, true, true); }
@Override protected LongInputStreamV1 createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new LongInputStreamV1(input, true); }
@Override protected BooleanInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new BooleanInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override protected FloatInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new FloatInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override protected ByteInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new ByteInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override protected ByteArrayInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new ByteArrayInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override protected DecimalInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new DecimalInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override protected DoubleInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new DoubleInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }