@Test public void testSkipsValue() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); writeBigInteger(baos, BigInteger.valueOf(Long.MAX_VALUE)); writeBigInteger(baos, BigInteger.valueOf(Long.MIN_VALUE)); OrcInputStream inputStream = orcInputStreamFor("skip test", baos.toByteArray()); DecimalInputStream stream = new DecimalInputStream(inputStream); stream.skip(1); assertEquals(stream.nextLong(), Long.MIN_VALUE); }
private static void assertReadsLongValue(BigInteger value) throws IOException { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(value)); Slice decimal = unscaledDecimal(); stream.nextLongDecimal(decimal); assertEquals(unscaledDecimalToBigInteger(decimal), value); }
private static void assertReadsShortValue(long value) throws IOException { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(BigInteger.valueOf(value))); assertEquals(stream.nextLong(), value); }
long sourceScale = scaleStream.next(); if (decimalType.isShort()) { long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); decimalType.writeLong(builder, rescaledDecimal); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); decimalType.writeSlice(builder, rescaledDecimal); long sourceScale = scaleStream.next(); if (decimalType.isShort()) { long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); decimalType.writeLong(builder, rescaledDecimal); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); decimalType.writeSlice(builder, rescaledDecimal);
@Override protected DecimalInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new DecimalInputStream(input); }
@Override protected Long readValue(DecimalInputStream valueStream) throws IOException { return valueStream.nextLong(); } }
@Override protected Slice readValue(DecimalInputStream valueStream) throws IOException { Slice decimal = unscaledDecimal(); valueStream.nextLongDecimal(decimal); return decimal; } }
private void seekToOffset() throws IOException { if (readOffset > 0) { if (presentStream != null) { // skip ahead the present bit reader, but count the set bits // and use this as the skip size for the data reader readOffset = presentStream.countBitsSet(readOffset); } if (readOffset > 0) { if (decimalStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but decimal stream is not present"); } if (scaleStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but scale stream is not present"); } decimalStream.skip(readOffset); scaleStream.skip(readOffset); } } }
private static void assertShortValueReadFails(BigInteger value) { assertThrows(OrcCorruptionException.class, () -> { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(value)); stream.nextLong(); }); }
long sourceScale = scaleStream.next(); if (decimalType.isShort()) { long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); decimalType.writeLong(builder, rescaledDecimal); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); decimalType.writeSlice(builder, rescaledDecimal); long sourceScale = scaleStream.next(); if (decimalType.isShort()) { long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); decimalType.writeLong(builder, rescaledDecimal); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); decimalType.writeSlice(builder, rescaledDecimal);
@Override protected DecimalInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new DecimalInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
public void nextShortDecimalVector(int items, BlockBuilder builder, DecimalType targetType, long[] sourceScale) throws IOException { for (int i = 0; i < items; i++) { long value = nextLong(); long rescaledDecimal = Decimals.rescale(value, (int) sourceScale[i], targetType.getScale()); targetType.writeLong(builder, rescaledDecimal); } }
@Override protected Slice readValue(DecimalInputStream valueStream) throws IOException { Slice decimal = unscaledDecimal(); valueStream.nextLongDecimal(decimal); return decimal; } }
private void seekToOffset() throws IOException { if (readOffset > 0) { if (presentStream != null) { // skip ahead the present bit reader, but count the set bits // and use this as the skip size for the data reader readOffset = presentStream.countBitsSet(readOffset); } if (readOffset > 0) { if (decimalStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but decimal stream is not present"); } if (scaleStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but scale stream is not present"); } decimalStream.skip(readOffset); scaleStream.skip(readOffset); } } }
@Test public void testSkipsValue() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); writeBigInteger(baos, BigInteger.valueOf(Long.MAX_VALUE)); writeBigInteger(baos, BigInteger.valueOf(Long.MIN_VALUE)); OrcInputStream inputStream = orcInputStreamFor("skip test", baos.toByteArray()); DecimalInputStream stream = new DecimalInputStream(inputStream); stream.skip(1); assertEquals(stream.nextLong(), Long.MIN_VALUE); }
private static void assertLongValueReadFails(BigInteger value) { Slice decimal = unscaledDecimal(); assertThrows(OrcCorruptionException.class, () -> { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(value)); stream.nextLongDecimal(decimal); }); }
private static void assertReadsShortValue(long value) throws IOException { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(BigInteger.valueOf(value))); assertEquals(stream.nextLong(), value); }
return createLongStream(inputStream, encoding, type, true, usesVInt); case DECIMAL: return new DecimalInputStream(inputStream);
public void nextShortDecimalVector(int items, BlockBuilder builder, DecimalType targetType, long[] sourceScale, boolean[] isNull) throws IOException { for (int i = 0; i < items; i++) { if (!isNull[i]) { long rescaledDecimal = Decimals.rescale(nextLong(), (int) sourceScale[i], targetType.getScale()); targetType.writeLong(builder, rescaledDecimal); } else { builder.appendNull(); } } }
private static void assertReadsLongValue(BigInteger value) throws IOException { DecimalInputStream stream = new DecimalInputStream(decimalInputStream(value)); Slice decimal = unscaledDecimal(); stream.nextLongDecimal(decimal); assertEquals(unscaledDecimalToBigInteger(decimal), value); }