private void validateWritePageChecksum() throws IOException { if (writeChecksumBuilder.isPresent()) { Block[] blocks = new Block[streamReaders.length]; for (int columnIndex = 0; columnIndex < streamReaders.length; columnIndex++) { blocks[columnIndex] = readBlock(includedColumns.get(columnIndex), columnIndex); } Page page = new Page(currentBatchSize, blocks); writeChecksumBuilder.get().addPage(page); rowGroupStatisticsValidation.get().addPage(page); stripeStatisticsValidation.get().addPage(page); fileStatisticsValidation.get().addPage(page); } }
private static void assertCurrentBatch(OrcRecordReader reader, int rowIndex, int batchSize) throws IOException { Block block = reader.readBlock(BIGINT, 0); for (int i = 0; i < batchSize; i++) { assertEquals(BIGINT.getLong(block, i), (rowIndex + i) * 3); } }
private static void assertCurrentBatch(OrcRecordReader reader, int stripe) throws IOException { Block block = reader.readBlock(BIGINT, 0); for (int i = 0; i < 20; i++) { assertEquals(BIGINT.getLong(block, i), ((stripe * 20L) + i) * 3); } }
@Benchmark public Object readBooleanWithNull(BooleanWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BOOLEAN, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimalWithNull(DecimalWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readBooleanNoNull(BooleanNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BOOLEAN, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readLongDirectNoNull(BigintNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BIGINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readSliceDictionaryNoNull(VarcharNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(VARCHAR, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readSliceDictionaryWithNull(VarcharWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(VARCHAR, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimal(BenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readByteNoNull(TinyIntNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TINYINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimalNoNull(DecimalNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDoubleWithNull(DoubleWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DOUBLE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readFloatNoNull(FloatNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(REAL, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readByteWithNull(TinyIntWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TINYINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDoubleNoNull(DoubleNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DOUBLE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readFloatWithNull(FloatWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(REAL, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readLongDirectWithNull(BigintWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BIGINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readTimestampNoNull(TimestampNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TIMESTAMP, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readTimestampWithNull(TimestampWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TIMESTAMP, 0); blocks.add(block); } return blocks; }