private void initializeInternalReader(ParquetInputSplit split, Configuration configuration) throws IOException { Path path = split.getPath(); long[] rowGroupOffsets = split.getRowGroupOffsets(); // if task.side.metadata is set, rowGroupOffsets is null ParquetReadOptions.Builder optionsBuilder = HadoopReadOptions.builder(configuration); if (rowGroupOffsets != null) { optionsBuilder.withOffsets(rowGroupOffsets); } else { optionsBuilder.withRange(split.getStart(), split.getEnd()); } // open a reader with the metadata filter ParquetFileReader reader = ParquetFileReader.open( HadoopInputFile.fromPath(path, configuration), optionsBuilder.build()); if (rowGroupOffsets != null) { // verify a row group was found for each offset List<BlockMetaData> blocks = reader.getFooter().getBlocks(); if (blocks.size() != rowGroupOffsets.length) { throw new IllegalStateException( "All of the offsets in the split should be found in the file." + " expected: " + Arrays.toString(rowGroupOffsets) + " found: " + blocks); } } if (!reader.getRowGroups().isEmpty()) { checkDeltaByteArrayProblem( reader.getFooter().getFileMetaData(), configuration, reader.getRowGroups().get(0)); } internalReader.initialize(reader, configuration); }
private void initializeInternalReader(ParquetInputSplit split, Configuration configuration) throws IOException { Path path = split.getPath(); long[] rowGroupOffsets = split.getRowGroupOffsets(); // if task.side.metadata is set, rowGroupOffsets is null ParquetReadOptions.Builder optionsBuilder = HadoopReadOptions.builder(configuration); if (rowGroupOffsets != null) { optionsBuilder.withOffsets(rowGroupOffsets); } else { optionsBuilder.withRange(split.getStart(), split.getEnd()); } // open a reader with the metadata filter ParquetFileReader reader = ParquetFileReader.open( HadoopInputFile.fromPath(path, configuration), optionsBuilder.build()); if (rowGroupOffsets != null) { // verify a row group was found for each offset List<BlockMetaData> blocks = reader.getFooter().getBlocks(); if (blocks.size() != rowGroupOffsets.length) { throw new IllegalStateException( "All of the offsets in the split should be found in the file." + " expected: " + Arrays.toString(rowGroupOffsets) + " found: " + blocks); } } if (!reader.getRowGroups().isEmpty()) { checkDeltaByteArrayProblem( reader.getFooter().getFileMetaData(), configuration, reader.getRowGroups().get(0)); } internalReader.initialize(reader, configuration); }