private RowRanges getRowRanges(int blockIndex) { RowRanges rowRanges = blockRowRanges.get(blockIndex); if (rowRanges == null) { rowRanges = ColumnIndexFilter.calculateRowRanges(options.getRecordFilter(), getColumnIndexStore(blockIndex), paths.keySet(), blocks.get(blockIndex).getRowCount()); blockRowRanges.set(blockIndex, rowRanges); } return rowRanges; }
private List<BlockMetaData> filterRowGroups(List<BlockMetaData> blocks) throws IOException { // set up data filters based on configured levels List<RowGroupFilter.FilterLevel> levels = new ArrayList<>(); if (options.useStatsFilter()) { levels.add(STATISTICS); } if (options.useDictionaryFilter()) { levels.add(DICTIONARY); } FilterCompat.Filter recordFilter = options.getRecordFilter(); if (recordFilter != null) { return RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, this); } return blocks; }
private List<BlockMetaData> filterRowGroups(List<BlockMetaData> blocks) throws IOException { // set up data filters based on configured levels List<RowGroupFilter.FilterLevel> levels = new ArrayList<>(); if (options.useStatsFilter()) { levels.add(STATISTICS); } if (options.useDictionaryFilter()) { levels.add(DICTIONARY); } FilterCompat.Filter recordFilter = options.getRecordFilter(); if (recordFilter != null) { return RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, this); } return blocks; }
private void initReader() throws IOException { if (reader != null) { reader.close(); reader = null; } if (filesIterator.hasNext()) { InputFile file = filesIterator.next(); ParquetFileReader fileReader = ParquetFileReader.open(file, options); reader = new InternalParquetRecordReader<>(readSupport, options.getRecordFilter()); reader.initialize(fileReader, options); } }
private void initReader() throws IOException { if (reader != null) { reader.close(); reader = null; } if (filesIterator.hasNext()) { InputFile file = filesIterator.next(); ParquetFileReader fileReader = ParquetFileReader.open(file, options); reader = new InternalParquetRecordReader<>(readSupport, options.getRecordFilter()); reader.initialize(fileReader, options); } }