@Override public Footer call() throws Exception { try { return new Footer(currentFile.getPath(), readFooter(configuration, currentFile, filter(skipRowGroups))); } catch (IOException e) { throw new IOException("Could not read footer for file " + currentFile, e); } } });
/** * read all the footers of the files provided * (not using summary files) * @param configuration the conf to access the File System * @param partFiles the files to read * @param skipRowGroups to skip the rowGroup info * @return the footers * @throws IOException */ public static List<Footer> readAllFootersInParallel(final Configuration configuration, List<FileStatus> partFiles, final boolean skipRowGroups) throws IOException { List<Callable<Footer>> footers = new ArrayList<Callable<Footer>>(); for (final FileStatus currentFile : partFiles) { footers.add(new Callable<Footer>() { @Override public Footer call() throws Exception { try { return new Footer(currentFile.getPath(), readFooter(configuration, currentFile, filter(skipRowGroups))); } catch (IOException e) { throw new IOException("Could not read footer for file " + currentFile, e); } } }); } try { return runAllInParallel(configuration.getInt(PARQUET_READ_PARALLELISM, 5), footers); } catch (ExecutionException e) { throw new IOException("Could not read footer: " + e.getMessage(), e.getCause()); } }
static List<Footer> footersFromSummaryFile(final Path parent, ParquetMetadata mergedFooters) { Map<Path, ParquetMetadata> footers = new HashMap<Path, ParquetMetadata>(); List<BlockMetaData> blocks = mergedFooters.getBlocks(); for (BlockMetaData block : blocks) { String path = block.getPath(); Path fullPath = new Path(parent, path); ParquetMetadata current = footers.get(fullPath); if (current == null) { current = new ParquetMetadata(mergedFooters.getFileMetaData(), new ArrayList<BlockMetaData>()); footers.put(fullPath, current); } current.getBlocks().add(block); } List<Footer> result = new ArrayList<Footer>(); for (Entry<Path, ParquetMetadata> entry : footers.entrySet()) { result.add(new Footer(entry.getKey(), entry.getValue())); } return result; }
@Override public Map<Path, Footer> call() throws Exception { ParquetMetadata mergedMetadata = readSummaryMetadata(configuration, path, skipRowGroups); if (mergedMetadata != null) { final List<Footer> footers; if (skipRowGroups) { footers = new ArrayList<Footer>(); for (FileStatus f : partFiles) { footers.add(new Footer(f.getPath(), mergedMetadata)); } } else { footers = footersFromSummaryFile(path, mergedMetadata); } Map<Path, Footer> map = new HashMap<Path, Footer>(); for (Footer footer : footers) { // the folder may have been moved footer = new Footer(new Path(path, footer.getFile().getName()), footer.getParquetMetadata()); map.put(footer.getFile(), footer); } return map; } else { return Collections.emptyMap(); } } });
public FootersCacheValue(FileStatusWrapper status, Footer footer) { this.modificationTime = status.getModificationTime(); this.footer = new Footer(footer.getFile(), footer.getParquetMetadata()); }