/** * read all the footers of the files provided * (not using summary files) * @param configuration the conf to access the File System * @param partFiles the files to read * @param skipRowGroups to skip the rowGroup info * @return the footers * @throws IOException */ public static List<Footer> readAllFootersInParallel(final Configuration configuration, List<FileStatus> partFiles, final boolean skipRowGroups) throws IOException { List<Callable<Footer>> footers = new ArrayList<Callable<Footer>>(); for (final FileStatus currentFile : partFiles) { footers.add(new Callable<Footer>() { @Override public Footer call() throws Exception { try { return new Footer(currentFile.getPath(), readFooter(configuration, currentFile, filter(skipRowGroups))); } catch (IOException e) { throw new IOException("Could not read footer for file " + currentFile, e); } } }); } try { return runAllInParallel(configuration.getInt(PARQUET_READ_PARALLELISM, 5), footers); } catch (ExecutionException e) { throw new IOException("Could not read footer: " + e.getMessage(), e.getCause()); } }
List<Map<Path, Footer>> footersFromSummaries = runAllInParallel(configuration.getInt(PARQUET_READ_PARALLELISM, 5), summaries); for (Map<Path, Footer> footers : footersFromSummaries) { cache.putAll(footers);