private ParquetMetadata readFooterFromFile(final Path file, final FileSystem fs, final FileStatus stat, MetadataFilter filter) throws IOException { InputFile inputFile = new InputFile() { @Override public SeekableInputStream newStream() throws IOException { return HadoopStreams.wrap(fs.open(file)); } @Override public long getLength() throws IOException { return stat.getLen(); } }; return ParquetFileReader.readFooter(inputFile, filter); }
return ParquetFileReader.readFooter(new ParquetFooterInputFromCache(footerData), filter); } finally { metadataCache.decRefBuffer(footerData); return ParquetFileReader.readFooter(new ParquetFooterInputFromCache(footerData), filter); } finally { metadataCache.decRefBuffer(footerData);
final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(jobConf, finalPath); final List<BlockMetaData> blocks = parquetMetadata.getBlocks(); final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
ParquetMetadata footer = readFooter(config, file, range(0, length));
ParquetMetadata footer = readFooter(config, file, range(0, length));
ParquetMetadata footer = readFooter(config, file, range(0, length));
footer = readFooter(configuration, file, range(split.getStart(), split.getEnd())); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = getFilter(configuration); } else { footer = readFooter(configuration, file, NO_FILTER); Set<Long> offsets = new HashSet<>(); for (long offset : rowGroupOffsets) {
footer = readFooter(configuration, file, range(split.getStart(), split.getEnd())); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = getFilter(configuration); } else { footer = readFooter(configuration, file, NO_FILTER); Set<Long> offsets = new HashSet<>(); for (long offset : rowGroupOffsets) {
footer = readFooter(configuration, file, range(split.getStart(), split.getEnd())); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = getFilter(configuration); } else { footer = readFooter(configuration, file, NO_FILTER); Set<Long> offsets = new HashSet<>(); for (long offset : rowGroupOffsets) {
/** * @param configuration a configuration * @param file the Parquet File * @return the metadata with row groups. * @throws IOException if an error occurs while reading the file * @deprecated will be removed in 2.0.0; * use {@link ParquetFileReader#open(InputFile, ParquetReadOptions)} */ @Deprecated public static final ParquetMetadata readFooter(Configuration configuration, FileStatus file) throws IOException { return readFooter(configuration, file, NO_FILTER); }
/** * @param configuration a configuration * @param file the Parquet File * @return the metadata with row groups. * @throws IOException if an error occurs while reading the file * @deprecated will be removed in 2.0.0; * use {@link ParquetFileReader#open(InputFile, ParquetReadOptions)} */ @Deprecated public static final ParquetMetadata readFooter(Configuration configuration, FileStatus file) throws IOException { return readFooter(configuration, file, NO_FILTER); }
@Override public Footer call() throws Exception { try { return new Footer(currentFile.getPath(), readFooter(configuration, currentFile, filter(skipRowGroups))); } catch (IOException e) { throw new IOException("Could not read footer for file " + currentFile, e); } } });
@Override public ParquetMetadata call() throws Exception { try { ParquetMetadata footer = ParquetFileReader.readFooter(configuration, currentFile, NO_FILTER); return footer; } catch (Exception e) { throw new ParquetDecodingException("could not read footer", e); } } }));
@Override public ParquetMetadata call() throws Exception { try { ParquetMetadata footer = ParquetFileReader.readFooter(configuration, currentFile, NO_FILTER); return footer; } catch (Exception e) { throw new ParquetDecodingException("could not read footer", e); } } }));
@Override public Footer call() throws Exception { try { return new Footer(currentFile.getPath(), readFooter(configuration, currentFile, filter(skipRowGroups))); } catch (IOException e) { throw new IOException("Could not read footer for file " + currentFile, e); } } });
public ParquetMetadata getFooter() { if (footer == null) { try { // don't read the row groups because this.blocks is always set this.footer = readFooter(file, options, f, converter); } catch (IOException e) { throw new ParquetDecodingException("Unable to read file footer", e); } } return footer; }
public ParquetMetadata getFooter() { if (footer == null) { try { // don't read the row groups because this.blocks is always set this.footer = readFooter(file, options, f, converter); } catch (IOException e) { throw new ParquetDecodingException("Unable to read file footer", e); } } return footer; }
public static ParquetMetadata readMetadata(Configuration conf, Path parquetFilePath) { ParquetMetadata footer; try { // TODO(vc): Should we use the parallel reading version here? footer = ParquetFileReader .readFooter(getFs(parquetFilePath.toString(), conf).getConf(), parquetFilePath); } catch (IOException e) { throw new HoodieIOException("Failed to read footer for parquet " + parquetFilePath, e); } return footer; }
@Override public RecordReader getRecordReader(OperatorContext context, FileSystemWrapper fs, FileStatus status) throws ExecutionSetupException { try { final ParquetMetadata footer = ParquetFileReader.readFooter(fsPlugin.getFsConf(), status, ParquetMetadataConverter.NO_FILTER); if (footer.getBlocks().size() == 0) { return null; } return new PreviewReader(context, fs, status, footer); } catch (IOException e) { throw new ExecutionSetupException(e); } }
public ParquetFileReader(InputFile file, ParquetReadOptions options) throws IOException { this.converter = new ParquetMetadataConverter(options); this.file = file; this.f = file.newStream(); this.options = options; this.footer = readFooter(file, options, f, converter); this.fileMetaData = footer.getFileMetaData(); this.blocks = filterRowGroups(footer.getBlocks()); for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) { paths.put(ColumnPath.get(col.getPath()), col); } }