/** * @param f the Parquet file (will be opened for read in this constructor) * @param blocks the blocks to read * @param colums the columns to read (their path) * @param codecClassName the codec used to compress the blocks * @throws IOException if the file can not be opened */ public ParquetFileReader(Configuration configuration, Path filePath, List<BlockMetaData> blocks, List<ColumnDescriptor> columns) throws IOException { this.filePath = filePath; FileSystem fs = filePath.getFileSystem(configuration); this.f = fs.open(filePath); this.blocks = blocks; for (ColumnDescriptor col : columns) { paths.put(ColumnPath.get(col.getPath()), col); } this.codecFactory = new CodecFactory(configuration); }
public BytesDecompressor getDecompressor(CompressionCodecName codecName) { BytesDecompressor decomp = decompressors.get(codecName); if (decomp == null) { CompressionCodec codec = getCodec(codecName); decomp = new BytesDecompressor(codec); decompressors.put(codecName, decomp); } return decomp; }
@Override public void close() throws IOException { f.close(); this.codecFactory.release(); }
fileWriter.start(); CodecFactory codecFactory = new CodecFactory(conf); CodecFactory.BytesCompressor compressor = codecFactory.getCompressor(compressionCodecName, 0); this.writer = new InternalParquetRecordWriter<T>( fileWriter,
+ " pages ending at file offset " + (descriptor.fileOffset + pos())); BytesDecompressor decompressor = codecFactory.getDecompressor(descriptor.metadata.getCodec()); return new ColumnChunkPageReader(decompressor, pagesInChunk, dictionaryPage);
final WriteSupport<T> writeSupport = getWriteSupport(conf); CodecFactory codecFactory = new CodecFactory(conf); long blockSize = getLongBlockSize(conf); if (INFO) LOG.info("Parquet block size to " + blockSize); init.getExtraMetaData(), blockSize, pageSize, codecFactory.getCompressor(codec, pageSize), dictionaryPageSize, enableDictionary,
public BytesCompressor getCompressor(CompressionCodecName codecName, int pageSize) { BytesCompressor comp = compressors.get(codecName); if (comp == null) { CompressionCodec codec = getCodec(codecName); comp = new BytesCompressor(codecName, codec, pageSize); compressors.put(codecName, comp); } return comp; }