public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
public void write(DataOutput out) throws IOException { Utils.writeString(out, defaultCompressionAlgorithm.getName()); Utils.writeVInt(out, listRegions.size()); for (BlockRegion region : listRegions) { region.write(out); } } }
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s\n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
BlockRegion region = reader.readerBCF.dataIndex.getBlockRegionList().get(i); dataSize += region.getCompressedSize(); dataSizeUncompressed += region.getRawSize(); for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) { MetaIndexEntry e = it.next(); metaSize += e.getRegion().getCompressedSize(); metaSizeUncompressed += e.getRegion().getRawSize(); if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) { calculateCompression = true; out.printf("%s %s %s %s %s ", Align.format(Align.format(i, blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align .format(region.getOffset(), offsetWidth, Align.LEFT), Align .format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT)); byte[] key = indexEntry.key; e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth, Align.LEFT), Align.format(region.getCompressedSize(), blkLenWidth, Align.LEFT), Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT), Align.format(blkCompression, compressionWidth, Align.LEFT));
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.getDecompressor(); try { this.in = compressAlgo .createDecompressionStream(new BoundedRangeFileInputStream( fsin, this.region.getOffset(), this.region .getCompressedSize()), decompressor, TFile .getFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.getDecompressor(); try { this.in = compressAlgo .createDecompressionStream(new BoundedRangeFileInputStream( fsin, this.region.getOffset(), this.region .getCompressedSize()), decompressor, TFile .getFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.getDecompressor(); try { this.in = compressAlgo .createDecompressionStream(new BoundedRangeFileInputStream( fsin, this.region.getOffset(), this.region .getCompressedSize()), decompressor, TFile .getFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.getDecompressor(); try { this.in = compressAlgo .createDecompressionStream(new BoundedRangeFileInputStream( fsin, this.region.getOffset(), this.region .getCompressedSize()), decompressor, TFile .getFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.region = region; this.decompressor = compressionAlgo.getDecompressor(); try { this.in = compressAlgo .createDecompressionStream(new BoundedRangeFileInputStream( fsin, this.region.getOffset(), this.region .getCompressedSize()), decompressor, TFile .getFSInputBufferSize(conf)); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
public void register(long raw, long begin, long end) { dataIndex.addBlockRegion(new BlockRegion(begin, end - begin, raw)); } }
public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
public MetaIndexEntry(DataInput in) throws IOException { String fullMetaName = Utils.readString(in); if (fullMetaName.startsWith(defaultPrefix)) { metaName = fullMetaName.substring(defaultPrefix.length(), fullMetaName .length()); } else { throw new IOException("Corrupted Meta region Index"); } compressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); region = new BlockRegion(in); }
public DataIndex(DataInput in) throws IOException { defaultCompressionAlgorithm = Compression.getCompressionAlgorithmByName(Utils.readString(in)); int n = Utils.readVInt(in); listRegions = new ArrayList<BlockRegion>(n); for (int i = 0; i < n; i++) { BlockRegion region = new BlockRegion(in); listRegions.add(region); } }