@Test public void testTrailerForV2NonPBCompatibility() throws Exception { if (version == 2) { FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); t.setDataIndexCount(3); t.setEntryCount(((long) Integer.MAX_VALUE) + 1); t.setLastDataBlockOffset(291); t.setNumDataIndexLevels(3); t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass()); t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. t.setUncompressedDataIndexSize(827398717L); // Something random. t.setLoadOnOpenOffset(128); t.setMetaIndexCount(7); t.setTotalUncompressedBytes(129731987); serializeAsWritable(dos, t); dos.flush(); assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size()); assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version)); FixedFileTrailer t2 = new FixedFileTrailer(version, HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM); t2.deserialize(dis); assertEquals(-1, bais.read()); // Ensure we have read everything. checkLoadedTrailer(version, t, t2);
private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) throws IOException { assertEquals(version, loaded.getMajorVersion()); assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); assertEquals(Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), loaded.getEntryCount()); if (version == 1) { assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); } if (version == 2) { assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); assertEquals(expected.createComparator().getClass().getName(), loaded.createComparator().getClass().getName()); assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( expected.createComparator() instanceof CellComparatorImpl); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); }
private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) throws IOException { BlockType.TRAILER.write(output); output.writeLong(fft.getFileInfoOffset()); output.writeLong(fft.getLoadOnOpenDataOffset()); output.writeInt(fft.getDataIndexCount()); output.writeLong(fft.getUncompressedDataIndexSize()); output.writeInt(fft.getMetaIndexCount()); output.writeLong(fft.getTotalUncompressedBytes()); output.writeLong(fft.getEntryCount()); output.writeInt(fft.getCompressionCodec().ordinal()); output.writeInt(fft.getNumDataIndexLevels()); output.writeLong(fft.getFirstDataBlockOffset()); output.writeLong(fft.getLastDataBlockOffset()); Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); }
CellComparator createComparator() throws IOException { expectAtLeastMajorVersion(2); return createComparator(comparatorClassName); }
/** * File version check is a little sloppy. We read v3 files but can also read v2 files if their * content has been pb'd; files written with 0.98. */ private void checkFileVersion() { int majorVersion = trailer.getMajorVersion(); if (majorVersion == getMajorVersion()) return; int minorVersion = trailer.getMinorVersion(); if (majorVersion == 2 && minorVersion >= MIN_V2_MINOR_VERSION_WITH_PB) return; // We can read v3 or v2 versions of hfile. throw new IllegalArgumentException("Invalid HFile version: major=" + trailer.getMajorVersion() + ", minor=" + trailer.getMinorVersion() + ": expected at least " + "major=2 and minor=" + MAX_MINOR_VERSION); }
int majorVersion = extractMajorVersion(version); int minorVersion = extractMinorVersion(version); int trailerSize = getTrailerSize(majorVersion); FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion); fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(), buf.arrayOffset() + bufferSize - trailerSize, trailerSize))); return fft;
FixedFileTrailer.readFromStream(fsdis, fileSize); assertEquals(3, trailer.getMajorVersion()); assertEquals(entryCount, trailer.getEntryCount()); HFileContext meta = new HFileContextBuilder() .withCompression(compressAlgo) CellComparator comparator = trailer.createComparator(); HFileBlockIndex.BlockIndexReader dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, trailer.getNumDataIndexLevels()); HFileBlockIndex.BlockIndexReader metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize()); blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount()); .getByteStream(), trailer.getMetaIndexCount()); while (curBlockPos <= trailer.getLastDataBlockOffset()) { HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader); while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) { LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " + trailer.getLoadOnOpenDataOffset()); HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false)
writeInlineBlocks(true); FixedFileTrailer trailer = new FixedFileTrailer(getMajorVersion(), getMinorVersion()); trailer.setLoadOnOpenOffset(rootIndexOffset); trailer.setNumDataIndexLevels(dataBlockIndexWriter.getNumLevels()); trailer.setUncompressedDataIndexSize( dataBlockIndexWriter.getTotalUncompressedSize()); trailer.setFirstDataBlockOffset(firstDataBlockOffset); trailer.setLastDataBlockOffset(lastDataBlockOffset); trailer.setComparatorClass(comparator.getClass()); trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
super(path, trailer, fsdis, fsdisNoFsChecksum, size, closeIStream, cacheConf, hfs); trailer.expectMajorVersion(2); validateMinorVersion(path, trailer.getMinorVersion()); HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fsdisNoFsChecksum, compressAlgo, fileSize, trailer.getMinorVersion(), hfs, path); this.fsBlockReader = fsBlockReaderV2; // upcast comparator = trailer.createComparator(); dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator, trailer.getNumDataIndexLevels(), this); metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader( Bytes.BYTES_RAWCOMPARATOR, 1); trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize()); trailer.getDataIndexCount()); trailer.getMetaIndexCount());
Configuration conf) throws IOException { this.trailer = trailer; this.compressAlgo = trailer.getCompressionCodec(); this.cacheConf = cacheConf; this.fileSize = fileSize; comparator = trailer.createComparator(); dataBlockIndexReader = new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator, trailer.getNumDataIndexLevels(), this); metaBlockIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize()); trailer.getDataIndexCount()); trailer.getMetaIndexCount());
long fileSize = fs.getFileStatus(file).getLen(); FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset(); HFileBlock block; while (offset <= max) {
istream.seek(trailer.getFileInfoOffset()); fileInfo = new FileInfo(); fileInfo.readFields(istream); new HFileBlockIndex.BlockIndexReader(Bytes.BYTES_RAWCOMPARATOR, 1); int sizeToLoadOnOpen = (int) (fileSize - trailer.getLoadOnOpenDataOffset() - trailer.getTrailerSize()); byte[] dataAndMetaIndex = readAllIndex(istream, trailer.getLoadOnOpenDataOffset(), sizeToLoadOnOpen); if (trailer.getDataIndexCount() > 0) BlockType.INDEX_V1.readAndCheck(dis); dataBlockIndexReader.readRootIndex(dis, trailer.getDataIndexCount()); if (trailer.getMetaIndexCount() > 0) BlockType.INDEX_V1.readAndCheck(dis); metaBlockIndexReader.readRootIndex(dis, trailer.getMetaIndexCount());
/** * Positions this scanner at the start of the file. * * @return false if empty file; i.e. a call to next would return false and * the current key and value are undefined. * @throws IOException */ @Override public boolean seekTo() throws IOException { if (reader == null) { return false; } if (reader.getTrailer().getEntryCount() == 0) { // No data blocks. return false; } long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset(); if (curBlock != null && curBlock.getOffset() == firstDataBlockOffset) { return processFirstDataBlock(); } readAndUpdateNewBlock(firstDataBlockOffset); return true; }
reader.getTrailer().getNumDataIndexLevels()); reader.getTrailer().getLoadOnOpenDataOffset()); HFileBlock block; List<byte[]> blockKeys = new ArrayList<>(); reader.getTrailer().getUncompressedDataIndexSize());
@Test public void testComparatorIsHBase1Compatible() { FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION); t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass()); assertEquals(CellComparatorImpl.COMPARATOR.getClass().getName(), t.getComparatorClassName()); HFileProtos.FileTrailerProto pb = t.toProtobuf(); assertEquals(KeyValue.COMPARATOR.getClass().getName(), pb.getComparatorClassName()); t.setComparatorClass(CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass()); pb = t.toProtobuf(); assertEquals(KeyValue.META_COMPARATOR.getClass().getName(), pb.getComparatorClassName()); }
/** @return number of KV entries in this HFile */ @Override public long getEntries() { return trailer.getEntryCount(); }
boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum(); assert !isHBaseChecksum; // Initially we must read with FS checksum. trailer = FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size); switch (trailer.getMajorVersion()) { case 2: LOG.debug("Opening HFile v2 with v3 reader"); primaryReplicaReader, conf); default: throw new IllegalArgumentException("Invalid HFile version " + trailer.getMajorVersion());
throw new IOException("Block index not loaded"); long trailerOffset = trailer.getLoadOnOpenDataOffset(); if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) { throw new IOException("Requested block is out of range: " + dataBlockOffset + ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset() + ", trailer.getLoadOnOpenDataOffset: " + trailerOffset);
long end = 0; try { end = getTrailer().getLoadOnOpenDataOffset(); if (LOG.isTraceEnabled()) { LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end));
FixedFileTrailer trailer = new FixedFileTrailer(1, HFileBlock.MINOR_VERSION_NO_CHECKSUM); trailer.setLoadOnOpenOffset(writeBlockIndex(this.outputStream, this.blockKeys, this.blockOffsets, this.blockDataSizes)); LOG.info("Wrote a version 1 block index with " + this.blockKeys.size() trailer.setDataIndexCount(blockKeys.size());