/** * Creates a half file reader for a normal hfile. * @param fs fileystem to read from * @param p path to hfile * @param cacheConf * @param r original reference file (contains top or bottom) * @param conf Configuration * @throws IOException */ public HalfStoreFileReader(FileSystem fs, Path p, CacheConfig cacheConf, Reference r, boolean isPrimaryReplicaStoreFile, AtomicInteger refCount, boolean shared, Configuration conf) throws IOException { super(fs, p, cacheConf, isPrimaryReplicaStoreFile, refCount, shared, conf); // This is not actual midkey for this half-file; its just border // around which we split top and bottom. Have to look in files to find // actual last and first keys for bottom and top halves. Half-files don't // have an actual midkey themselves. No midkey is how we indicate file is // not splittable. this.splitkey = r.getSplitKey(); this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length); // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); }
/** * @param splitRow * @return A {@link Reference} that points at top half of a an hfile */ public static Reference createTopReference(final byte [] splitRow) { return new Reference(splitRow, Range.top); }
top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow); return r.write(fs, p);
/** @return True if the store file is a top Reference */ public boolean isTopReference() { return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion()); }
/** * Split a storefile into a top and bottom half, maintaining the metadata, recreating bloom * filters, etc. */ @VisibleForTesting static void splitStoreFile(Configuration conf, Path inFile, ColumnFamilyDescriptor familyDesc, byte[] splitKey, Path bottomOut, Path topOut) throws IOException { // Open reader with no block cache, and not in-memory Reference topReference = Reference.createTopReference(splitKey); Reference bottomReference = Reference.createBottomReference(splitKey); copyHFileHalf(conf, inFile, topOut, topReference, familyDesc); copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc); }
/** * Write out a merge reference under the given merges directory. Package local * so it doesnt leak out of regionserver. * @param mergedRegion {@link RegionInfo} of the merged region * @param familyName Column Family Name * @param f File to create reference. * @param mergedDir * @return Path to created reference. * @throws IOException */ public Path mergeStoreFile(RegionInfo mergedRegion, String familyName, HStoreFile f, Path mergedDir) throws IOException { Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName); // A whole reference to the store file. Reference r = Reference.createTopReference(regionInfoForFs.getStartKey()); // Add the referred-to regions name as a dot separated suffix. // See REF_NAME_REGEX regex above. The referred-to regions name is // up in the path of the passed in <code>f</code> -- parentdir is family, // then the directory above is the region name. String mergingRegionName = regionInfoForFs.getEncodedName(); // Write reference with same file id only with the other region name as // suffix and into the new region location (under same family). Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName); return r.write(fs, p); }
if (LOG.isTraceEnabled()) LOG.trace(p + " is a link"); } else if (isReference(p)) { this.reference = Reference.read(fs, p); Path referencePath = getReferredToFile(p); if (HFileLink.isHFileLink(referencePath)) { if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() + " reference to " + referencePath); } else if (isHFile(p)) {
if (ProtobufUtil.isPBMagicPrefix(pbuf)) return convert(FSProtos.Reference.parseFrom(in)); Reference r = new Reference(); DataInputStream dis = new DataInputStream(in); r.readFields(dis); return r; } finally {
/** * Write out a split reference. Package local so it doesnt leak out of * regionserver. * @param fs * @param splitDir Presumes path format is actually * <code>SOME_DIRECTORY/REGIONNAME/FAMILY</code>. * @param f File to split. * @param splitRow * @param range * @return Path to created reference. * @throws IOException */ static Path split(final FileSystem fs, final Path splitDir, final StoreFile f, final byte [] splitRow, final Reference.Range range) throws IOException { // A reference to the bottom half of the hsf store file. Reference r = new Reference(splitRow, range); // Add the referred-to regions name as a dot separated suffix. // See REF_NAME_REGEX regex above. The referred-to regions name is // up in the path of the passed in <code>f</code> -- parentdir is family, // then the directory above is the region name. String parentRegionName = f.getPath().getParent().getParent().getName(); // Write reference with same file id only with the other region name as // suffix and into the new region location (under same family). Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName); return r.write(fs, p); }
/** * Read a Reference from FileSystem. * @param fs * @param p * @return New Reference made from passed <code>p</code> * @throws IOException */ public static Reference read(final FileSystem fs, final Path p) throws IOException { FSDataInputStream in = fs.open(p); try { Reference r = new Reference(); r.readFields(in); return r; } finally { in.close(); } } }
@Override public void storeFile(final HRegionFileSystem region, final Path familyDir, final StoreFileInfo storeFile) throws IOException { Path referenceFile = new Path(familyDir, storeFile.getPath().getName()); boolean success = true; if (storeFile.isReference()) { // write the Reference object to the snapshot storeFile.getReference().write(workingDirFs, referenceFile); } else { // create "reference" to this store file. It is intentionally an empty file -- all // necessary information is captured by its fs location and filename. This allows us to // only figure out what needs to be done via a single nn operation (instead of having to // open and read the files as well). success = workingDirFs.createNewFile(referenceFile); } if (!success) { throw new IOException("Failed to create reference file:" + referenceFile); } } }
public void write(DataOutput out) throws IOException { // Write true if we're doing top of the file. out.writeBoolean(isTopFileRegion(this.region)); Bytes.writeByteArray(out, this.splitkey); }
return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } else if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); Path referencePath = StoreFileInfo.getReferredToFile(path); if (HFileLink.isHFileLink(referencePath)) {
/** * Use this when writing to a stream and you want to use the pb mergeDelimitedFrom * (w/o the delimiter, pb reads to EOF which may not be what you want). * @return This instance serialized as a delimited protobuf w/ a magic pb prefix. * @throws IOException */ byte [] toByteArray() throws IOException { return ProtobufUtil.prependPBMagic(convert().toByteArray()); }
/** * helper function to compute HDFS blocks distribution of a given reference * file.For reference file, we don't compute the exact value. We use some * estimate instead given it might be good enough. we assume bottom part * takes the first half of reference file, top part takes the second half * of the reference file. This is just estimate, given * midkey ofregion != midkey of HFile, also the number and size of keys vary. * If this estimate isn't good enough, we can improve it later. * @param fs The FileSystem * @param reference The reference * @param status The reference FileStatus * @return HDFS blocks distribution */ private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution( final FileSystem fs, final Reference reference, final FileStatus status) throws IOException { if (status == null) { return null; } long start = 0; long length = 0; if (Reference.isTopFileRegion(reference.getFileRegion())) { start = status.getLen()/2; length = status.getLen() - status.getLen()/2; } else { start = 0; length = status.getLen()/2; } return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length); }
Path parentdir = new Path(tabledir, parent.getEncodedName()); Path storedir = HStore.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName()); Reference ref = Reference.createTopReference(Bytes.toBytes("ccc")); long now = System.currentTimeMillis(); Path path = ref.write(fs, p); assertTrue(fs.exists(path)); LOG.info("Created reference " + path);
LOG.debug("Store file " + p + " is a link"); } else if (isReference(p)) { this.reference = Reference.read(fs, p); this.referencePath = getReferredToFile(this.path); if (HFileLink.isHFileLink(this.referencePath)) { this.link = new HFileLink(conf, this.referencePath); LOG.debug("Store file " + p + " is a " + reference.getFileRegion() + " reference to " + this.referencePath); } else if (!isHFile(p)) {
if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen); if (ProtobufUtil.isPBMagicPrefix(pbuf)) return convert(FSProtos.Reference.parseFrom(in)); Reference r = new Reference(); DataInputStream dis = new DataInputStream(in); r.readFields(dis); return r; } finally {
/** * Split a storefile into a top and bottom half, maintaining * the metadata, recreating bloom filters, etc. */ static void splitStoreFile( Configuration conf, Path inFile, HColumnDescriptor familyDesc, byte[] splitKey, Path bottomOut, Path topOut) throws IOException { // Open reader with no block cache, and not in-memory Reference topReference = Reference.createTopReference(splitKey); Reference bottomReference = Reference.createBottomReference(splitKey); copyHFileHalf(conf, inFile, topOut, topReference, familyDesc); copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc); }