return link.getFileStatus(fs); } catch (FileNotFoundException ex) { return link.getFileStatus(fs); } catch (FileNotFoundException ex) {
p = link.getOriginPath(); al = uniqueHFilesSize; size = link.getFileStatus(fs).getLen();
hfilesMobCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfilesSize.addAndGet(size); hfilesCount.incrementAndGet();
.build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } else if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference, link); } else { .build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference);
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
FileStatus fstat = link.getFileStatus(fs); if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) { String msg = "hfile: " + fileName + " size does not match with the expected one. " +
status = this.link.getFileStatus(fs); } else if (this.reference != null) {
public void storeFile (final String region, final String family, final String hfile) throws IOException { Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); long size = new HFileLink(conf, path).getFileStatus(fs).getLen(); files.add(new Pair<Path, Long>(path, size)); }
/** * compute HDFS block distribution, for reference file, it is an estimate */ private void computeHDFSBlockDistribution() throws IOException { if (isReference()) { FileStatus status; if (this.link != null) { status = this.link.getFileStatus(fs); } else { status = fs.getFileStatus(this.referencePath); } this.hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution( this.fs, this.reference, status); } else { FileStatus status; if (isLink()) { status = link.getFileStatus(fs); } else { status = this.fs.getFileStatus(path); } long length = status.getLen(); this.hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution( this.fs, status, 0, length); } }
private FileStatus getFileStatus(final FileSystem fs, final Path path) { try { if (HFileLink.isHFileLink(path)) { HFileLink link = new HFileLink(inputRoot, inputArchive, path); return link.getFileStatus(fs); } else if (isHLogLinkPath(path)) { String serverName = path.getParent().getName(); String logName = path.getName(); return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs); } return fs.getFileStatus(path); } catch (IOException e) { LOG.warn("Unable to get the status for file=" + path); return null; } }
public void storeFile (final String region, final String family, final String hfile) throws IOException { Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); HFileLink link = new HFileLink(conf, path); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.addAndGet(1); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.addAndGet(1); } } catch (FileNotFoundException e) { hfilesMissing.addAndGet(1); } if (showFiles) { System.out.printf("%8s %s/%s/%s/%s %s%n", (size < 0 ? "-" : StringUtils.humanReadableInt(size)), table, region, family, hfile, (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : "")); } }
/** * Creates a half file reader for a hfile referred to by an hfilelink. * @param fs fileystem to read from * @param p path to hfile * @param link * @param cacheConf * @param r original reference file (contains top or bottom) * @param preferredEncodingInCache * @throws IOException */ public HalfStoreFileReader(final FileSystem fs, final Path p, final HFileLink link, final CacheConfig cacheConf, final Reference r, DataBlockEncoding preferredEncodingInCache) throws IOException { super(fs, p, link, link.getFileStatus(fs).getLen(), cacheConf, preferredEncodingInCache, true); // This is not actual midkey for this half-file; its just border // around which we split top and bottom. Have to look in files to find // actual last and first keys for bottom and top halves. Half-files don't // have an actual midkey themselves. No midkey is how we indicate file is // not splittable. this.splitkey = r.getSplitKey(); // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); }
return link.getFileStatus(fs); } catch (FileNotFoundException ex) { return link.getFileStatus(fs); } catch (FileNotFoundException ex) {
hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet();
/** * Returns a StoreFileInfo from the given FileStatus. Secondary replicas refer to the * files of the primary region, so an HFileLink is used to construct the StoreFileInfo. This * way ensures that the secondary will be able to continue reading the store files even if * they are moved to archive after compaction * @throws IOException */ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, HRegionInfo regionInfo, HRegionInfo regionInfoForFs, String familyName, Path path) throws IOException { // if this is a primary region, just return the StoreFileInfo constructed from path if (regionInfo.equals(regionInfoForFs)) { return new StoreFileInfo(conf, fs, path); } // else create a store file link. The link file does not exists on filesystem though. HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); } return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
@Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (storeFile.hasReference()) { // copied as part of the manifest } else { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size)); } }
FileStatus fstat = link.getFileStatus(fs); if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) { String msg = "hfile: " + fileName + " size does not match with the expected one. " +
status = this.link.getFileStatus(fs); } else if (this.reference != null) {