Path filePath = fStat.getPath(); if (HFileLink.isHFileLink(filePath)) return true; if (HFileLink.isBackReferencesDir(parentDir)) { Path hfilePath = null; try { hfilePath = HFileLink.getHFileFromBackReference( new Path(FSUtils.getRootDir(getConf()), HConstants.HBASE_TEMP_DIRECTORY), filePath); if (fs.exists(hfilePath)) { hfilePath = HFileLink.getHFileFromBackReference(MobUtils.getMobHome(getConf()), filePath); if (fs.exists(hfilePath)) { return false; hfilePath = HFileLink.getHFileFromBackReference(FSUtils.getRootDir(getConf()), filePath); return !fs.exists(hfilePath); } catch (IOException e) { backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); return FSUtils.listStatus(fs, backRefDir) == null; } catch (IOException e) {
/** * Returns the location where the inputPath will be copied. */ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException { Path path = null; switch (inputInfo.getType()) { case HFILE: Path inputPath = new Path(inputInfo.getHfile()); String family = inputPath.getParent().getName(); TableName table =HFileLink.getReferencedTableName(inputPath.getName()); String region = HFileLink.getReferencedRegionName(inputPath.getName()); String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); path = new Path(FSUtils.getTableDir(new Path("./"), table), new Path(region, new Path(family, hfile))); break; case WAL: LOG.warn("snapshot does not keeps WALs: " + inputInfo); break; default: throw new IOException("Invalid File Type: " + inputInfo.getType().toString()); } return new Path(outputArchive, path); }
refPath = StoreFileInfo.getReferredToFile(refPath); String refRegion = refPath.getParent().getParent().getName(); refPath = HFileLink.createPath(table, refRegion, family, refPath.getName()); if (!HFileLink.buildFromHFileLinkPattern(conf, refPath).exists(fs)) { throw new CorruptedSnapshotException( "Missing parent hfile for: " + fileName + " path=" + refPath, if (refPath != null && HFileLink.isHFileLink(refPath)) { linkPath = new Path(family, refPath.getName()); } else if (HFileLink.isHFileLink(fileName)) { linkPath = new Path(family, fileName); } else { linkPath = new Path(family, HFileLink.createHFileLinkName( table, regionInfo.getEncodedName(), fileName)); link = HFileLink.buildFromHFileLinkPattern(MobUtils.getQualifiedMobRootDir(conf), HFileArchiveUtil.getArchivePath(conf), linkPath); } else { link = HFileLink.buildFromHFileLinkPattern(conf, linkPath); FileStatus fstat = link.getFileStatus(fs); if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) { String msg = "hfile: " + fileName + " size does not match with the expected one. " + link.getOriginPath() + ") or archive (" + link.getArchivePath() + ") directory for the primary table."; LOG.error(msg);
/** * Create an HFileLink instance from table/region/family/hfile location * @param conf {@link Configuration} from which to extract specific archive locations * @param table Table name * @param region Region Name * @param family Family Name * @param hfile HFile Name * @return Link to the file with the specified table/region/family/hfile location * @throws IOException on unexpected error. */ public static HFileLink build(final Configuration conf, final TableName table, final String region, final String family, final String hfile) throws IOException { return HFileLink.buildFromHFileLinkPattern(conf, createPath(table, region, family, hfile)); }
/** * Create an HFileLink relative path for the table/region/family/hfile location * @param table Table name * @param region Region Name * @param family Family Name * @param hfile HFile Name * @return the relative Path to open the specified table/region/family/hfile link */ public static Path createPath(final TableName table, final String region, final String family, final String hfile) { if (HFileLink.isHFileLink(hfile)) { return new Path(family, hfile); } return new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String hfile = storeFile.getName(); if (HFileLink.isHFileLink(hfile)) { names.add(HFileLink.getReferencedHFileName(hfile)); } else { names.add(hfile); } } });
Assert.assertTrue("Failed validating:" + name, HFileLink.isHFileLink(name)); HFileLink.getReferencedTableName(testName)); Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); Assert.assertEquals(testName, HFileLink.createHFileLinkName(TableName.valueOf(name.getMethodName()), "fefefe", "0123456")); HFileLink.getReferencedTableName(testName)); Assert.assertEquals("fefefe", HFileLink.getReferencedRegionName(testName)); Assert.assertEquals("0123456", HFileLink.getReferencedHFileName(testName)); Assert.assertEquals(testName, HFileLink.createHFileLinkName(TableName.valueOf("ns", name.getMethodName()), "fefefe", "0123456")); Assert.assertEquals(HFileLink.getReferencedTableName(name), TableName.valueOf(m.group(1), m.group(2))); Assert.assertEquals(HFileLink.getReferencedRegionName(name), m.group(3)); Assert.assertEquals(HFileLink.getReferencedHFileName(name), m.group(4));
private FileLink getFileLink(Path path, Configuration conf) throws IOException{ String regionName = HFileLink.getReferencedRegionName(path.getName()); TableName tableName = HFileLink.getReferencedTableName(path.getName()); if(MobUtils.getMobRegionInfo(tableName).getEncodedName().equals(regionName)) { return HFileLink.buildFromHFileLinkPattern(MobUtils.getQualifiedMobRootDir(conf), HFileArchiveUtil.getArchivePath(conf), path); } return HFileLink.buildFromHFileLinkPattern(inputRoot, inputArchive, path); }
for (Path path : allFiles.values()) { HFileLink actualLink = HFileLink.buildFromHFileLinkPattern(conf, path); if (actualLink.exists(fs)) continue; // good, expected .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()), HFileLink.getReferencedRegionName(path.getName().toString()), path.getParent().getName()), HFileLink.getReferencedHFileName(path.getName().toString())); success = sidelineFile(fs, hbaseRoot, backRefPath);
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
.getNameAsString(); String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); String refName = createBackReferenceName(tableName, regionName); Path backRefPath = null; if (createBackRef) { Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); fs.mkdirs(backRefssDir);
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { HFileLink link = HFileLink.build(conf, snapshot.getTableName(), regionInfo.getEncodedName(), family, storeFile.getName()); long size; int c = 0; if (fs.exists(link.getArchivePath())) { p = link.getArchivePath(); al = uniqueHFilesArchiveSize; size = fs.getFileStatus(p).getLen(); } else if (fs.exists(link.getMobPath())) { p = link.getMobPath(); al = uniqueHFilesMobSize; size = fs.getFileStatus(p).getLen(); } else { p = link.getOriginPath(); al = uniqueHFilesSize; size = link.getFileStatus(fs).getLen();
if (HFileLink.isHFileLink(path) || StoreFileInfo.isHFile(path)) { HFileLink link = HFileLink .build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } else if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); Path referencePath = StoreFileInfo.getReferredToFile(path); if (HFileLink.isHFileLink(referencePath)) { HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, referencePath); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference, link); } else { .build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference);
/** * Create a new {@link HFileLink} to reference the store file. * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference. * <ul> * <li>hfile: abc -> table=region-abc * <li>reference: abc.1234 -> table=region-abc.1234 * <li>hfilelink: table=region-hfile -> table=region-hfile * </ul> * @param familyDir destination directory for the store file * @param regionInfo destination region info for the table * @param createBackRef - Whether back reference should be created. Defaults to true. * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); } else if (StoreFileInfo.isReference(hfileName)) { restoreReferenceFile(familyDir, regionInfo, storeFile); } else { HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); } }
@Test public void testMobFilePath() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); Path rootDir = FSUtils.getRootDir(conf); Path tableDir = FSUtils.getTableDir(rootDir, tableName); Path archiveDir = FSUtils.getTableDir(HFileArchiveUtil.getArchivePath(conf), tableName); String fileName = "mobFile"; String encodedRegionName = MobUtils.getMobRegionInfo(tableName).getEncodedName(); String columnFamily = "columnFamily"; Path regionDir = new Path(tableDir, encodedRegionName); Path archivedRegionDir = new Path(archiveDir, encodedRegionName); Path expectedMobFilePath = new Path(MobUtils.getMobFamilyPath(conf, tableName, columnFamily), fileName).makeQualified(fs.getUri(), fs.getWorkingDirectory()); Path expectedOriginPath = new Path(new Path(regionDir, columnFamily), fileName).makeQualified( fs.getUri(), fs.getWorkingDirectory()); Path expectedArchivePath = new Path(new Path(archivedRegionDir, columnFamily), fileName) .makeQualified(fs.getUri(), fs.getWorkingDirectory()); String hfileLinkName = tableName.getNameAsString() + "=" + encodedRegionName + "-" + fileName; Path hfileLinkPath = new Path(columnFamily, hfileLinkName); HFileLink hfileLink = HFileLink.buildFromHFileLinkPattern(conf, hfileLinkPath); Assert.assertEquals(expectedMobFilePath, hfileLink.getMobPath()); Assert.assertEquals(expectedOriginPath, hfileLink.getOriginPath()); Assert.assertEquals(expectedArchivePath, hfileLink.getArchivePath()); } }
if (!HFileLink.isHFileLink(fileName)) { refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName); linkPath = new Path(familyDir, HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName)); InputStream in; if (linkPath != null) { in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs); } else { linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
public void storeFile (final String region, final String family, final String hfile) throws IOException { Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); HFileLink link = new HFileLink(conf, path); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.addAndGet(1); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.addAndGet(1); } } catch (FileNotFoundException e) { hfilesMissing.addAndGet(1); } if (showFiles) { System.out.printf("%8s %s/%s/%s/%s %s%n", (size < 0 ? "-" : StringUtils.humanReadableInt(size)), table, region, family, hfile, (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : "")); } }
HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
if (!HFileLink.isHFileLink(fileName)) { refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName); HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName)); InputStream in = new HFileLink(conf, linkPath).open(fs); OutputStream out = fs.create(outPath); IOUtils.copyBytes(in, out, conf);
@Override public boolean accept(Path p) { return HFileLink.isHFileLink(p); } }