private void verifySnapshot(final Configuration baseConf, final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException { // Update the conf with the current root dir, since may be a different cluster Configuration conf = new Configuration(baseConf); FSUtils.setRootDir(conf, rootDir); FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf)); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc); }
@VisibleForTesting HRegionFileSystem getFileSystem(Connection connection) throws IOException { Admin admin = connection.getAdmin(); return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(), FSUtils.getCurrentFileSystem(admin.getConfiguration()), FSUtils.getTableDir(FSUtils.getRootDir(admin.getConfiguration()), region.getTable()), region, true); }
/** * @return The set of files in the specified family directory. */ private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException { FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir); if (hfiles == null) return Collections.emptySet(); Set<String> familyFiles = new HashSet<>(hfiles.length); for (int i = 0; i < hfiles.length; ++i) { String hfileName = hfiles[i].getPath().getName(); familyFiles.add(hfileName); } return familyFiles; }
/** * Construct a FSTableDescriptors instance using the hbase root dir of the given * conf and the filesystem where that root dir lives. * This instance can do write operations (is not read only). */ public FSTableDescriptors(final Configuration conf) throws IOException { this(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf)); }
private static void recoverLease(final Configuration conf, final Path path) { try { final FileSystem dfs = FSUtils.getCurrentFileSystem(conf); FSUtils fsUtils = FSUtils.getInstance(dfs, conf); fsUtils.recoverFileLease(dfs, path, conf, new CancelableProgressable() { @Override public boolean progress() { LOG.debug("Still trying to recover WAL lease: " + path); return true; } }); } catch (IOException e) { LOG.warn("unable to recover lease for WAL: " + path, e); } }
public static int getRegionReferenceFileCount(final FileSystem fs, final Path p) { int result = 0; try { for (Path familyDir:getFamilyDirs(fs, p)){ result += getReferenceFilePaths(fs, familyDir).size(); } } catch (IOException e) { LOG.warn("Error Counting reference files.", e); } return result; }
private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException { MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Set<String> families = new HashSet<>(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); for (Path regionDir : FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) { for (Path familyDir : FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) { families.add(familyDir.getName()); } } return families; }
public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); Path hbasedir = FSUtils.getRootDir(conf); URI defaultFs = hbasedir.getFileSystem(conf).getUri(); FSUtils.setFsDefault(conf, new Path(defaultFs)); int ret = ToolRunner.run(conf, new RestoreDriver(), args); System.exit(ret); }
Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration()); for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) { for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) { for (FileStatus status : fs.listStatus(storeDir)) { System.out.println(status.getPath());
protected void setupConfiguration(Configuration conf) throws Exception { FSUtils.setRootDir(conf, util.getDataTestDir()); conf.setBoolean(WALProcedureStore.USE_HSYNC_CONF_KEY, false); conf.setInt(WALProcedureStore.SYNC_WAIT_MSEC_CONF_KEY, 10); conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, PROC_NTHREADS); conf.setInt(RSProcedureDispatcher.RS_RPC_STARTUP_WAIT_TIME_CONF_KEY, 1000); conf.setInt(AssignmentManager.ASSIGN_MAX_ATTEMPTS, getAssignMaxAttempts()); }
/** * We don't have an easy way to verify that a flush completed, so we loop until we find a * legitimate hfile and return it. * @param fs * @param table * @return Path of a flushed hfile. * @throws IOException */ Path getFlushedHFile(FileSystem fs, TableName table) throws IOException { Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table); Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0); Path famDir = new Path(regionDir, FAM_STR); // keep doing this until we get a legit hfile while (true) { FileStatus[] hfFss = fs.listStatus(famDir); if (hfFss.length == 0) { continue; } for (FileStatus hfs : hfFss) { if (!hfs.isDirectory()) { return hfs.getPath(); } } } }
private void restoreSnapshotForPeerCluster(Configuration conf, String peerQuorumAddress) throws IOException { Configuration peerConf = HBaseConfiguration.createClusterConf(conf, peerQuorumAddress, PEER_CONFIG_PREFIX); FileSystem.setDefaultUri(peerConf, peerFSAddress); FSUtils.setRootDir(peerConf, new Path(peerFSAddress, peerHBaseRootAddress)); FileSystem fs = FileSystem.get(peerConf); RestoreSnapshotHelper.copySnapshotForScanner(peerConf, fs, FSUtils.getRootDir(peerConf), new Path(peerFSAddress, peerSnapshotTmpDir), peerSnapshotName); }
/** * Get the full path to the archive directory on the configured * {@link org.apache.hadoop.hbase.master.MasterFileSystem} * @param conf to look for archive directory name and root directory. Cannot be null. Notes for * testing: requires a FileSystem root directory to be specified. * @return the full {@link Path} to the archive directory, as defined by the configuration * @throws IOException if an unexpected error occurs */ public static Path getArchivePath(Configuration conf) throws IOException { return getArchivePath(FSUtils.getRootDir(conf)); }
@Override public void editRegion(final RegionInfo regionInfo) throws IOException { snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo); } });
final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); assertTrue(fs.exists(tableDir)); FSUtils.logFileSystemState(fs, tableDir, LOG); List<Path> unwantedRegionDirs = FSUtils.getRegionDirs(fs, tableDir); for (int i = 0; i < regions.length; ++i) { Path regionDir = new Path(tableDir, regions[i].getEncodedName()); assertTrue(regions[i] + " region dir does not exist", fs.exists(regionDir)); assertTrue(unwantedRegionDirs.remove(regionDir)); List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir); for (int j = 0; j < family.length; ++j) { final Path familyDir = new Path(regionDir, family[j]);
public MasterFileSystem(Configuration conf) throws IOException { this.conf = conf; // Set filesystem to be that of this.rootdir else we get complaints about // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is // default localfs. Presumption is that rootdir is fully-qualified before // we get to here with appropriate fs scheme. this.rootdir = FSUtils.getRootDir(conf); this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY); // Cover both bases, the old way of setting default fs and the new. // We're supposed to run on 0.20 and 0.21 anyways. this.fs = this.rootdir.getFileSystem(conf); this.walRootDir = FSUtils.getWALRootDir(conf); this.walFs = FSUtils.getWALFileSystem(conf); FSUtils.setFsDefault(conf, new Path(this.walFs.getUri())); walFs.setConf(conf); FSUtils.setFsDefault(conf, new Path(this.fs.getUri())); // make sure the fs has the same conf fs.setConf(conf); this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700")); this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")); // setup the filesystem variable createInitialFileSystemLayout(); HFileSystem.addLocationsOrderInterceptor(conf); }
Path rootDir = FSUtils.getRootDir(getConf()); FileSystem fs = rootDir.getFileSystem(getConf()); List<Path> paths = FSUtils.getTableDirs(fs, rootDir); for (Path path : paths) { TableName tableName = FSUtils.getTableName(path); if ((!checkMetaOnly && isTableIncluded(tableName)) || + " file."); setShouldRerun(); FSUtils.setVersion(fs, rootDir, getConf().getInt( HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt( HConstants.VERSION_FILE_WRITE_ATTEMPTS,
/** * Clean up directories with prefix "exportSnapshot-", which are generated when exporting * snapshots. * @throws IOException exception */ protected static void cleanupExportSnapshotLog(Configuration conf) throws IOException { FileSystem fs = FSUtils.getCurrentFileSystem(conf); Path stagingDir = new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory() .toString())); FileStatus[] files = FSUtils.listStatus(fs, stagingDir); if (files == null) { return; } for (FileStatus file : files) { if (file.getPath().getName().startsWith("exportSnapshot-")) { LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName()); if (FSUtils.delete(fs, file.getPath(), true) == false) { LOG.warn("Can not delete " + file.getPath()); } } } }
public void deleteTableDir(TableName table) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); Path p = FSUtils.getTableDir(rootDir, table); HBaseFsck.debugLsr(conf, p); boolean success = fs.delete(p, true); LOG.info("Deleted " + p + " sucessfully? " + success); }
throws IOException { FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS, HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS)); if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt( HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) { FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); clusterId = FSUtils.getClusterId(fs, rd); if (!FSUtils.metaRegionExists(fs, rd)) { bootstrap(rd, c);