@VisibleForTesting public static void createDirectory(MasterFileSystem mfs, NamespaceDescriptor nsDescriptor) throws IOException { mfs.getFileSystem().mkdirs(FSUtils.getNamespaceDir(mfs.getRootDir(), nsDescriptor.getName())); }
final boolean archive) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final FileSystem fs = mfs.getFileSystem(); final Path tempdir = mfs.getTempDir(); final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); final Path tempTableDir = FSUtils.getTableDir(tempdir, tableName); continue; HFileArchiver.archiveRegion(fs, mfs.getRootDir(), tempTableDir, files[i].getPath()); for (RegionInfo hri : regions) { LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS"); HFileArchiver.archiveRegion(fs, mfs.getRootDir(), tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName())); Path mobTableDir = FSUtils.getTableDir(new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME), tableName); Path regionDir = new Path(mobTableDir, MobUtils.getMobRegionInfo(tableName).getEncodedName()); if (fs.exists(regionDir)) { HFileArchiver.archiveRegion(fs, mfs.getRootDir(), mobTableDir, regionDir);
public SplitWALManager(MasterServices master) { this.master = master; this.conf = master.getConfiguration(); this.splitWorkerAssigner = new SplitWorkerAssigner(this.master, conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER)); this.rootDir = master.getMasterFileSystem().getWALRootDir(); this.fs = master.getMasterFileSystem().getFileSystem(); }
@VisibleForTesting protected void createDirForRemoteWAL(MasterProcedureEnv env) throws IOException { MasterFileSystem mfs = env.getMasterFileSystem(); Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId); FileSystem walFs = mfs.getWALFileSystem(); if (walFs.exists(remoteWALDirForPeer)) { LOG.warn("Wal dir {} already exists, usually this should not happen, continue anyway", remoteWALDirForPeer); } else if (!walFs.mkdirs(remoteWALDirForPeer)) { throw new IOException("Failed to create remote wal dir " + remoteWALDirForPeer); } }
/** * Remove the column family from the file system **/ public static void deleteColumnFamilyFromFileSystem( final MasterProcedureEnv env, final TableName tableName, final List<RegionInfo> regionInfoList, final byte[] familyName, final boolean hasMob) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); if (LOG.isDebugEnabled()) { LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName); } for (RegionInfo hri : regionInfoList) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (hasMob) { // Delete the mob region Path mobRootDir = new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME); RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName); mfs.deleteFamilyFromFS(mobRootDir, mobRegionInfo, familyName); } } }
/** * @return the directory for a give {@code region}. */ public Path getRegionDir(RegionInfo region) { return FSUtils.getRegionDir(FSUtils.getTableDir(getRootDir(), region.getTable()), region); }
this.masterActiveTime = System.currentTimeMillis(); this.fileSystemManager = new MasterFileSystem(this, this, metrics, masterRecovery); new FSTableDescriptors(this.fileSystemManager.getFileSystem(), this.fileSystemManager.getRootDir()); ClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId()); if (!masterRecovery) { this.executorService = new ExecutorService(getServerName().toString());
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); assertEquals(masterRoot, rootDir); assertEquals(FSUtils.getWALRootDir(UTIL.getConfiguration()), fs.getWALRootDir()); } }
/** * Check to see if the snapshot is one of the currently completed snapshots * Returns true if the snapshot exists in the "completed snapshots folder". * * @param snapshot expected snapshot to check * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is * not stored * @throws IOException if the filesystem throws an unexpected exception, * @throws IllegalArgumentException if snapshot name is invalid. */ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException { try { final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); FileSystem fs = master.getMasterFileSystem().getFileSystem(); // check to see if the snapshot already exists return fs.exists(snapshotDir); } catch (IllegalArgumentException iae) { throw new UnknownSnapshotException("Unexpected exception thrown", iae); } }
protected final Path getRemoteWALDir(MasterFileSystem mfs, String peerId) { Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); return getRemoteWALDir(remoteWALDir, peerId); }
/** * Create region layout in file system. * @param env MasterProcedureEnv * @throws IOException */ private List<RegionInfo> createFsLayout( final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory(tempTableDir, TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions( env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); // Move Table temp mob directory to the hbase root location Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName()); if (mfs.getFileSystem().exists(tempMobTableDir)) { moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir); } return newRegions; }
}; checkRootDir(this.rootdir, conf, this.fs); checkTempDir(this.tempdir, conf, this.fs); for (String subDir : protectedSubDirs) { checkSubDir(new Path(this.rootdir, subDir), HBASE_DIR_PERMS); checkSubDir(new Path(this.walRootDir, subDir), perms); checkStagingDir();
protected static List<RegionInfo> createFsLayout(final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory( tempTableDir, tableDescriptor, false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
/** * Create initial layout in filesystem. * <ol> * <li>Check if the meta region exists and is readable, if not create it. * Create hbase.version and the hbase:meta directory if not one. * </li> * <li>Create a log archive directory for RS to put archived logs</li> * </ol> * Idempotent. */ private Path createInitialFileSystemLayout() throws IOException { // check if the root directory exists checkRootDir(this.rootdir, conf, this.fs); // check if temp directory exists and clean it checkTempDir(this.tempdir, conf, this.fs); Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME); // Make sure the region servers can archive their old logs if(!this.fs.exists(oldLogDir)) { this.fs.mkdirs(oldLogDir); } return oldLogDir; }
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); // make sure the fs and the found root dir have the same scheme LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf())); LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.getConfiguration())); // make sure the set uri matches by forcing it. assertEquals(masterRoot, rootDir); } }
this.fileSystemManager = new MasterFileSystem(this, this); ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId()); this.serverManager = createServerManager(this, this); this.fileSystemManager.getFailedServersFromLogFolders();
initializeMemStoreChunkCreator(); this.fileSystemManager = new MasterFileSystem(conf); this.walManager = new MasterWalManager(this); ClusterId clusterId = fileSystemManager.getClusterId(); status.setStatus("Publishing Cluster ID " + clusterId + " in ZooKeeper"); ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId()); this.clusterId = clusterId.toString();
public MasterFileSystem(Configuration conf) throws IOException { this.conf = conf; // Set filesystem to be that of this.rootdir else we get complaints about // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is // default localfs. Presumption is that rootdir is fully-qualified before // we get to here with appropriate fs scheme. this.rootdir = FSUtils.getRootDir(conf); this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY); // Cover both bases, the old way of setting default fs and the new. // We're supposed to run on 0.20 and 0.21 anyways. this.fs = this.rootdir.getFileSystem(conf); this.walRootDir = FSUtils.getWALRootDir(conf); this.walFs = FSUtils.getWALFileSystem(conf); FSUtils.setFsDefault(conf, new Path(this.walFs.getUri())); walFs.setConf(conf); FSUtils.setFsDefault(conf, new Path(this.fs.getUri())); // make sure the fs has the same conf fs.setConf(conf); this.secureRootSubDirPerms = new FsPermission(conf.get("hbase.rootdir.perms", "700")); this.isSecurityEnabled = "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication")); // setup the filesystem variable createInitialFileSystemLayout(); HFileSystem.addLocationsOrderInterceptor(conf); }
public void deleteFamilyFromFS(RegionInfo region, byte[] familyName) throws IOException { deleteFamilyFromFS(rootdir, region, familyName); }
super(conf); Superusers.initialize(conf); this.fileSystemManager = new MasterFileSystem(conf); this.walManager = new MasterWalManager(this);