public MasterFileSystem getMasterFileSystem() { return master.getMasterFileSystem(); }
public MasterWalManager(MasterServices services) throws IOException { this(services.getConfiguration(), services.getMasterFileSystem().getWALFileSystem(), services.getMasterFileSystem().getWALRootDir(), services); }
public SplitWALManager(MasterServices master) { this.master = master; this.conf = master.getConfiguration(); this.splitWorkerAssigner = new SplitWorkerAssigner(this.master, conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER)); this.rootDir = master.getMasterFileSystem().getWALRootDir(); this.fs = master.getMasterFileSystem().getFileSystem(); }
/** * Create the namespace directory * @param env MasterProcedureEnv * @param nsDescriptor NamespaceDescriptor */ protected static void createDirectory(MasterProcedureEnv env, NamespaceDescriptor nsDescriptor) throws IOException { createDirectory(env.getMasterServices().getMasterFileSystem(), nsDescriptor); }
public SyncReplicationReplayWALManager(MasterServices services) throws IOException, ReplicationException { this.serverManager = services.getServerManager(); this.fs = services.getMasterFileSystem().getWALFileSystem(); this.walRootDir = services.getMasterFileSystem().getWALRootDir(); this.remoteWALDir = new Path(this.walRootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME); serverManager.registerListener(new ServerListener() { @Override public void serverAdded(ServerName serverName) { MasterProcedureScheduler scheduler = services.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler(); for (UsedReplayWorkersForPeer usedWorkers : usedWorkersByPeer.values()) { synchronized (usedWorkers) { usedWorkers.wake(scheduler); } } } }); }
protected static void moveTempDirectoryToHBaseRoot( final MasterProcedureEnv env, final TableDescriptor tableDescriptor, final Path tempTableDir) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); FileSystem fs = mfs.getFileSystem(); if (!fs.delete(tableDir, true) && fs.exists(tableDir)) { throw new IOException("Couldn't delete " + tableDir); } if (!fs.rename(tempTableDir, tableDir)) { throw new IOException("Unable to move table from temp=" + tempTableDir + " to hbase root=" + tableDir); } }
/** * Delete the namespace directories from the file system * @param env MasterProcedureEnv * @param namespaceName name of the namespace in string format */ private static void deleteDirectory(MasterProcedureEnv env, String namespaceName) throws IOException { MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); FileSystem fs = mfs.getFileSystem(); Path p = FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName); try { for (FileStatus status : fs.listStatus(p)) { if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) { throw new IOException("Namespace directory contains table dir: " + status.getPath()); } } if (!fs.delete(FSUtils.getNamespaceDir(mfs.getRootDir(), namespaceName), true)) { throw new IOException("Failed to remove namespace: " + namespaceName); } } catch (FileNotFoundException e) { // File already deleted, continue LOG.debug("deleteDirectory throws exception: " + e); } }
/** * Fully specify all necessary components of a snapshot manager. Exposed for testing. * @param master services for the master where the manager is running * @param coordinator procedure coordinator instance. exposed for testing. * @param pool HBase ExecutorServcie instance, exposed for testing. */ @VisibleForTesting SnapshotManager(final MasterServices master, ProcedureCoordinator coordinator, ExecutorService pool, int sentinelCleanInterval) throws IOException, UnsupportedOperationException { this.master = master; this.rootDir = master.getMasterFileSystem().getRootDir(); Configuration conf = master.getConfiguration(); checkSnapshotSupport(conf, master.getMasterFileSystem()); this.coordinator = coordinator; this.executorService = pool; resetTempDir(); snapshotHandlerChoreCleanerTask = this.scheduleThreadPool.scheduleAtFixedRate( this::cleanupSentinels, sentinelCleanInterval, sentinelCleanInterval, TimeUnit.SECONDS); }
/** * Check to see if the snapshot is one of the currently completed snapshots * Returns true if the snapshot exists in the "completed snapshots folder". * * @param snapshot expected snapshot to check * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is * not stored * @throws IOException if the filesystem throws an unexpected exception, * @throws IllegalArgumentException if snapshot name is invalid. */ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException { try { final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); FileSystem fs = master.getMasterFileSystem().getFileSystem(); // check to see if the snapshot already exists return fs.exists(snapshotDir); } catch (IllegalArgumentException iae) { throw new UnknownSnapshotException("Unexpected exception thrown", iae); } }
/** * Remove the column family from the file system **/ public static void deleteColumnFamilyFromFileSystem( final MasterProcedureEnv env, final TableName tableName, final List<RegionInfo> regionInfoList, final byte[] familyName, final boolean hasMob) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); if (LOG.isDebugEnabled()) { LOG.debug("Removing family=" + Bytes.toString(familyName) + " from table=" + tableName); } for (RegionInfo hri : regionInfoList) { // Delete the family directory in FS for all the regions one by one mfs.deleteFamilyFromFS(hri, familyName); } if (hasMob) { // Delete the mob region Path mobRootDir = new Path(mfs.getRootDir(), MobConstants.MOB_DIR_NAME); RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName); mfs.deleteFamilyFromFS(mobRootDir, mobRegionInfo, familyName); } } }
private SnapshotManager getNewManager(Configuration conf, int intervalSeconds) throws IOException, KeeperException { Mockito.reset(services); Mockito.when(services.getConfiguration()).thenReturn(conf); Mockito.when(services.getMasterFileSystem()).thenReturn(mfs); Mockito.when(mfs.getFileSystem()).thenReturn(fs); Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir()); return new SnapshotManager(services, coordinator, pool, intervalSeconds); }
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) throws IOException { // get the existing store files FileSystem fs = services.getMasterFileSystem().getFileSystem(); fs.mkdirs(storedir); // create the store files in the parent for (int i = 0; i < count; i++) { Path storeFile = new Path(storedir, "_store" + i); FSDataOutputStream dos = fs.create(storeFile, true); dos.writeBytes("Some data: " + i); dos.close(); } LOG.debug("Adding " + count + " store files to the storedir:" + storedir); // make sure the mock store files are there FileStatus[] storeFiles = fs.listStatus(storedir); assertEquals("Didn't have expected store files", count, storeFiles.length); return storeFiles; }
/** * Clean up a merged region * @param env MasterProcedureEnv */ private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); regionFs.cleanupMergedRegion(mergedRegion); }
private Path createReferences(final MasterServices services, final TableDescriptor td, final HRegionInfo parent, final HRegionInfo daughter, final byte [] midkey, final boolean top) throws IOException { Path rootdir = services.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable()); Path storedir = HStore.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName()); Reference ref = top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey); long now = System.currentTimeMillis(); // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); FileSystem fs = services.getMasterFileSystem().getFileSystem(); ref.write(fs, p); return p; }
protected static List<RegionInfo> createFsLayout(final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory( tempTableDir, tableDescriptor, false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
/** * Action before cloning from snapshot. * @param env MasterProcedureEnv * @throws IOException * @throws InterruptedException */ private void preCloneSnapshot(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { // Check and update namespace quota final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); ProcedureSyncWait.getMasterQuotaManager(env) .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); } final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableAction(tableDescriptor, null, getUser()); } }
@Override protected Flow executeFromState(MasterProcedureEnv env, MasterProcedureProtos.TruncateTableState state) throws InterruptedException { if (!failOnce && state == MasterProcedureProtos.TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT) { try { // To emulate an HDFS failure, create only the first region directory RegionInfo regionInfo = getFirstRegionInfo(); Configuration conf = env.getMasterConfiguration(); MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); Path tempdir = mfs.getTempDir(); Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable()); Path regionDir = FSUtils.getRegionDir(tableDir, regionInfo); FileSystem fs = FileSystem.get(conf); fs.mkdirs(regionDir); failOnce = true; return Flow.HAS_MORE_STATE; } catch (IOException e) { fail("failed to create a region directory: " + e); } } return super.executeFromState(env, state); } }
@Override protected Flow executeFromState(MasterProcedureEnv env, MasterProcedureProtos.CreateTableState state) throws InterruptedException { if (!failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT) { try { // To emulate an HDFS failure, create only the first region directory RegionInfo regionInfo = getFirstRegionInfo(); Configuration conf = env.getMasterConfiguration(); MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); Path tempdir = mfs.getTempDir(); Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable()); Path regionDir = FSUtils.getRegionDir(tableDir, regionInfo); FileSystem fs = FileSystem.get(conf); fs.mkdirs(regionDir); failOnce = true; return Flow.HAS_MORE_STATE; } catch (IOException e) { fail("failed to create a region directory: " + e); } } return super.executeFromState(env, state); } }
/** * Create daughter regions */ @VisibleForTesting public void createDaughterRegions(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), getTableName()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); regionFs.createSplitsDir(); Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs); assertReferenceFileCount(fs, expectedReferences.getFirst(), regionFs.getSplitsDir(daughter_1_RI)); //Move the files from the temporary .splits to the final /table/region directory regionFs.commitDaughterRegion(daughter_1_RI); assertReferenceFileCount(fs, expectedReferences.getFirst(), new Path(tabledir, daughter_1_RI.getEncodedName())); assertReferenceFileCount(fs, expectedReferences.getSecond(), regionFs.getSplitsDir(daughter_2_RI)); regionFs.commitDaughterRegion(daughter_2_RI); assertReferenceFileCount(fs, expectedReferences.getSecond(), new Path(tabledir, daughter_2_RI.getEncodedName())); }
/** * Create a merged region * @param env MasterProcedureEnv */ private void createMergedRegion(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); regionFs.createMergesDir(); mergeStoreFiles(env, regionFs, regionFs.getMergesDir()); HRegionFileSystem regionFs2 = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[1], false); mergeStoreFiles(env, regionFs2, regionFs.getMergesDir()); regionFs.commitMergedRegion(mergedRegion); //Prepare to create merged regions env.getAssignmentManager().getRegionStates(). getOrCreateRegionStateNode(mergedRegion).setState(State.MERGING_NEW); }