protected static void moveTempDirectoryToHBaseRoot( final MasterProcedureEnv env, final TableDescriptor tableDescriptor, final Path tempTableDir) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableDescriptor.getTableName()); FileSystem fs = mfs.getFileSystem(); if (!fs.delete(tableDir, true) && fs.exists(tableDir)) { throw new IOException("Couldn't delete " + tableDir); } if (!fs.rename(tempTableDir, tableDir)) { throw new IOException("Unable to move table from temp=" + tempTableDir + " to hbase root=" + tableDir); } }
@VisibleForTesting public static void createDirectory(MasterFileSystem mfs, NamespaceDescriptor nsDescriptor) throws IOException { mfs.getFileSystem().mkdirs(FSUtils.getNamespaceDir(mfs.getRootDir(), nsDescriptor.getName())); }
public SplitWALManager(MasterServices master) { this.master = master; this.conf = master.getConfiguration(); this.splitWorkerAssigner = new SplitWorkerAssigner(this.master, conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER)); this.rootDir = master.getMasterFileSystem().getWALRootDir(); this.fs = master.getMasterFileSystem().getFileSystem(); }
private SnapshotManager getNewManager(Configuration conf, int intervalSeconds) throws IOException, KeeperException { Mockito.reset(services); Mockito.when(services.getConfiguration()).thenReturn(conf); Mockito.when(services.getMasterFileSystem()).thenReturn(mfs); Mockito.when(mfs.getFileSystem()).thenReturn(fs); Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir()); return new SnapshotManager(services, coordinator, pool, intervalSeconds); }
private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException { MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Set<String> families = new HashSet<>(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); for (Path regionDir : FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) { for (Path familyDir : FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) { families.add(familyDir.getName()); } } return families; }
public static void deleteArchiveDirectory(final HBaseTestingUtility util) throws IOException { // Ensure the archiver to be empty MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); mfs.getFileSystem().delete(archiveDir, true); }
private FileSystem getFileSystem() { return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); }
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); // make sure the fs and the found root dir have the same scheme LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf())); LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.getConfiguration())); // make sure the set uri matches by forcing it. assertEquals(masterRoot, rootDir); } }
/** * Clean up a merged region * @param env MasterProcedureEnv */ private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); regionFs.cleanupMergedRegion(mergedRegion); }
@Before public void setup() throws Exception { fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); admin = UTIL.getAdmin(); originalTableName = TableName.valueOf("test" + testName.getMethodName()); cloneTableName = TableName.valueOf("test-clone-" + originalTableName); snapshotNameAsString = "snapshot_" + originalTableName; snapshotName = Bytes.toBytes(snapshotNameAsString); originalTable = createTable(originalTableName, TEST_FAM); loadData(originalTable, TEST_FAM); countOriginalTable = countRows(originalTable); System.out.println("Original table has: " + countOriginalTable + " rows"); }
public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName, final String family) throws IOException { // verify htd TableDescriptor htd = master.getTableDescriptors().get(tableName); assertTrue(htd != null); assertFalse(htd.hasColumnFamily(family.getBytes())); // verify fs final FileSystem fs = master.getMasterFileSystem().getFileSystem(); final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { final Path familyDir = new Path(regionDir, family); assertFalse(family + " family dir should not exist", fs.exists(familyDir)); } }
public static void confirmSnapshotValid(HBaseTestingUtility testUtil, SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) throws IOException { MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(), testUtil.getAdmin(), mfs.getFileSystem()); }
private void verifyTableDescriptor(final TableName tableName, final byte[]... families) throws Exception { // Verify descriptor from master TableDescriptor htd = admin.getDescriptor(tableName).get(); verifyTableDescriptor(htd, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); verifyTableDescriptor(td, tableName, families); }
private Path createReferences(final MasterServices services, final TableDescriptor td, final HRegionInfo parent, final HRegionInfo daughter, final byte [] midkey, final boolean top) throws IOException { Path rootdir = services.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable()); Path storedir = HStore.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName()); Reference ref = top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey); long now = System.currentTimeMillis(); // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); FileSystem fs = services.getMasterFileSystem().getFileSystem(); ref.write(fs, p); return p; }
/** * Basic end-to-end test of simple-flush-based snapshots */ @Test public void testFlushCreateListDestroy() throws Exception { LOG.debug("------- Starting Snapshot test -------------"); // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // load the table so we have some data SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); String snapshotName = "flushSnapshotCreateListDestroy"; FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM), snapshotName, rootDir, fs, true); }
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); assertEquals(masterRoot, rootDir); assertEquals(FSUtils.getWALRootDir(UTIL.getConfiguration()), fs.getWALRootDir()); } }
public static void validateTableDeletion( final HMaster master, final TableName tableName) throws IOException { // check filesystem final FileSystem fs = master.getMasterFileSystem().getFileSystem(); final Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); assertFalse(fs.exists(tableDir)); // check meta assertFalse(MetaTableAccessor.tableExists(master.getConnection(), tableName)); assertEquals(0, countMetaRegions(master, tableName)); // check htd assertTrue("found htd of deleted table", master.getTableDescriptors().get(tableName) == null); }
private void verifyTableDescriptor(final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); verifyTableDescriptor(htd, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); verifyTableDescriptor(td, tableName, families); }
@BeforeClass public static void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); UTIL.startMiniCluster(NUM_RS); fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getAdmin(); // Verify descriptor from master TableDescriptor htd = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = td.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }