protected TableDescriptors getFsTableDescriptors() throws IOException { return new FSTableDescriptors(this.conf, this.fs, this.rootDir, !canUpdateTableDescriptor(), false, getMetaTableObserver()); }
public void consolidate() throws IOException { if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) { Path rootDir = FSUtils.getRootDir(conf); LOG.info("Using old Snapshot Format"); // write a copy of descriptor to the snapshot directory new FSTableDescriptors(conf, workingDirFs, rootDir) .createTableDescriptorForTableDirectory(workingDir, htd, false); } else { LOG.debug("Convert to Single Snapshot Manifest"); convertToV2SingleManifest(); } }
/** Loads table info's for tables that may not have been included, since there are no * regions reported for the table, but table dir is there in hdfs */ private void loadTableInfosForTablesWithNoRegion() throws IOException { Map<String, TableDescriptor> allTables = new FSTableDescriptors(getConf()).getAll(); for (TableDescriptor htd : allTables.values()) { if (checkMetaOnly && !htd.isMetaTable()) { continue; } TableName tableName = htd.getTableName(); if (isTableIncluded(tableName) && !tablesInfo.containsKey(tableName)) { TableInfo tableInfo = new TableInfo(tableName); tableInfo.htds.add(htd); tablesInfo.put(htd.getTableName(), tableInfo); } } }
private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating hbase:meta region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstrap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, rd, c, setInfoFamilyCachingForMeta(metaDescriptor, false), null); meta.close(); } catch (IOException e) { e = e instanceof RemoteException ? ((RemoteException)e).unwrapRemoteException() : e; LOG.error("bootstrap", e); throw e; } }
/** * You must call {@link #closeRootAndMeta()} when done after calling this * method. It does cleanup. * @throws IOException */ protected void createMetaRegion() throws IOException { FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf); meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf, fsTableDescriptors.get(TableName.META_TABLE_NAME)); }
public SnapshotBuilder(final Configuration conf, final FileSystem fs, final Path rootDir, final TableDescriptor htd, final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions) throws IOException { this.fs = fs; this.conf = conf; this.rootDir = rootDir; this.htd = htd; this.desc = desc; this.tableRegions = tableRegions; this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); new FSTableDescriptors(conf) .createTableDescriptorForTableDirectory(snapshotDir, htd, false); }
/** * This borrows code from MasterFileSystem.bootstrap(). Explicitly creates it's own WAL, so be * sure to close it as well as the region when you're finished. * @param walFactoryID A unique identifier for WAL factory. Filesystem implementations will use * this ID to make a directory inside WAL directory path. * @return an open hbase:meta HRegion */ private HRegion createNewMeta(String walFactoryID) throws IOException { Path rootdir = FSUtils.getRootDir(getConf()); Configuration c = getConf(); RegionInfo metaHRI = RegionInfoBuilder.FIRST_META_REGIONINFO; TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, false); // The WAL subsystem will use the default rootDir rather than the passed in rootDir // unless I pass along via the conf. Configuration confForWAL = new Configuration(c); confForWAL.set(HConstants.HBASE_DIR, rootdir.toString()); WAL wal = new WALFactory(confForWAL, walFactoryID).getWAL(metaHRI); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, metaDescriptor, wal); MasterFileSystem.setInfoFamilyCachingForMeta(metaDescriptor, true); return meta; }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
@Test public void testReadingInvalidDirectoryFromFS() throws IOException { FileSystem fs = FileSystem.get(UTIL.getConfiguration()); try { new FSTableDescriptors(UTIL.getConfiguration(), fs, FSUtils.getRootDir(UTIL.getConfiguration())) .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY)); fail("Shouldn't be able to read a table descriptor for the archive directory."); } catch (Exception e) { LOG.debug("Correctly got error when reading a table descriptor from the archive directory: " + e.getMessage()); } }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); htds.add(htd); htds.add(htd); htds.add(htd); }
@Test public void testShouldCreateNewTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertTrue("Should create new table descriptor", fstd.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(), false)); }
@Test public void testShouldNotCreateTheSameTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detritus laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); fstd.add(htd); assertFalse("Should not create new table descriptor", fstd.createTableDescriptor(htd, false)); }
@Test public void testShouldAllowForcefulCreationOfAlreadyExistingTableDescriptor() throws Exception { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); fstd.createTableDescriptor(htd, false); assertTrue("Should create new table descriptor", fstd.createTableDescriptor(htd, true)); }
/** * Execute the restore operation * @param snapshotDir The snapshot directory to use as "restore source" * @param sd The snapshot descriptor * @param htdClone The HTableDescriptor of the table to restore/clone. */ private void testRestore(final Path snapshotDir, final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException { LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); new FSTableDescriptors(conf).createTableDescriptor(htdClone); RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); helper.restoreHdfsRegions(); LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); }
@Test public void testReadingHTDFromFS() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); fstd.createTableDescriptor(htd); TableDescriptor td2 = FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName()); assertTrue(htd.equals(td2)); }
@Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); Path descriptorFile = fstd.updateTableDescriptor(htd); try (FSDataOutputStream out = fs.create(descriptorFile, true)) { out.write(TableDescriptorBuilder.toByteArray(htd)); } FSTableDescriptors fstd2 = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor td2 = fstd2.get(htd.getTableName()); assertEquals(htd, td2); FileStatus descriptorFile2 = FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName())); byte[] buffer = TableDescriptorBuilder.toByteArray(htd); try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) { in.readFully(buffer); } TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer); assertEquals(htd, td3); }
@Test public void testRemoves() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); htds.add(htd); assertNotNull(htds.remove(htd.getTableName())); assertNull(htds.remove(htd.getTableName())); }
@Test public void testCreateAndUpdate() throws IOException { Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd)); FileStatus [] statuses = fs.listStatus(testdir); assertTrue("statuses.length="+statuses.length, statuses.length == 1); for (int i = 0; i < 10; i++) { fstd.updateTableDescriptor(htd); } statuses = fs.listStatus(testdir); assertTrue(statuses.length == 1); Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp"); statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); }
@Test public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException { Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd)); htd = TableDescriptorBuilder.newBuilder(htd) .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")) .build(); assertTrue(fstd.createTableDescriptor(htd)); //this will re-create Path tableDir = fstd.getTableDir(htd.getTableName()); Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR); FileStatus[] statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); }
@Test public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(convert(ri), basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, convert(ri), fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. RegionInfo deserializedRi = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); HBaseTestingUtility.closeRegionAndWAL(r); }