/** * Returns the latest table descriptor for the given table directly from the file system * if it exists, bypassing the local cache. * Returns null if it's not found. */ public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, Path hbaseRootDir, TableName tableName) throws IOException { Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName); return getTableDescriptorFromFs(fs, tableDir); }
/** * Checks if a current table info file exists for the given table * * @param tableName name of table * @return true if exists * @throws IOException */ public boolean isTableInfoExists(TableName tableName) throws IOException { return getTableInfoPath(tableName) != null; }
public void consolidate() throws IOException { if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) { Path rootDir = FSUtils.getRootDir(conf); LOG.info("Using old Snapshot Format"); // write a copy of descriptor to the snapshot directory new FSTableDescriptors(conf, workingDirFs, rootDir) .createTableDescriptorForTableDirectory(workingDir, htd, false); } else { LOG.debug("Convert to Single Snapshot Manifest"); convertToV2SingleManifest(); } }
/** * Find the most current table info file for the given table in the hbase root directory. * @return The file status of the current table info file or null if it does not exist */ private FileStatus getTableInfoPath(final TableName tableName) throws IOException { Path tableDir = getTableDir(tableName); return getTableInfoPath(tableDir); }
/** * Deletes all the table descriptor files from the file system. * Used in unit tests only. * @throws NotImplementedException if in read only mode */ public void deleteTableDescriptorIfExists(TableName tableName) throws IOException { if (fsreadonly) { throw new NotImplementedException("Cannot delete a table descriptor - in read only mode"); } Path tableDir = getTableDir(tableName); Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE); }
/** * Create new TableDescriptor in HDFS. Happens when we are creating table. If * forceCreation is true then even if previous table descriptor is present it * will be overwritten * * @return True if we successfully created file. */ public boolean createTableDescriptor(TableDescriptor htd, boolean forceCreation) throws IOException { Path tableDir = getTableDir(htd.getTableName()); return createTableDescriptorForTableDirectory(tableDir, htd, forceCreation); }
@Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); Path descriptorFile = fstd.updateTableDescriptor(htd); try (FSDataOutputStream out = fs.create(descriptorFile, true)) { out.write(TableDescriptorBuilder.toByteArray(htd)); } FSTableDescriptors fstd2 = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor td2 = fstd2.get(htd.getTableName()); assertEquals(htd, td2); FileStatus descriptorFile2 = FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName())); byte[] buffer = TableDescriptorBuilder.toByteArray(htd); try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) { in.readFully(buffer); } TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer); assertEquals(htd, td3); }
@Test public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException { Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd)); htd = TableDescriptorBuilder.newBuilder(htd) .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")) .build(); assertTrue(fstd.createTableDescriptor(htd)); //this will re-create Path tableDir = fstd.getTableDir(htd.getTableName()); Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR); FileStatus[] statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); }
/** * You must call {@link #closeRootAndMeta()} when done after calling this * method. It does cleanup. * @throws IOException */ protected void createMetaRegion() throws IOException { FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf); meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf, fsTableDescriptors.get(TableName.META_TABLE_NAME)); }
@Test public void testReadingHTDFromFS() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); fstd.createTableDescriptor(htd); TableDescriptor td2 = FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName()); assertTrue(htd.equals(td2)); }
readTableDescriptor(fs, getCurrentTableInfoStatus(fs, oldTablePath)); if(FSTableDescriptors.getTableInfoPath(fs, newTablePath) == null) { LOG.info("Creating new tableDesc for ACL"); HTableDescriptor newDesc = new HTableDescriptor(oldDesc); newDesc.setName(newTableName); new FSTableDescriptors(this.conf).createTableDescriptorForTableDirectory( newTablePath, newDesc, true); final WALFactory walFactory = new WALFactory(conf, null, fakeServer.toString()); WAL metawal = walFactory.getMetaWAL(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()); FSTableDescriptors fst = new FSTableDescriptors(conf); HRegion meta = HRegion.openHRegion(rootDir, HRegionInfo.FIRST_META_REGIONINFO, fst.get(TableName.META_TABLE_NAME), metawal, conf); HRegion region = null; try {
continue; TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table); FileSystem targetFs = target.getFileSystem(conf); FSTableDescriptors descriptors = new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf)); descriptors.createTableDescriptorForTableDirectory(target, orig, false); LOG.debug("Attempting to copy table info for:" + table + " target: " + target + " descriptor: " + orig);
int j = 0; int numFailedCase = 0; FSTableDescriptors fstd = new FSTableDescriptors(getConf()); while (iter.hasNext()) { Entry<TableName, Set<String>> entry = TableDescriptor htd = htds[j]; LOG.info("fixing orphan table: " + tableName + " from cache"); fstd.createTableDescriptor(htd, true); j++; iter.remove();
/** * Update table descriptor on the file system * @throws IOException Thrown if failed update. * @throws NotImplementedException if in read only mode */ @VisibleForTesting Path updateTableDescriptor(TableDescriptor td) throws IOException { if (fsreadonly) { throw new NotImplementedException("Cannot update a table descriptor - in read only mode"); } TableName tableName = td.getTableName(); Path tableDir = getTableDir(tableName); Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(tableDir)); if (p == null) throw new IOException("Failed update"); LOG.info("Updated tableinfo=" + p); if (usecache) { this.cache.put(td.getTableName(), td); } return p; }
@Test public void testSequenceIdAdvancesOnTableInfo() throws IOException { Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); Path p0 = fstd.updateTableDescriptor(htd); int i0 = FSTableDescriptors.getTableInfoSequenceId(p0); Path p1 = fstd.updateTableDescriptor(htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p0)); int i1 = FSTableDescriptors.getTableInfoSequenceId(p1); assertTrue(i1 == i0 + 1); Path p2 = fstd.updateTableDescriptor(htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p1)); int i2 = FSTableDescriptors.getTableInfoSequenceId(p2); assertTrue(i2 == i1 + 1); Path p3 = fstd.updateTableDescriptor(htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p2)); int i3 = FSTableDescriptors.getTableInfoSequenceId(p3); assertTrue(i3 == i2 + 1); TableDescriptor descriptor = fstd.get(htd.getTableName()); assertEquals(descriptor, htd); }
@Test public void testShouldNotCreateTheSameTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detritus laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); fstd.add(htd); assertFalse("Should not create new table descriptor", fstd.createTableDescriptor(htd, false)); }
htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name + i)) != null); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i)); htds.updateTableDescriptor(builder.build()); assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name + i)) != null);
nonchtds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); assertTrue(chtds.get(TableName.valueOf(name + i)) != null); assertTrue(nonchtds.getAll().size() == chtds.getAll().size()); nonchtds.createTableDescriptor(htd); assertTrue(nonchtds.getAll().size() == chtds.getAll().size()); for (Map.Entry<String, TableDescriptor> entry: nonchtds.getAll().entrySet()) { String t = (String) entry.getKey(); TableDescriptor nchtd = entry.getValue(); assertTrue("expected " + htd.toString() + " got: " + chtds.get(TableName.valueOf(t)).toString(), (nchtd.equals(chtds.get(TableName.valueOf(t)))));
FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd); fsd.createTableDescriptor(fsd.get(TableName.META_TABLE_NAME));
throw new NotImplementedException("Cannot create a table descriptor - in read only mode"); FileStatus status = getTableInfoPath(fs, tableDir); if (status != null) { LOG.debug("Current path=" + status.getPath()); if (!forceCreation) { if (fs.exists(status.getPath()) && status.getLen() > 0) { if (readTableDescriptor(fs, status).equals(htd)) { LOG.trace("TableInfo already exists.. Skipping creation"); return false; Path p = writeTableDescriptor(fs, htd, tableDir, status); return p != null;