private void compactRegion(final Path tableDir, final TableDescriptor htd, final Path regionDir, final boolean compactOnce, final boolean major) throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); } }
/** * @return the set of the regions contained in the table */ private List<RegionInfo> getTableRegions() throws IOException { LOG.debug("get table regions: " + tableDir); FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); if (regionDirs == null) return null; List<RegionInfo> regions = new ArrayList<>(regionDirs.length); for (int i = 0; i < regionDirs.length; ++i) { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); regions.add(hri); } LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getTableName().getNameAsString()); return regions; }
/** * Read the .regioninfo file from the file system. If there is no * .regioninfo, add it to the orphan hdfs region list. */ private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException { Path regionDir = hbi.getHdfsRegionDir(); if (regionDir == null) { if (hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) { // Log warning only for default/ primary replica with no region dir LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry); } return; } if (hbi.hdfsEntry.hri != null) { // already loaded data return; } FileSystem fs = FileSystem.get(getConf()); RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); LOG.debug("RegionInfo read: " + hri.toString()); hbi.hdfsEntry.hri = hri; }
/** * Execute the compaction on the specified path. * * @param path Directory path on which to run compaction. * @param compactOnce Execute just a single step of compaction. * @param major Request major compaction. */ public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { if (isFamilyDir(fs, path)) { Path regionDir = path.getParent(); Path tableDir = regionDir.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); compactStoreFiles(tableDir, htd, hri, path.getName(), compactOnce, major); } else if (isRegionDir(fs, path)) { Path tableDir = path.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); compactRegion(tableDir, htd, path, compactOnce, major); } else if (isTableDir(fs, path)) { compactTable(path, compactOnce, major); } else { throw new IOException( "Specified path is not a table, region or family directory. path=" + path); } }
Path regionDir = regionFs.getRegionDir(); try { HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); } catch (IOException e) { LOG.info("Caught expected IOE due missing .regioninfo file, due: " + e.getMessage() + " skipping region open.");
@Test public void testOnDiskRegionCreation() throws IOException { Path rootDir = TEST_UTIL.getDataTestDirOnTestFS(name.getMethodName()); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); // Create a Region RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); // Verify if the region is on disk Path regionDir = regionFs.getRegionDir(); assertTrue("The region folder should be created", fs.exists(regionDir)); // Verify the .regioninfo RegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); assertEquals(hri, hriVerify); // Open the region regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri, false); assertEquals(regionDir, regionFs.getRegionDir()); // Delete the region HRegionFileSystem.deleteRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); assertFalse("The region folder should be removed", fs.exists(regionDir)); fs.delete(rootDir, true); }
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. org.apache.hadoop.hbase.client.RegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(org.apache.hadoop.hbase.client.RegionInfo.COMPARATOR.compare(hri, deserializedHri) == 0); HBaseTestingUtility.closeRegionAndWAL(r); }
@Test public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(convert(ri), basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, convert(ri), fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. RegionInfo deserializedRi = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); HBaseTestingUtility.closeRegionAndWAL(r); }
private void compactRegion(final Path tableDir, final TableDescriptor htd, final Path regionDir, final boolean compactOnce, final boolean major) throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); } }
private void compactRegion(final Path tableDir, final TableDescriptor htd, final Path regionDir, final boolean compactOnce, final boolean major) throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); } }
private void compactRegion(final Path tableDir, final HTableDescriptor htd, final Path regionDir, final boolean compactOnce, final boolean major) throws IOException { HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); } }
/** * Execute the compaction on the specified path. * * @param path Directory path on which to run compaction. * @param compactOnce Execute just a single step of compaction. * @param major Request major compaction. */ public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { if (isFamilyDir(fs, path)) { Path regionDir = path.getParent(); Path tableDir = regionDir.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); compactStoreFiles(tableDir, htd, hri, path.getName(), compactOnce, major); } else if (isRegionDir(fs, path)) { Path tableDir = path.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); compactRegion(tableDir, htd, path, compactOnce, major); } else if (isTableDir(fs, path)) { compactTable(path, compactOnce, major); } else { throw new IOException( "Specified path is not a table, region or family directory. path=" + path); } }
/** * @return the set of the regions contained in the table */ private List<HRegionInfo> getTableRegions() throws IOException { LOG.debug("get table regions: " + tableDir); FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); if (regionDirs == null) return null; List<HRegionInfo> regions = new ArrayList<HRegionInfo>(regionDirs.length); for (int i = 0; i < regionDirs.length; ++i) { HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); regions.add(hri); } LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getTableName().getNameAsString()); return regions; }
/** * Execute the compaction on the specified path. * * @param path Directory path on which to run compaction. * @param compactOnce Execute just a single step of compaction. * @param major Request major compaction. */ public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { if (isFamilyDir(fs, path)) { Path regionDir = path.getParent(); Path tableDir = regionDir.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); compactStoreFiles(tableDir, htd, hri, path.getName(), compactOnce, major); } else if (isRegionDir(fs, path)) { Path tableDir = path.getParent(); TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); compactRegion(tableDir, htd, path, compactOnce, major); } else if (isTableDir(fs, path)) { compactTable(path, compactOnce, major); } else { throw new IOException( "Specified path is not a table, region or family directory. path=" + path); } }
/** * Execute the compaction on the specified path. * * @param path Directory path on which to run compaction. * @param compactOnce Execute just a single step of compaction. * @param major Request major compaction. */ public void compact(final Path path, final boolean compactOnce, final boolean major) throws IOException { if (isFamilyDir(fs, path)) { Path regionDir = path.getParent(); Path tableDir = regionDir.getParent(); HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); compactStoreFiles(tableDir, htd, hri, path.getName(), compactOnce, major); } else if (isRegionDir(fs, path)) { Path tableDir = path.getParent(); HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); compactRegion(tableDir, htd, path, compactOnce, major); } else if (isTableDir(fs, path)) { compactTable(path, compactOnce, major); } else { throw new IOException( "Specified path is not a table, region or family directory. path=" + path); } }
/** * Read the .regioninfo file from the file system. If there is no * .regioninfo, add it to the orphan hdfs region list. */ private void loadHdfsRegioninfo(HbckInfo hbi) throws IOException { Path regionDir = hbi.getHdfsRegionDir(); if (regionDir == null) { LOG.warn("No HDFS region dir found: " + hbi + " meta=" + hbi.metaEntry); return; } if (hbi.hdfsEntry.hri != null) { // already loaded data return; } FileSystem fs = FileSystem.get(getConf()); HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); LOG.debug("HRegionInfo read: " + hri.toString()); hbi.hdfsEntry.hri = hri; }
Path regionDir = regionFs.getRegionDir(); try { HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); } catch (IOException e) { LOG.info("Caught expected IOE due missing .regioninfo file, due: " + e.getMessage() + " skipping region open.");
@Test public void testOnDiskRegionCreation() throws IOException { Path rootDir = TEST_UTIL.getDataTestDirOnTestFS(name.getMethodName()); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); // Create a Region RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); // Verify if the region is on disk Path regionDir = regionFs.getRegionDir(); assertTrue("The region folder should be created", fs.exists(regionDir)); // Verify the .regioninfo RegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); assertEquals(hri, hriVerify); // Open the region regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri, false); assertEquals(regionDir, regionFs.getRegionDir()); // Delete the region HRegionFileSystem.deleteRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); assertFalse("The region folder should be removed", fs.exists(regionDir)); fs.delete(rootDir, true); }
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. org.apache.hadoop.hbase.client.RegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(org.apache.hadoop.hbase.client.RegionInfo.COMPARATOR.compare(hri, deserializedHri) == 0); HBaseTestingUtility.closeRegionAndWAL(r); }
@Test public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(convert(ri), basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, convert(ri), fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. RegionInfo deserializedRi = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); HBaseTestingUtility.closeRegionAndWAL(r); }