@Override public HRegionFileSystem regionOpen(final RegionInfo regionInfo) throws IOException { HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf, workingDirFs, snapshotDir, regionInfo); return snapshotRegionFs; }
private void compactRegion(final Path tableDir, final TableDescriptor htd, final Path regionDir, final boolean compactOnce, final boolean major) throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { compactStoreFiles(tableDir, htd, hri, familyDir.getName(), compactOnce, major); } }
final FileSystem fs, final Path tableDir, final RegionInfo regionInfo, boolean readOnly) throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo); Path regionDir = regionFs.getRegionDir(); regionFs.cleanupTempDir(); regionFs.cleanupSplitsDir(); regionFs.cleanupMergesDir(); regionFs.checkRegionInfoOnFilesystem(); } else { if (LOG.isDebugEnabled()) {
/** * Move the file from a build/temp location to the main family store directory. * @param familyName Family that will gain the file * @param buildPath {@link Path} to the file to commit. * @return The new {@link Path} of the committed file * @throws IOException */ public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException { Path dstPath = preCommitStoreFile(familyName, buildPath, -1, false); return commitStoreFile(buildPath, dstPath); }
/** * Clean up any split detritus that may have been left around from previous split attempts. */ void cleanupSplitsDir() throws IOException { deleteDir(getSplitsDir()); }
/** * Clean up any merge detritus that may have been left around from previous merge attempts. */ void cleanupMergesDir() throws IOException { deleteDir(getMergesDir()); }
.setEndKey(endKey) .build(); HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); regions[i] = new RegionData(tableDir, hri, 3); for (int j = 0; j < regions[i].files.length; ++j) { Path storeFile = createStoreFile(rfs.createTempName()); regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); endKey = Bytes.toBytes(3 + i * 2); hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length); for (int j = 0; j < regions[i].files.length; ++j) { String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName(); Path refFile = createStoreFile(new Path(rootDir, refName)); regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
@Test public void testNonIdempotentOpsWithRetries() throws IOException { Path rootDir = TEST_UTIL.getDataTestDirOnTestFS(name.getMethodName()); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); // Create a Region RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri); assertTrue(fs.exists(regionFs.getRegionDir())); regionFs = new HRegionFileSystem(conf, new MockFileSystemForCreate(), rootDir, hri); boolean result = regionFs.createDir(new Path("/foo/bar")); assertTrue("Couldn't create the directory", result); regionFs = new HRegionFileSystem(conf, new MockFileSystem(), rootDir, hri); result = regionFs.rename(new Path("/foo/bar"), new Path("/foo/bar2")); assertTrue("Couldn't rename the directory", result); regionFs = new HRegionFileSystem(conf, new MockFileSystem(), rootDir, hri); result = regionFs.deleteDir(new Path("/foo/bar")); assertTrue("Couldn't delete the directory", result); fs.delete(rootDir, true); }
@Test public void testOnDiskRegionCreation() throws IOException { Path rootDir = TEST_UTIL.getDataTestDirOnTestFS(name.getMethodName()); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); // Create a Region RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); // Verify if the region is on disk Path regionDir = regionFs.getRegionDir(); assertTrue("The region folder should be created", fs.exists(regionDir)); // Verify the .regioninfo RegionInfo hriVerify = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); assertEquals(hri, hriVerify); // Open the region regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri, false); assertEquals(regionDir, regionFs.getRegionDir()); // Delete the region HRegionFileSystem.deleteRegionFromFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); assertFalse("The region folder should be removed", fs.exists(regionDir)); fs.delete(rootDir, true); }
HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, rootDir, hri); Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); Path buildPath = regionFs.createTempName(); fs.createNewFile(buildPath); storeFiles = regionFs.getStoreFiles(familyName); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); Path dstPath = regionFs.commitStoreFile(familyName, buildPath); storeFiles = regionFs.getStoreFiles(familyName); assertEquals(0, storeFiles != null ? storeFiles.size() : 0); assertFalse(fs.exists(buildPath));
HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); .withFilePath(regionFs.createTempName()) .withFileContext(meta) .build(); writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY)); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath,
HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, FSUtils.getTableDir(rootDir, hri.getTable()), hri); Path regionDir = regionFs.getRegionDir(); try { HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir); } catch (IOException e) { LOG.info("Caught expected IOE due missing .regioninfo file, due: " + e.getMessage() + " skipping region open.");
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. org.apache.hadoop.hbase.client.RegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(org.apache.hadoop.hbase.client.RegionInfo.COMPARATOR.compare(hri, deserializedHri) == 0); HBaseTestingUtility.closeRegionAndWAL(r); }
HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri); .withFilePath(regionFs.createTempName()) .withFileContext(meta) .build(); writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem( testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hriClone); Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath,
Path tableDir = FSUtils.getTableDir(rootDir, info.getTable()); HRegionFileSystem .createRegionOnFileSystem(CONF, TEST_UTIL.getTestFileSystem(), tableDir, info); region = HRegion.newHRegion(tableDir, wal, TEST_UTIL.getTestFileSystem(), CONF, info, htd, rs); Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
final WALFactory wals = new WALFactory(CONF, method); try { Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes(); Path tmpDir = new Path(region.getRegionFileSystem().getTempDir(), Bytes.toString(family)); FileStatus[] files = FSUtils.listStatus(fs, tmpDir); String errorMsg = "Expected to find 1 file in the region temp directory " assertEquals(errorMsg, 1, files.length); Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family), files[0].getPath()); .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family, storeFiles, Lists.newArrayList(newFile), region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
/** * Clean up any split detritus that may have been left around from previous * split attempts. * Call this method on initial region deploy. * @throws IOException */ void cleanupAnySplitDetritus() throws IOException { Path splitdir = this.getSplitsDir(); if (!fs.exists(splitdir)) return; // Look at the splitdir. It could have the encoded names of the daughter // regions we tried to make. See if the daughter regions actually got made // out under the tabledir. If here under splitdir still, then the split did // not complete. Try and do cleanup. This code WILL NOT catch the case // where we successfully created daughter a but regionserver crashed during // the creation of region b. In this case, there'll be an orphan daughter // dir in the filesystem. TOOD: Fix. FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs)); if (daughters != null) { for (FileStatus daughter: daughters) { Path daughterDir = new Path(getTableDir(), daughter.getPath().getName()); if (fs.exists(daughterDir) && !deleteDir(daughterDir)) { throw new IOException("Failed delete of " + daughterDir); } } } cleanupSplitsDir(); LOG.info("Cleaned up old failed split transaction detritus: " + splitdir); }
HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo); Path regionDir = regionFs.getRegionDir(); if (fs.exists(regionDir)) { LOG.warn("Trying to create a region that already exists on disk: " + regionDir); if (!createDirOnFileSystem(fs, conf, regionDir)) { LOG.warn("Unable to create the region directory: " + regionDir); throw new IOException("Unable to create region directory: " + regionDir); regionFs.writeRegionInfoOnFilesystem(false); } else { if (LOG.isDebugEnabled())
private Path splitStoreFile(final HRegionFileSystem regionFs, final HRegionInfo hri, final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef) throws IOException { FileSystem fs = regionFs.getFileSystem(); Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null); if (null == path) { return null; } Path regionDir = regionFs.commitDaughterRegion(hri); return new Path(new Path(regionDir, family), path.getName()); }
/** * Create the region splits directory. */ public void createSplitsDir() throws IOException { Path splitdir = getSplitsDir(); if (fs.exists(splitdir)) { LOG.info("The " + splitdir + " directory exists. Hence deleting it to recreate it"); if (!deleteDir(splitdir)) { throw new IOException("Failed deletion of " + splitdir + " before creating them again."); } } // splitDir doesn't exists now. No need to do an exists() call for it. if (!createDir(splitdir)) { throw new IOException("Failed create of " + splitdir); } }