/** * @param conf {@link Configuration} from which to extract specific archive locations * @param serverName Region Server owner of the log * @param logName WAL file name * @throws IOException on unexpected error. */ public WALLink(final Configuration conf, final String serverName, final String logName) throws IOException { this(CommonFSUtils.getWALRootDir(conf), serverName, logName); }
public WALProcedureStore(final Configuration conf, final LeaseRecovery leaseRecovery) throws IOException { this(conf, new Path(CommonFSUtils.getWALRootDir(conf), MASTER_PROCEDURE_LOGDIR), new Path(CommonFSUtils.getWALRootDir(conf), HConstants.HREGION_OLDLOGDIR_NAME), leaseRecovery); }
public static FileSystem getWALFileSystem(final Configuration c) throws IOException { Path p = getWALRootDir(c); FileSystem fs = p.getFileSystem(c); // hadoop-core does fs caching, so need to propogate this if set String enforceStreamCapability = c.get(UNSAFE_STREAM_CAPABILITY_ENFORCE); if (enforceStreamCapability != null) { fs.getConf().set(UNSAFE_STREAM_CAPABILITY_ENFORCE, enforceStreamCapability); } return fs; }
/** * Checks for the presence of the WAL log root path (using the provided conf object) in the given * path. If it exists, this method removes it and returns the String representation of remaining * relative path. * @param path must not be null * @param conf must not be null * @return String representation of the remaining relative path * @throws IOException from underlying filesystem */ public static String removeWALRootPath(Path path, final Configuration conf) throws IOException { Path root = getWALRootDir(conf); String pathStr = path.toString(); // check that the path is absolute... it has the root path in it. if (!pathStr.startsWith(root.toString())) { return pathStr; } // if not, return as it is. return pathStr.substring(root.toString().length() + 1);// remove the "/" too. }
/** * Returns the Table directory under the WALRootDir for the specified table name * @param conf configuration used to get the WALRootDir * @param tableName Table to get the directory for * @return a path to the WAL table directory for the specified table * @throws IOException if there is an exception determining the WALRootDir */ public static Path getWALTableDir(final Configuration conf, final TableName tableName) throws IOException { return new Path(new Path(getWALRootDir(conf), tableName.getNamespaceAsString()), tableName.getQualifierAsString()); }
/** * @return List of all RegionServer WAL dirs; i.e. this.rootDir/HConstants.HREGION_LOGDIR_NAME. */ public FileStatus[] getWALDirPaths(final PathFilter filter) throws IOException { Path walDirPath = new Path(CommonFSUtils.getWALRootDir(conf), HConstants.HREGION_LOGDIR_NAME); FileStatus[] walDirForServerNames = FSUtils.listStatus( CommonFSUtils.getWALFileSystem(conf), walDirPath, filter); return walDirForServerNames == null? new FileStatus[0]: walDirForServerNames; }
/** * Removes the empty Meta recovery WAL directory. * @param walFactoryId A unique identifier for WAL factory which was used by Filesystem to make a * Meta recovery WAL directory inside WAL directory path. */ private void removeHBCKMetaRecoveryWALDir(String walFactoryId) throws IOException { Path walLogDir = new Path(new Path(CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_LOGDIR_NAME), walFactoryId); FileSystem fs = CommonFSUtils.getWALFileSystem(getConf()); FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); if (walFiles == null || walFiles.length == 0) { LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); if (!FSUtils.deleteDirectory(fs, walLogDir)) { LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); } } }
@Override protected AsyncFSWAL createWAL() throws IOException { return new AsyncFSWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null, eventLoopGroup, channelClass); }
@Override protected FSHLog createWAL() throws IOException { return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); }
@Test(expected=IllegalStateException.class) public void testGetWALRootDirIllegalWALDir() throws IOException { Path root = new Path("file:///hbase/root"); Path invalidWALDir = new Path("file:///hbase/root/logroot"); CommonFSUtils.setRootDir(conf, root); CommonFSUtils.setWALRootDir(conf, invalidWALDir); CommonFSUtils.getWALRootDir(conf); }
private FSHLog createWAL() throws IOException { String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId; return new IOTestWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); }
private int doSearch(Configuration conf, String keysDir) throws Exception { Path inputDir = new Path(keysDir); getConf().set(SEARCHER_INPUTDIR_KEY, inputDir.toString()); SortedSet<byte []> keys = readKeysToSearch(getConf()); if (keys.isEmpty()) throw new RuntimeException("No keys to find"); LOG.info("Count of keys to find: " + keys.size()); for(byte [] key: keys) LOG.info("Key: " + Bytes.toStringBinary(key)); // Now read all WALs. In two dirs. Presumes certain layout. Path walsDir = new Path(CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_LOGDIR_NAME); Path oldWalsDir = new Path( CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_OLDLOGDIR_NAME); LOG.info("Running Search with keys inputDir=" + inputDir + " against " + getConf().get(HConstants.HBASE_DIR)); int ret = ToolRunner.run(new WALSearcher(getConf()), new String [] {walsDir.toString(), ""}); if (ret != 0) return ret; return ToolRunner.run(new WALSearcher(getConf()), new String [] {oldWalsDir.toString(), ""}); }
private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException { Path logRoot = new Path(CommonFSUtils.getWALRootDir(c), HConstants.HREGION_LOGDIR_NAME); FileSystem fs = logRoot.getFileSystem(c); RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true); List<FileStatus> logFiles = new ArrayList<FileStatus>(); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) { logFiles.add(lfs); LOG.info(Objects.toString(lfs)); } } return logFiles; }
@Test public void testGetWALRootDir() throws IOException { Path root = new Path("file:///hbase/root"); Path walRoot = new Path("file:///hbase/logroot"); CommonFSUtils.setRootDir(conf, root); assertEquals(root, CommonFSUtils.getRootDir(conf)); assertEquals(root, CommonFSUtils.getWALRootDir(conf)); CommonFSUtils.setWALRootDir(conf, walRoot); assertEquals(walRoot, CommonFSUtils.getWALRootDir(conf)); }
public int run(Path inputDir, int numMappers) throws Exception { getConf().set(SEARCHER_INPUTDIR_KEY, inputDir.toString()); SortedSet<byte []> keys = readKeysToSearch(getConf()); if (keys.isEmpty()) throw new RuntimeException("No keys to find"); LOG.info("Count of keys to find: " + keys.size()); for(byte [] key: keys) LOG.info("Key: " + Bytes.toStringBinary(key)); // Now read all WALs. In two dirs. Presumes certain layout. Path walsDir = new Path( CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_LOGDIR_NAME); Path oldWalsDir = new Path( CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_OLDLOGDIR_NAME); LOG.info("Running Search with keys inputDir=" + inputDir +", numMappers=" + numMappers + " against " + getConf().get(HConstants.HBASE_DIR)); int ret = ToolRunner.run(getConf(), new WALSearcher(getConf()), new String [] {walsDir.toString(), ""}); if (ret != 0) { return ret; } return ToolRunner.run(getConf(), new WALSearcher(getConf()), new String [] {oldWalsDir.toString(), ""}); }
@Test(expected = IOException.class) public void testFailedToCreateWALIfParentRenamed() throws IOException, CommonFSUtils.StreamLacksCapabilityException { final String name = "testFailedToCreateWALIfParentRenamed"; AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null); long filenum = System.currentTimeMillis(); Path path = wal.computeFilename(filenum); wal.createWriterInstance(path); Path parent = path.getParent(); path = wal.computeFilename(filenum + 1); Path newPath = new Path(parent.getParent(), parent.getName() + "-splitting"); FS.rename(parent, newPath); wal.createWriterInstance(path); fail("It should fail to create the new WAL"); }
@Test public void testRemoveWALRootPath() throws Exception { CommonFSUtils.setRootDir(conf, new Path("file:///user/hbase")); Path testFile = new Path(CommonFSUtils.getRootDir(conf), "test/testfile"); Path tmpFile = new Path("file:///test/testfile"); assertEquals("test/testfile", CommonFSUtils.removeWALRootPath(testFile, conf)); assertEquals(tmpFile.toString(), CommonFSUtils.removeWALRootPath(tmpFile, conf)); CommonFSUtils.setWALRootDir(conf, new Path("file:///user/hbaseLogDir")); assertEquals(testFile.toString(), CommonFSUtils.removeWALRootPath(testFile, conf)); Path logFile = new Path(CommonFSUtils.getWALRootDir(conf), "test/testlog"); assertEquals("test/testlog", CommonFSUtils.removeWALRootPath(logFile, conf)); }
@Test(expected = WALClosedException.class) public void testRollWriterForClosedWAL() throws IOException { String testName = currentTest.getMethodName(); AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null); wal.close(); wal.rollWriter(); } }
@Test public void testSyncNoAppend() throws IOException { String testName = currentTest.getMethodName(); AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName, CONF, null, true, null, null); try { wal.sync(); } finally { wal.close(); } }
/** * A loaded WAL coprocessor won't break existing WAL test cases. */ @Test public void testWALCoprocessorLoaded() throws Exception { // test to see whether the coprocessor is loaded or not. AbstractFSWAL<?> wal = null; try { wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null); WALCoprocessorHost host = wal.getCoprocessorHost(); Coprocessor c = host.findCoprocessor(SampleRegionWALCoprocessor.class); assertNotNull(c); } finally { if (wal != null) { wal.close(); } } }