/** * @param conf {@link Configuration} from which to extract specific archive locations * @param serverName Region Server owner of the log * @param logName WAL file name * @throws IOException on unexpected error. */ public WALLink(final Configuration conf, final String serverName, final String logName) throws IOException { this(CommonFSUtils.getWALRootDir(conf), serverName, logName); }
@SuppressWarnings("deprecation") @Override protected void initOutput(FileSystem fs, Path path, boolean overwritable, int bufferSize, short replication, long blockSize) throws IOException, StreamLacksCapabilityException { this.output = CommonFSUtils.createForWal(fs, path, overwritable, bufferSize, replication, blockSize, false); if (fs.getConf().getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true)) { if (!CommonFSUtils.hasCapability(output, "hflush")) { throw new StreamLacksCapabilityException("hflush"); } if (!CommonFSUtils.hasCapability(output, "hsync")) { throw new StreamLacksCapabilityException("hsync"); } } }
@Test public void testRemoveWALRootPath() throws Exception { CommonFSUtils.setRootDir(conf, new Path("file:///user/hbase")); Path testFile = new Path(CommonFSUtils.getRootDir(conf), "test/testfile"); Path tmpFile = new Path("file:///test/testfile"); assertEquals("test/testfile", CommonFSUtils.removeWALRootPath(testFile, conf)); assertEquals(tmpFile.toString(), CommonFSUtils.removeWALRootPath(tmpFile, conf)); CommonFSUtils.setWALRootDir(conf, new Path("file:///user/hbaseLogDir")); assertEquals(testFile.toString(), CommonFSUtils.removeWALRootPath(testFile, conf)); Path logFile = new Path(CommonFSUtils.getWALRootDir(conf), "test/testlog"); assertEquals("test/testlog", CommonFSUtils.removeWALRootPath(logFile, conf)); }
/** * @return List of all RegionServer WAL dirs; i.e. this.rootDir/HConstants.HREGION_LOGDIR_NAME. */ public FileStatus[] getWALDirPaths(final PathFilter filter) throws IOException { Path walDirPath = new Path(CommonFSUtils.getWALRootDir(conf), HConstants.HREGION_LOGDIR_NAME); FileStatus[] walDirForServerNames = FSUtils.listStatus( CommonFSUtils.getWALFileSystem(conf), walDirPath, filter); return walDirForServerNames == null? new FileStatus[0]: walDirForServerNames; }
/** * @param c configuration * @return {@link Path} to hbase log root directory: e.g. {@value HBASE_WAL_DIR} from * configuration as a qualified Path. Defaults to HBase root dir. * @throws IOException e */ public static Path getWALRootDir(final Configuration c) throws IOException { Path p = new Path(c.get(HBASE_WAL_DIR, c.get(HConstants.HBASE_DIR))); if (!isValidWALRootDir(p, c)) { return getRootDir(c); } FileSystem fs = p.getFileSystem(c); return p.makeQualified(fs.getUri(), fs.getWorkingDirectory()); }
@Test(expected=IllegalStateException.class) public void testGetWALRootDirIllegalWALDir() throws IOException { Path root = new Path("file:///hbase/root"); Path invalidWALDir = new Path("file:///hbase/root/logroot"); CommonFSUtils.setRootDir(conf, root); CommonFSUtils.setWALRootDir(conf, invalidWALDir); CommonFSUtils.getWALRootDir(conf); }
!(CommonFSUtils.hasCapability(out, "hflush") && CommonFSUtils.hasCapability(out, "hsync"))) { out.close(); throw new CommonFSUtils.StreamLacksCapabilityException("hflush and hsync");
public WALProcedureStore(final Configuration conf, final LeaseRecovery leaseRecovery) throws IOException { this(conf, new Path(CommonFSUtils.getWALRootDir(conf), MASTER_PROCEDURE_LOGDIR), new Path(CommonFSUtils.getRootDir(conf), HConstants.HREGION_OLDLOGDIR_NAME), leaseRecovery); }
/** * @param conf must not be null * @return Returns the filesystem of the hbase rootdir. * @throws IOException from underlying FileSystem */ public static FileSystem getCurrentFileSystem(Configuration conf) throws IOException { return getRootDir(conf).getFileSystem(conf); }
@VisibleForTesting public WALProcedureStore(final Configuration conf, final Path walDir, final Path walArchiveDir, final LeaseRecovery leaseRecovery) throws IOException { this.conf = conf; this.leaseRecovery = leaseRecovery; this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = CommonFSUtils.getWALFileSystem(conf); this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { if (!fs.mkdirs(walDir)) { throw new IOException("Unable to mkdir " + walDir); } } // Now that it exists, set the log policy String storagePolicy = conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); CommonFSUtils.setStoragePolicy(fs, walDir, storagePolicy); // Create archive dir up front. Rename won't work w/o it up on HDFS. if (this.walArchiveDir != null && !this.fs.exists(this.walArchiveDir)) { if (this.fs.mkdirs(this.walArchiveDir)) { LOG.debug("Created Procedure Store WAL archive dir {}", this.walArchiveDir); } else { LOG.warn("Failed create of {}", this.walArchiveDir); } } }
@Test public void testSetWALRootDir() throws Exception { Path p = new Path("file:///hbase/root"); CommonFSUtils.setWALRootDir(conf, p); assertEquals(p.toString(), conf.get(CommonFSUtils.HBASE_WAL_DIR)); }
/** * Sets storage policy for given path. * If the passed path is a directory, we'll set the storage policy for all files * created in the future in said directory. Note that this change in storage * policy takes place at the FileSystem level; it will persist beyond this RS's lifecycle. * If we're running on a version of FileSystem that doesn't support the given storage policy * (or storage policies at all), then we'll issue a log message and continue. * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html * * @param fs We only do anything it implements a setStoragePolicy method * @param path the Path whose storage policy is to be set * @param storagePolicy Policy to set on <code>path</code>; see hadoop 2.6+ * org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g * 'COLD', 'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'. */ public static void setStoragePolicy(final FileSystem fs, final Path path, final String storagePolicy) { try { setStoragePolicy(fs, path, storagePolicy, false); } catch (IOException e) { // should never arrive here LOG.warn("We have chosen not to throw exception but some unexpectedly thrown out", e); } }
/** * Test path compare and prefix checking. */ @Test public void testMatchingTail() throws IOException { Path rootdir = htu.getDataTestDir(); final FileSystem fs = rootdir.getFileSystem(conf); assertTrue(rootdir.depth() > 1); Path partPath = new Path("a", "b"); Path fullPath = new Path(rootdir, partPath); Path fullyQualifiedPath = fs.makeQualified(fullPath); assertFalse(CommonFSUtils.isMatchingTail(fullPath, partPath)); assertFalse(CommonFSUtils.isMatchingTail(fullPath, partPath.toString())); assertTrue(CommonFSUtils.isStartingWithPath(rootdir, fullPath.toString())); assertTrue(CommonFSUtils.isStartingWithPath(fullyQualifiedPath, fullPath.toString())); assertFalse(CommonFSUtils.isStartingWithPath(rootdir, partPath.toString())); assertFalse(CommonFSUtils.isMatchingTail(fullyQualifiedPath, partPath)); assertTrue(CommonFSUtils.isMatchingTail(fullyQualifiedPath, fullPath)); assertTrue(CommonFSUtils.isMatchingTail(fullyQualifiedPath, fullPath.toString())); assertTrue(CommonFSUtils.isMatchingTail(fullyQualifiedPath, fs.makeQualified(fullPath))); assertTrue(CommonFSUtils.isStartingWithPath(rootdir, fullyQualifiedPath.toString())); assertFalse(CommonFSUtils.isMatchingTail(fullPath, new Path("x"))); assertFalse(CommonFSUtils.isMatchingTail(new Path("x"), fullPath)); }
CommonFSUtils.setStoragePolicy(fs, this.walDir, storagePolicy); this.walFileSuffix = (suffix == null) ? "" : URLEncoder.encode(suffix, "UTF8"); this.prefixPathStr = new Path(walDir, walFilePrefix + WAL_FILE_NAME_DELIMITER).toString(); final FileStatus[] walFiles = CommonFSUtils.listStatus(fs, walDir, ourFiles); if (null != walFiles && 0 != walFiles.length) { throw new IOException("Target WAL already exists within directory " + walDir);
/** * Compare path component of the Path URI; e.g. if hdfs://a/b/c and /a/b/c, it will compare the * '/a/b/c' part. Does not consider schema; i.e. if schemas different but path or subpath matches, * the two will equate. * @param pathToSearch Path we will be trying to match against. * @param pathTail what to match * @return True if <code>pathTail</code> is tail on the path of <code>pathToSearch</code> */ public static boolean isMatchingTail(final Path pathToSearch, String pathTail) { return isMatchingTail(pathToSearch, new Path(pathTail)); }
/** * Create an entry stream over the given queue at the given start position * @param logQueue the queue of WAL paths * @param fs {@link FileSystem} to use to create {@link Reader} for this stream * @param conf {@link Configuration} to use to create {@link Reader} for this stream * @param startPosition the position in the first WAL to start reading at * @param serverName the server name which all WALs belong to * @param metrics replication metrics * @throws IOException */ public WALEntryStream(PriorityBlockingQueue<Path> logQueue, Configuration conf, long startPosition, WALFileLengthProvider walFileLengthProvider, ServerName serverName, MetricsSource metrics) throws IOException { this.logQueue = logQueue; this.fs = CommonFSUtils.getWALFileSystem(conf); this.conf = conf; this.currentPositionOfEntry = startPosition; this.walFileLengthProvider = walFileLengthProvider; this.serverName = serverName; this.metrics = metrics; }
@Override protected AsyncFSWAL createWAL() throws IOException { return new AsyncFSWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null, eventLoopGroup, channelClass); }
@Test(expected=IllegalStateException.class) public void testGetWALRootDirIllegalWALDir() throws IOException { Path root = new Path("file:///hbase/root"); Path invalidWALDir = new Path("file:///hbase/root/logroot"); CommonFSUtils.setRootDir(conf, root); CommonFSUtils.setWALRootDir(conf, invalidWALDir); CommonFSUtils.getWALRootDir(conf); }
@Test(expected=NullPointerException.class) public void streamCapabilitiesDoesNotAllowNullStream() { CommonFSUtils.hasCapability(null, "hopefully any string"); }
public static FileSystem getRootDirFileSystem(final Configuration c) throws IOException { Path p = getRootDir(c); return p.getFileSystem(c); }