/** * Sets storage policy for given path. * If the passed path is a directory, we'll set the storage policy for all files * created in the future in said directory. Note that this change in storage * policy takes place at the FileSystem level; it will persist beyond this RS's lifecycle. * If we're running on a version of FileSystem that doesn't support the given storage policy * (or storage policies at all), then we'll issue a log message and continue. * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html * * @param fs We only do anything it implements a setStoragePolicy method * @param path the Path whose storage policy is to be set * @param storagePolicy Policy to set on <code>path</code>; see hadoop 2.6+ * org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g * 'COLD', 'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'. */ public static void setStoragePolicy(final FileSystem fs, final Path path, final String storagePolicy) { try { setStoragePolicy(fs, path, storagePolicy, false); } catch (IOException e) { // should never arrive here LOG.warn("We have chosen not to throw exception but some unexpectedly thrown out", e); } }
@VisibleForTesting public WALProcedureStore(final Configuration conf, final Path walDir, final Path walArchiveDir, final LeaseRecovery leaseRecovery) throws IOException { this.conf = conf; this.leaseRecovery = leaseRecovery; this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = CommonFSUtils.getWALFileSystem(conf); this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { if (!fs.mkdirs(walDir)) { throw new IOException("Unable to mkdir " + walDir); } } // Now that it exists, set the log policy String storagePolicy = conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); CommonFSUtils.setStoragePolicy(fs, walDir, storagePolicy); // Create archive dir up front. Rename won't work w/o it up on HDFS. if (this.walArchiveDir != null && !this.fs.exists(this.walArchiveDir)) { if (this.fs.mkdirs(this.walArchiveDir)) { LOG.debug("Created Procedure Store WAL archive dir {}", this.walArchiveDir); } else { LOG.warn("Failed create of {}", this.walArchiveDir); } } }
CommonFSUtils.setStoragePolicy(fs, this.walDir, storagePolicy); this.walFileSuffix = (suffix == null) ? "" : URLEncoder.encode(suffix, "UTF8"); this.prefixPathStr = new Path(walDir, walFilePrefix + WAL_FILE_NAME_DELIMITER).toString();
/** * Sets storage policy for given path. * If the passed path is a directory, we'll set the storage policy for all files * created in the future in said directory. Note that this change in storage * policy takes place at the FileSystem level; it will persist beyond this RS's lifecycle. * If we're running on a version of FileSystem that doesn't support the given storage policy * (or storage policies at all), then we'll issue a log message and continue. * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html * * @param fs We only do anything it implements a setStoragePolicy method * @param path the Path whose storage policy is to be set * @param storagePolicy Policy to set on <code>path</code>; see hadoop 2.6+ * org.apache.hadoop.hdfs.protocol.HdfsConstants for possible list e.g * 'COLD', 'WARM', 'HOT', 'ONE_SSD', 'ALL_SSD', 'LAZY_PERSIST'. */ public static void setStoragePolicy(final FileSystem fs, final Path path, final String storagePolicy) { try { setStoragePolicy(fs, path, storagePolicy, false); } catch (IOException e) { // should never arrive here LOG.warn("We have chosen not to throw exception but some unexpectedly thrown out", e); } }
/** * Sets storage policy for given path according to config setting. * If the passed path is a directory, we'll set the storage policy for all files * created in the future in said directory. Note that this change in storage * policy takes place at the FileSystem level; it will persist beyond this RS's lifecycle. * If we're running on a FileSystem implementation that doesn't support the given storage policy * (or storage policies at all), then we'll issue a log message and continue. * * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html * * @param fs We only do anything it implements a setStoragePolicy method * @param conf used to look up storage policy with given key; not modified. * @param path the Path whose storage policy is to be set * @param policyKey Key to use pulling a policy from Configuration: * e.g. HConstants.WAL_STORAGE_POLICY (hbase.wal.storage.policy). * @param defaultPolicy if the configured policy is equal to this policy name, we will skip * telling the FileSystem to set a storage policy. */ public static void setStoragePolicy(final FileSystem fs, final Configuration conf, final Path path, final String policyKey, final String defaultPolicy) { String storagePolicy = conf.get(policyKey, defaultPolicy).toUpperCase(Locale.ROOT); if (storagePolicy.equals(defaultPolicy)) { if (LOG.isTraceEnabled()) { LOG.trace("default policy of " + defaultPolicy + " requested, exiting early."); } return; } setStoragePolicy(fs, path, storagePolicy); }
@VisibleForTesting public WALProcedureStore(final Configuration conf, final Path walDir, final Path walArchiveDir, final LeaseRecovery leaseRecovery) throws IOException { this.conf = conf; this.leaseRecovery = leaseRecovery; this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = walDir.getFileSystem(conf); this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { if (!fs.mkdirs(walDir)) { throw new IOException("Unable to mkdir " + walDir); } } // Now that it exists, set the log policy CommonFSUtils.setStoragePolicy(fs, conf, walDir, HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); // Create archive dir up front. Rename won't work w/o it up on HDFS. if (this.walArchiveDir != null && !this.fs.exists(this.walArchiveDir)) { if (this.fs.mkdirs(this.walArchiveDir)) { if (LOG.isDebugEnabled()) { LOG.debug("Created Procedure Store WAL archive dir " + this.walArchiveDir); } } else { LOG.warn("Failed create of " + this.walArchiveDir); } } }
@VisibleForTesting public WALProcedureStore(final Configuration conf, final Path walDir, final Path walArchiveDir, final LeaseRecovery leaseRecovery) throws IOException { this.conf = conf; this.leaseRecovery = leaseRecovery; this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = walDir.getFileSystem(conf); this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { if (!fs.mkdirs(walDir)) { throw new IOException("Unable to mkdir " + walDir); } } // Now that it exists, set the log policy String storagePolicy = conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); CommonFSUtils.setStoragePolicy(fs, walDir, storagePolicy); // Create archive dir up front. Rename won't work w/o it up on HDFS. if (this.walArchiveDir != null && !this.fs.exists(this.walArchiveDir)) { if (this.fs.mkdirs(this.walArchiveDir)) { LOG.debug("Created Procedure Store WAL archive dir {}", this.walArchiveDir); } else { LOG.warn("Failed create of {}", this.walArchiveDir); } } }