/** * Retrieve JHS bind address from configuration * * @param conf * @return InetSocketAddress */ public static InetSocketAddress getBindAddress(Configuration conf) { return conf.getSocketAddr(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS), conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_PORT, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_PORT)); }
static String getSargColumnIDsString(Configuration conf) { return conf.getBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, true) ? null : conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); }
private String getNonEmptyConfVar(Configuration conf, String param) throws IOException { String val = conf.get(param); if (val == null || val.trim().isEmpty()) { throw new IOException("Configuration parameter " + param + " should be set, " + WHEN_ZK_DSTORE_MSG); } return val; }
public Job createSubmittableJob(String[] args) throws IOException { FileSystem fs = sourceHashDir.getFileSystem(getConf()); if (!fs.exists(sourceHashDir)) { throw new IOException("Source hash dir not found: " + sourceHashDir); Path dataDir = new Path(sourceHashDir, HashTable.HASH_DATA_DIR); int dataSubdirCount = 0; for (FileStatus file : fs.listStatus(dataDir)) { Job job = Job.getInstance(getConf(),getConf().get("mapreduce.job.name", "syncTable_" + sourceTableName + "-" + targetTableName)); Configuration jobConf = job.getConfiguration(); job.setJarByClass(HashTable.class); jobConf.set(SOURCE_HASH_DIR_CONF_KEY, sourceHashDir.toString()); jobConf.set(SOURCE_TABLE_CONF_KEY, sourceTableName); jobConf.set(TARGET_TABLE_CONF_KEY, targetTableName); if (sourceZkCluster != null) { jobConf.set(SOURCE_ZK_CLUSTER_CONF_KEY, sourceZkCluster);
/** * commit the task by moving the output file out from the temporary directory. * @throws java.io.IOException */ @Override public void close() throws IOException { // enforce sequential close() calls synchronized (CLOSE_MUTEX) { try { this.recordWriter.close(this.context); } catch (InterruptedException e) { throw new IOException("Could not close RecordReader.", e); } if (this.outputCommitter.needsTaskCommit(this.context)) { this.outputCommitter.commitTask(this.context); } Path outputPath = new Path(this.configuration.get("mapred.output.dir")); // rename tmp-file to final name FileSystem fs = FileSystem.get(outputPath.toUri(), this.configuration); String taskNumberStr = Integer.toString(this.taskNumber); String tmpFileTemplate = "tmp-r-00000"; String tmpFile = tmpFileTemplate.substring(0, 11 - taskNumberStr.length()) + taskNumberStr; if (fs.exists(new Path(outputPath.toString() + "/" + tmpFile))) { fs.rename(new Path(outputPath.toString() + "/" + tmpFile), new Path(outputPath.toString() + "/" + taskNumberStr)); } } }
protected String getUseThisHostnameInstead(Configuration conf) throws IOException { String hostname = conf.get(RS_HOSTNAME_KEY); if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) { if (!StringUtils.isBlank(hostname)) { String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY + " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " to true while " + RS_HOSTNAME_KEY + " is used"; throw new IOException(msg); } else { return rpcServices.isa.getHostName(); } } else { return hostname; } }
throws IOException { TableName tableName = TableName.valueOf(args[0]); conf.set(TABLE_NAME, tableName.getNameAsString()); Path inputDir = new Path(args[1]); Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); job.setJarByClass(Importer.class); FileInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(SequenceFileInputFormat.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); throw new IOException(e); if (hfileOutPath != null && conf.getBoolean(HAS_LARGE_RESULT, false)) { LOG.info("Use Large Result!!"); try (Connection conn = ConnectionFactory.createConnection(conf); job.setMapperClass(CellSortImporter.class); job.setReducerClass(CellReducer.class); Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputKeyClass(CellWritableComparable.class); RawComparator.class); Path partitionsPath = new Path(TotalOrderPartitioner.getPartitionFile(job.getConfiguration())); FileSystem fs = FileSystem.get(job.getConfiguration()); fs.deleteOnExit(partitionsPath);
String tabName = args[1]; conf.setStrings(TABLES_KEY, tabName); conf.set(FileInputFormat.INPUT_DIR, inputDirs); Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + EnvironmentEdgeManager.currentTime())); job.setJarByClass(MapReduceHFileSplitterJob.class); job.setInputFormatClass(HFileInputFormat.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); job.setMapperClass(HFileCellMapper.class); job.setReducerClass(CellSortReducer.class); Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); org.apache.hbase.thirdparty.com.google.common.base.Preconditions.class); } else { throw new IOException("No bulk output directory specified");
if (hadoopConfig.get("hadoop.tmp.dir") == null) { String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(flinkConfig); File tmpDir = new File(tmpDirPaths[0], "hadoop-" + System.getProperty("user.name")); hadoopConfig.set("hadoop.tmp.dir", tmpDir.getPath()); String value = flinkConfig.getString(key, null); String newKey = "fs.swift." + key.substring(CONFIG_PREFIX.length()); hadoopConfig.set(newKey, value); + " Using Hadoop configuration from the classpath."); hadoopConfig = new org.apache.hadoop.conf.Configuration(); this.hadoopConfig = hadoopConfig; throw new IOException(e.getMessage(), e);
@Override public void start(CoprocessorEnvironment env) throws IOException { RegionCoprocessorEnvironment renv = (RegionCoprocessorEnvironment) env; try { this.cache = ((ZKDataHolder) renv.getSharedData().computeIfAbsent(ZKKEY, k -> { String ensemble = renv.getConfiguration().get(ZK_ENSEMBLE_KEY); int sessionTimeout = renv.getConfiguration().getInt(ZK_SESSION_TIMEOUT_KEY, ZK_SESSION_TIMEOUT_DEFAULT); return new ZKDataHolder(ensemble, sessionTimeout); })).acquire(); } catch (Exception e) { throw new IOException(e); } }
@VisibleForTesting org.apache.hadoop.conf.Configuration getHadoopConfiguration() { org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); if (flinkConfig == null) { return conf; } // read all configuration with prefix 'FLINK_CONFIG_PREFIXES' for (String key : flinkConfig.keySet()) { for (String prefix : FLINK_CONFIG_PREFIXES) { if (key.startsWith(prefix)) { String value = flinkConfig.getString(key, null); conf.set(key, value); if (CONFIG_KEYS_TO_SHADE.contains(key)) { conf.set(key, FLINK_SHADING_PREFIX + value); } LOG.debug("Adding Flink config entry for {} as {} to Hadoop config", key, conf.get(key)); } } } return conf; } }
static void setWorkOutputPath(TaskAttemptContext context) throws IOException { String outputPath = context.getConfiguration().get("mapred.output.dir"); //we need to do this to get the task path and set it for mapred implementation //since it can't be done automatically because of mapreduce->mapred abstraction if (outputPath != null) context.getConfiguration().set("mapred.work.output.dir", new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString()); } }
/** * Get the general working directory for snapshots - where they are built, where they are * temporarily copied on export, etc. * @param rootDir root directory of the HBase installation * @param conf Configuration of the HBase instance * @return Path to the snapshot tmp directory, relative to the passed root directory */ public static Path getWorkingSnapshotDir(final Path rootDir, final Configuration conf) { return new Path(conf.get(SNAPSHOT_WORKING_DIR, getDefaultWorkingSnapshotDir(rootDir).toString())); }
private static void addInputPath(Job job, Iterable<String> pathStrings, Class<? extends InputFormat> inputFormatClass) { Configuration conf = job.getConfiguration(); StringBuilder inputFormats = new StringBuilder( StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS)) ); String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { if (inputFormats.length() > 0) { inputFormats.append(','); } inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName()); } if (inputFormats.length() > 0) { conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString()); } // add last one separately for possible initialization in MultipleInputs MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass); }
/** * Creates the HDFS filesystem to store output files. * * @param conf Hadoop configuration */ private void createHdfsFilesystem(Configuration conf) throws Exception { // Inits HDFS file system object mFileSystem = FileSystem.get(URI.create(conf.get("fs.defaultFS")), conf); mOutputFilePath = new Path("./MapReduceOutputFile"); if (mFileSystem.exists(mOutputFilePath)) { mFileSystem.delete(mOutputFilePath, true); } }
private static Configuration getConfiguration(VoldemortConfig voldemortConfig, Path source) { String sourceScheme = source.toUri().getScheme(); final Configuration hadoopConfig = new Configuration(); hadoopConfig.setInt(ConfigurableSocketFactory.SO_RCVBUF, voldemortConfig.getFetcherBufferSize()); hadoopConfig.setInt(ConfigurableSocketFactory.SO_TIMEOUT, voldemortConfig.getFetcherSocketTimeout()); hadoopConfig.set("hadoop.rpc.socket.factory.class.ClientProtocol", ConfigurableSocketFactory.class.getName()); hadoopConfig.set("hadoop.security.group.mapping", "org.apache.hadoop.security.ShellBasedUnixGroupsMapping"); hadoopConfig.addResource(new Path(hadoopConfigPath + "/core-site.xml")); hadoopConfig.addResource(new Path(hadoopConfigPath + "/hdfs-site.xml")); String security = hadoopConfig.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
private void setupConf() { for (String f : LlapDaemonConfiguration.DAEMON_CONFIGS) { conf.addResource(f); } conf.reloadConfiguration(); // Setup timeouts for various services. // Once we move to a Hadoop-2.8 dependency, the following paramteer can be used. // conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC); conf.set("yarn.timeline-service.entity-group-fs-store.retry-policy-spec", conf.get(CONFIG_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC, CONFIG_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC_DEFAULT)); conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, conf.getLong(CONFIG_YARN_RM_TIMEOUT_MAX_WAIT_MS, CONFIG_YARN_RM_TIMEOUT_MAX_WAIT_MS_DEFAULT)); conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, conf.getLong(CONFIG_YARN_RM_RETRY_INTERVAL_MS, CONFIG_YARN_RM_RETRY_INTERVAL_MS_DEFAULT)); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(CONFIG_IPC_CLIENT_CONNECT_MAX_RETRIES, CONFIG_IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT)); conf.setLong(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY, conf.getLong(CONFIG_IPC_CLIENT_CONNECT_RETRY_INTERVAL_MS, CONFIG_IPC_CLIENT_CONNECT_RETRY_INTERVAL_MS_DEFAULT)); HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, ( conf.getLong(CONFIG_LLAP_ZK_REGISTRY_TIMEOUT_MS, CONFIG_LLAP_ZK_REGISTRY_TIMEOUT_MS_DEFAULT) + "ms")); llapRegistryConf = new Configuration(conf); }
static Configuration initHBaseConfiguration(Configuration jobConf) { // setup hraven hbase connection information Configuration hbaseConf = null; String hravenDir = System.getenv(HRAVEN_HBASE_CONF_DIR_ENV); if (hravenDir != null && !hravenDir.isEmpty()) { hbaseConf = new Configuration(false); Path hbaseConfFile = new Path(hravenDir, HRAVEN_HBASE_CONF_FILE); LOG.info("Loading hRaven HBase configuration from " + hbaseConfFile.toString()); hbaseConf.addResource(hbaseConfFile); } else { hbaseConf = HBaseConfiguration.create(); // override any hRaven connection properties from the job String hbaseZKQuorum = jobConf.get(HRAVEN_ZOOKEEPER_QUORUM); if (hbaseZKQuorum != null && !hbaseZKQuorum.isEmpty()) { try { LOG.info("Applying hRaven Zookeeper quorum information from job conf: " + hbaseZKQuorum); ZKUtil.applyClusterKeyToConf(hbaseConf, hbaseZKQuorum); } catch (IOException ioe) { throw new RuntimeException("Invalid cluster configuration for " + HRAVEN_ZOOKEEPER_QUORUM + " (\"" + hbaseZKQuorum + "\")"); } } else if (LOG.isDebugEnabled()) { LOG.debug("No cluster configuration found for " + HRAVEN_ZOOKEEPER_QUORUM + ", continuing with default"); } } return hbaseConf; }
BackupManager.decorateMasterConfiguration(conf1); BackupManager.decorateRegionServerConfiguration(conf1); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); TEST_UTIL2 = new HBaseTestingUtility(conf2); TEST_UTIL2.setZkCluster(TEST_UTIL.getZkCluster()); TEST_UTIL2.startMiniDFSCluster(3); String root2 = TEST_UTIL2.getConfiguration().get("fs.defaultFS"); Path p = new Path(new Path(root2), "/tmp/wal"); CommonFSUtils.setWALRootDir(TEST_UTIL2.getConfiguration(), p); TEST_UTIL2.startMiniCluster(); new Path(new Path(TEST_UTIL.getConfiguration().get("fs.defaultFS")), BACKUP_ROOT_DIR).toString(); LOG.info("ROOTDIR " + BACKUP_ROOT_DIR); if (useSecondCluster) { BACKUP_REMOTE_ROOT_DIR = new Path(new Path(TEST_UTIL2.getConfiguration().get("fs.defaultFS")) + BACKUP_REMOTE_ROOT_DIR).toString(); LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);