/** * Open and close a TableOutputFormat. The closing the RecordWriter should release HBase * Connection (ZK) resources, and will throw exception if they are exhausted. */ static void openCloseTableOutputFormat(int iter) throws IOException { LOG.info("Instantiating TableOutputFormat connection " + iter); JobConf conf = new JobConf(); conf.addResource(UTIL.getConfiguration()); conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE); TableMapReduceUtil.initTableMapJob(TABLE, FAMILY, TableMap.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, conf); TableOutputFormat tof = new TableOutputFormat(); RecordWriter rw = tof.getRecordWriter(null, conf, TABLE, null); rw.close(null); }
conf.addResource(new Path(jobConfFileName));
conf.addResource(new Path(jobConfFileName));
conf.addResource(new Path(jobConfFileName));
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); }
public JobConf getLocalConf() { JobConf jc = new JobConf(false); jc.addResource(CORE_DEFAULT_SITE); jc.addResource(MAPRED_DEFAULT_SITE); jc.addResource(YARN_DEFAULT_SITE); return jc; }
public static JobConf config() {// Hadoop集群的远程配置信息 JobConf conf = new JobConf(MainRun.class); conf.setJobName("MartrixMultiply"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); return conf; }
public static JobConf config() { JobConf conf = new JobConf(HdfsDAO.class); conf.setJobName("HdfsDAO"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); return conf; }
public static JobConf config() {// Hadoop集群的远程配置信息 JobConf conf = new JobConf(PageRankJob.class); conf.setJobName("PageRank"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); return conf; }
public static JobConf config() {// Hadoop集群的远程配置信息 JobConf conf = new JobConf(Purchase.class); conf.setJobName("purchase"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); return conf; }
public static JobConf config() {// Hadoop集群的远程配置信息 JobConf conf = new JobConf(Purchase.class); conf.setJobName("purchase"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); return conf; }
public static JobConf config() { JobConf conf = new JobConf(Recommend.class); conf.setJobName("Recommand"); conf.addResource("classpath:/hadoop/core-site.xml"); conf.addResource("classpath:/hadoop/hdfs-site.xml"); conf.addResource("classpath:/hadoop/mapred-site.xml"); conf.set("io.sort.mb", "1024"); return conf; }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
@Override public JobConf getExecConf(Properties properties) throws ExecException { JobConf jc = super.getExecConf(properties); jc.addResource(TezConfiguration.TEZ_SITE_XML); return jc; } }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
/** Construct a map/reduce configuration. * * @param config a Configuration-format XML job description file. */ public JobConf(Path config) { super(); addResource(config); checkAndWarnDeprecation(); }
private static JobConf createSleepMapperReducerJobConf() { JobConf jConf = new JobConf(true); jConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml"))); jConf.setMapperClass(SleepMapperReducerForTest.class); jConf.setReducerClass(SleepMapperReducerForTest.class); jConf.setOutputKeyClass(Text.class); jConf.setOutputValueClass(IntWritable.class); jConf.setInputFormat(TextInputFormat.class); jConf.setOutputFormat(TextOutputFormat.class); jConf.setNumReduceTasks(1); jConf.set(SleepMapperReducerForTest.SLEEP_TIME_MILLIS_KEY, "60000"); return jConf; }