public static void setClassLoaderAndJar(JobConf conf, Class jobClass) { conf.setClassLoader(Thread.currentThread().getContextClassLoader()); String jar = HadoopUtils.findContainingJar(jobClass, Thread.currentThread() .getContextClassLoader()); if(jar != null) { conf.setJar(jar); } } }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf, HadoopArchives.class); } // This is for test purposes since MR2, different from Streaming // here it is not possible to add a JAR to the classpath the tool // will when running the mapreduce job. String testJar = System.getProperty(TEST_HADOOP_ARCHIVES_JAR_PATH, null); if (testJar != null) { this.conf.setJar(testJar); } }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf, HadoopArchives.class); } // This is for test purposes since MR2, different from Streaming // here it is not possible to add a JAR to the classpath the tool // will when running the mapreduce job. String testJar = System.getProperty(TEST_HADOOP_ARCHIVES_JAR_PATH, null); if (testJar != null) { this.conf.setJar(testJar); } }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf, HadoopArchives.class); } // This is for test purposes since MR2, different from Streaming // here it is not possible to add a JAR to the classpath the tool // will when running the mapreduce job. String testJar = System.getProperty(TEST_HADOOP_ARCHIVES_JAR_PATH, null); if (testJar != null) { this.conf.setJar(testJar); } }
makeJarAvailableInSharedCache(jobJar, fileUploader); jobConf.setJar(jobJar.toString());
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = ClassUtil.findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job jar */ public void setJar(String jar) { ensureState(JobState.DEFINE); conf.setJar(jar); }
/** * Set the job jar */ public void setJar(String jar) { ensureState(JobState.DEFINE); conf.setJar(jar); }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = ClassUtil.findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job jar */ public void setJar(String jar) { ensureState(JobState.DEFINE); conf.setJar(jar); }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job jar */ public void setJar(String jar) { ensureState(JobState.DEFINE); conf.setJar(jar); }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = ClassUtil.findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job jar */ public void setJar(String jar) { ensureState(JobState.DEFINE); conf.setJar(jar); }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = ClassUtil.findContainingJar(cls); if (jar != null) { setJar(jar); } }
/** * Set the job's jar file by finding an example class location. * * @param cls the example class. */ public void setJarByClass(Class cls) { String jar = findContainingJar(cls); if (jar != null) { setJar(jar); } }
Configuration config = new Configuration(); config.set("fs.default.name", "hdfs://ip:port"); config.set("mapred.job.tracker", "hdfs://ip:port"); JobConf job = new JobConf(config); job.setJar("target/sample.jar");
static RunningJob launchJob(JobConf jobConf, Path inDir, Path outputPath, int numMaps, String jobName) throws IOException { jobConf.setJobName(jobName); jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class); jobConf.setOutputFormat(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(jobConf, inDir); FileOutputFormat.setOutputPath(jobConf, outputPath); jobConf.setMapperClass(IdentityMapper.class); jobConf.setReducerClass(IdentityReducer.class); jobConf.setOutputKeyClass(BytesWritable.class); jobConf.setOutputValueClass(BytesWritable.class); jobConf.setNumMapTasks(numMaps); jobConf.setNumReduceTasks(0); jobConf.setJar("build/test/mapred/testjar/testjob.jar"); return JobClient.runJob(jobConf); } }