/** * Submit a job to the map/reduce cluster. All of the necessary modifications * to the job to run under pipes are made to the configuration. * @param conf the job to submit to the cluster (MODIFIED) * @throws IOException * @deprecated Use {@link Submitter#runJob(JobConf)} */ @Deprecated public static RunningJob submitJob(JobConf conf) throws IOException { return runJob(conf); }
@SuppressWarnings("unchecked") public void configure(JobConf conf) { part = ReflectionUtils.newInstance(Submitter.getJavaPartitioner(conf), conf); }
GenericOptionsParser genericParser = new GenericOptionsParser(getConf(), args); CommandLine results = parser.parse(cli.options, genericParser.getRemainingArgs()); JobConf job = new JobConf(getConf()); setIsJavaRecordReader(job, true); job.setInputFormat(getClass(results, "inputformat", job, InputFormat.class)); setIsJavaRecordReader(job, true); setIsJavaMapper(job, true); job.setMapperClass(getClass(results, "map", job, Mapper.class)); job.setPartitionerClass(getClass(results, "partitioner", job, Partitioner.class)); setIsJavaReducer(job, true); job.setReducerClass(getClass(results, "reduce", job, Reducer.class)); setIsJavaRecordWriter(job, true); job.setOutputFormat(getClass(results, "writer", job, OutputFormat.class)); setExecutable(job, results.getOptionValue("program")); runJob(job); return 0; } catch (ParseException pe) {
private static void setupPipesJob(JobConf conf) throws IOException { if (!getIsJavaMapper(conf)) { conf.setMapRunnerClass(PipesMapRunner.class); setJavaPartitioner(conf, conf.getPartitionerClass()); conf.setPartitionerClass(PipesPartitioner.class); if (!getIsJavaReducer(conf)) { conf.setReducerClass(PipesReducer.class); if (!getIsJavaRecordWriter(conf)) { conf.setOutputFormat(NullOutputFormat.class); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_VALUE_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_VALUE_CLASS, textClassname); if (!getIsJavaRecordReader(conf) && !getIsJavaMapper(conf)) { conf.setClass(Submitter.INPUT_FORMAT, conf.getInputFormat().getClass(), InputFormat.class); String exec = getExecutable(conf); if (exec == null) { throw new IllegalArgumentException("No application program defined."); setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);
Submitter.setIsJavaMapper(conf, false); Submitter.setIsJavaReducer(conf, false); Submitter.setKeepCommandFile(conf, false); Submitter.setIsJavaRecordReader(conf, false); Submitter.setIsJavaRecordWriter(conf, false); PipesPartitioner<IntWritable, Text> partitioner = new PipesPartitioner<IntWritable, Text>(); partitioner.configure(conf); Submitter.setJavaPartitioner(conf, partitioner.getClass()); assertEquals(PipesPartitioner.class, (Submitter.getJavaPartitioner(conf))); Submitter.main(new String[0]); fail(); } catch (ExitUtil.ExitException e) { args[21] = "mapreduce.pipes.isjavarecordwriter=false,mapreduce.pipes.isjavarecordreader=false"; Submitter.main(args); fail(); } catch (ExitUtil.ExitException e) {
fs.delete(wordExec.getParent(), true); fs.copyFromLocalFile(program, wordExec); Submitter.setExecutable(job, fs.makeQualified(wordExec).toString()); Submitter.setIsJavaRecordReader(job, true); Submitter.setIsJavaRecordWriter(job, true); FileInputFormat.setInputPaths(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); RunningJob rJob = null; if (numReduces == 0) { rJob = Submitter.jobSubmit(job); rJob = Submitter.runJob(job);
dfs.getFileSystem().makeQualified(wordExec)); try { int ret = ToolRunner.run(new Submitter(), new String[]{"-conf", jobXml.toString(), "-input", inDir.toString(),
GenericOptionsParser genericParser = new GenericOptionsParser(getConf(), args); CommandLine results = parser.parse(cli.options, genericParser.getRemainingArgs()); JobConf job = new JobConf(getConf()); setIsJavaRecordReader(job, true); job.setInputFormat(getClass(results, "inputformat", job, InputFormat.class)); setIsJavaRecordReader(job, true); setIsJavaMapper(job, true); job.setMapperClass(getClass(results, "map", job, Mapper.class)); job.setPartitionerClass(getClass(results, "partitioner", job, Partitioner.class)); setIsJavaReducer(job, true); job.setReducerClass(getClass(results, "reduce", job, Reducer.class)); setIsJavaRecordWriter(job, true); job.setOutputFormat(getClass(results, "writer", job, OutputFormat.class)); setExecutable(job, (String) results.getOptionValue("program")); runJob(job); return 0;
private static void setupPipesJob(JobConf conf) throws IOException { if (!getIsJavaMapper(conf)) { conf.setMapRunnerClass(PipesMapRunner.class); setJavaPartitioner(conf, conf.getPartitionerClass()); conf.setPartitionerClass(PipesPartitioner.class); if (!getIsJavaReducer(conf)) { conf.setReducerClass(PipesReducer.class); if (!getIsJavaRecordWriter(conf)) { conf.setOutputFormat(NullOutputFormat.class); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_VALUE_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_VALUE_CLASS, textClassname); if (!getIsJavaRecordReader(conf) && !getIsJavaMapper(conf)) { conf.setClass(Submitter.INPUT_FORMAT, conf.getInputFormat().getClass(), InputFormat.class); String exec = getExecutable(conf); if (exec == null) { throw new IllegalArgumentException("No application program defined."); setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);
fs.delete(wordExec.getParent(), true); fs.copyFromLocalFile(program, wordExec); Submitter.setExecutable(job, fs.makeQualified(wordExec).toString()); Submitter.setIsJavaRecordReader(job, true); Submitter.setIsJavaRecordWriter(job, true); FileInputFormat.setInputPaths(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); RunningJob rJob = null; if (numReduces == 0) { rJob = Submitter.jobSubmit(job); rJob = Submitter.runJob(job);
dfs.getFileSystem().makeQualified(wordExec)); try { int ret = ToolRunner.run(new Submitter(), new String[]{"-conf", jobXml.toString(), "-input", inDir.toString(),
GenericOptionsParser genericParser = new GenericOptionsParser(getConf(), args); CommandLine results = parser.parse(cli.options, genericParser.getRemainingArgs()); JobConf job = new JobConf(getConf()); setIsJavaRecordReader(job, true); job.setInputFormat(getClass(results, "inputformat", job, InputFormat.class)); setIsJavaRecordReader(job, true); setIsJavaMapper(job, true); job.setMapperClass(getClass(results, "map", job, Mapper.class)); job.setPartitionerClass(getClass(results, "partitioner", job, Partitioner.class)); setIsJavaReducer(job, true); job.setReducerClass(getClass(results, "reduce", job, Reducer.class)); setIsJavaRecordWriter(job, true); job.setOutputFormat(getClass(results, "writer", job, OutputFormat.class)); setExecutable(job, results.getOptionValue("program")); runJob(job); return 0; } catch (ParseException pe) {
private static void setupPipesJob(JobConf conf) throws IOException { if (!getIsJavaMapper(conf)) { conf.setMapRunnerClass(PipesMapRunner.class); setJavaPartitioner(conf, conf.getPartitionerClass()); conf.setPartitionerClass(PipesPartitioner.class); if (!getIsJavaReducer(conf)) { conf.setReducerClass(PipesReducer.class); if (!getIsJavaRecordWriter(conf)) { conf.setOutputFormat(NullOutputFormat.class); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_VALUE_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_VALUE_CLASS, textClassname); if (!getIsJavaRecordReader(conf) && !getIsJavaMapper(conf)) { conf.setClass(Submitter.INPUT_FORMAT, conf.getInputFormat().getClass(), InputFormat.class); String exec = getExecutable(conf); if (exec == null) { throw new IllegalArgumentException("No application program defined."); setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);
/** * Submit a job to the map/reduce cluster. All of the necessary modifications * to the job to run under pipes are made to the configuration. * @param conf the job to submit to the cluster (MODIFIED) * @throws IOException * @deprecated Use {@link Submitter#runJob(JobConf)} */ @Deprecated public static RunningJob submitJob(JobConf conf) throws IOException { return runJob(conf); }
@SuppressWarnings("unchecked") public void configure(JobConf conf) { part = ReflectionUtils.newInstance(Submitter.getJavaPartitioner(conf), conf); }
GenericOptionsParser genericParser = new GenericOptionsParser(getConf(), args); CommandLine results = parser.parse(cli.options, genericParser.getRemainingArgs()); JobConf job = new JobConf(getConf()); setIsJavaRecordReader(job, true); job.setInputFormat(getClass(results, "inputformat", job, InputFormat.class)); setIsJavaRecordReader(job, true); setIsJavaMapper(job, true); job.setMapperClass(getClass(results, "map", job, Mapper.class)); job.setPartitionerClass(getClass(results, "partitioner", job, Partitioner.class)); setIsJavaReducer(job, true); job.setReducerClass(getClass(results, "reduce", job, Reducer.class)); setIsJavaRecordWriter(job, true); job.setOutputFormat(getClass(results, "writer", job, OutputFormat.class)); setExecutable(job, results.getOptionValue("program")); runJob(job); return 0; } catch (ParseException pe) {
private static void setupPipesJob(JobConf conf) throws IOException { if (!getIsJavaMapper(conf)) { conf.setMapRunnerClass(PipesMapRunner.class); setJavaPartitioner(conf, conf.getPartitionerClass()); conf.setPartitionerClass(PipesPartitioner.class); if (!getIsJavaReducer(conf)) { conf.setReducerClass(PipesReducer.class); if (!getIsJavaRecordWriter(conf)) { conf.setOutputFormat(NullOutputFormat.class); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.MAP_OUTPUT_VALUE_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_KEY_CLASS, textClassname); setIfUnset(conf, MRJobConfig.OUTPUT_VALUE_CLASS, textClassname); if (!getIsJavaRecordReader(conf) && !getIsJavaMapper(conf)) { conf.setClass(Submitter.INPUT_FORMAT, conf.getInputFormat().getClass(), InputFormat.class); String exec = getExecutable(conf); if (exec == null) { throw new IllegalArgumentException("No application program defined."); setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript); setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);