/** {@inheritDoc} */ @Override public void configure(Job job) throws IOException { super.configure(job); FileOutputFormat.setOutputPath(job, mFilePath); job.setNumReduceTasks(mNumSplits); } }
/** * Configures the output for a MapReduce job. * * @param job The job to configure. * @throws IOException If there is an error. */ public void configure(Job job) throws IOException { job.setOutputFormatClass(getOutputFormatClass()); }
/** * Creates a job output from a command-line flag space-separated parameters map. * * @param ssm Space-separated parameters map from the command-line. * @return a job output configured from the command-line parameters. * @throws IOException on I/O error. */ public MapReduceJobOutput fromSpaceSeparatedMap(String ssm) throws IOException { final Map<String, String> params = SpaceSeparatedMapParser.create().parse(ssm); final String format = params.get(JobIOConfKeys.FORMAT_KEY); try { final MapReduceJobOutput output = createJobOutput(Format.parse(format)); output.initialize(params); return output; } catch (JobIOSpecParseException exn) { throw new IOException(String.format( "Invalid job output specification: '%s': %s", ssm, exn.getMessage()), exn); } }
/** * Configures the output for the job. * * @param job The job to configure. * @throws IOException If there is an error. */ protected void configureOutput(Job job) throws IOException { if (null == mJobOutput) { throw new JobConfigurationException("Must specify job output."); } mJobOutput.configure(job); }
/** {@inheritDoc} */ @Override public void configure(Job job) throws IOException { // sets Hadoop output format according to getOutputFormatClass() super.configure(job); final Configuration conf = job.getConfiguration(); conf.set(FijiConfKeys.FIJI_OUTPUT_TABLE_URI, mTableURI.toString()); job.setNumReduceTasks(getNumReduceTasks()); // Adds HBase dependency jars to the distributed cache so they appear on the task classpath: GenericTableMapReduceUtil.addAllDependencyJars(job); }
/** {@inheritDoc} */ @Override public void configure(Job job) throws IOException { super.configure(job); // sets the Hadoop output format final Configuration conf = job.getConfiguration(); conf.set(FijiConfKeys.FIJI_OUTPUT_TABLE_URI, mJobOutput.getOutputTableURI().toString()); // Fiji table context: conf.setClass( FijiConfKeys.FIJI_TABLE_CONTEXT_CLASS, HFileWriterContext.class, FijiTableContext.class); // Set the output path. FileOutputFormat.setOutputPath(job, mJobOutput.getPath()); job.setNumReduceTasks(mJobOutput.getNumReduceTasks()); }
/** {@inheritDoc} */ @Override protected void configureOutput(Job job) throws IOException { final MapReduceJobOutput output = getJobOutput(); if (null == output) { throw new JobConfigurationException("Must specify job output."); } final FijiReducer reducer = getReducer(); if (output instanceof HFileMapReduceJobOutput) { if (reducer instanceof IdentityReducer) { output.configure(job); } else { // Cannot use the HFile output format if the reducer is not IdentityReducer: // Writing HFile from a Fiji reducer requires an extra map/reduce to sort the HFile keys. // This forces the output format of this MapReduce job to be SequenceFile. final HFileMapReduceJobOutput hfileOutput = (HFileMapReduceJobOutput) output; LOG.warn("Reducing to HFiles will require an extra MapReduce job."); new HFileReducerMapReduceJobOutput(hfileOutput).configure(job); } } else { output.configure(job); } } }