/** * Adds a named output for the job. * <p/> * * @param conf job conf to add the named output * @param namedOutput named output name, it has to be a word, letters * and numbers only, cannot be the word 'part' as * that is reserved for the * default output. * @param outputFormatClass OutputFormat class. * @param schema Schema to used for this namedOutput */ public static void addNamedOutput(JobConf conf, String namedOutput, Class<? extends OutputFormat> outputFormatClass, Schema schema) { addNamedOutput(conf, namedOutput, false, outputFormatClass, schema); }
@SuppressWarnings("rawtypes") private AvroCollector getCollector(String namedOutput,Schema schema, Reporter reporter, String baseFileName) throws IOException{ //namedOutputs.add(baseFileName); return getCollector(namedOutput,null,reporter,baseFileName,schema); }
public void configure(JobConf Job) { this.amos = new AvroMultipleOutputs(Job); }
/** * Creates and initializes multiple named outputs support, it should be * instantiated in the Mapper/Reducer configure method. * * @param job the job configuration object */ public AvroMultipleOutputs(JobConf job) { this.conf = job; outputFormat = new InternalFileOutputFormat(); namedOutputs = Collections.unmodifiableSet( new HashSet<>(AvroMultipleOutputs.getNamedOutputsList(job))); recordWriters = new HashMap<>(); countersEnabled = getCountersEnabled(job); }
throws IOException { checkNamedOutputName(namedOutput); if (!namedOutputs.contains(namedOutput)) { throw new IllegalArgumentException("Undefined named output '" + namedOutput + "'"); boolean multi = isMultiNamedOutput(conf, namedOutput); checkTokenName(multiName); getRecordWriter(namedOutput, baseFileName, reporter,schema);
/** * Adds a named output for the job. * <p/> * * @param conf job conf to add the named output * @param namedOutput named output name, it has to be a word, letters * and numbers only, cannot be the word 'part' as * that is reserved for the * default output. * @param multi indicates if the named output is multi * @param outputFormatClass OutputFormat class. * @param schema Schema to used for this namedOutput */ private static void addNamedOutput(JobConf conf, String namedOutput, boolean multi, Class<? extends OutputFormat> outputFormatClass, Schema schema) { checkNamedOutputName(namedOutput); checkNamedOutput(conf, namedOutput, true); boolean isMapOnly = conf.getNumReduceTasks() == 0; if(schema!=null) conf.set(MO_PREFIX+namedOutput+".schema", schema.toString()); conf.set(NAMED_OUTPUTS, conf.get(NAMED_OUTPUTS, "") + " " + namedOutput); conf.setClass(MO_PREFIX + namedOutput + FORMAT, outputFormatClass, OutputFormat.class); conf.setBoolean(MO_PREFIX + namedOutput + MULTI, multi); }
@Override public void reduce(Utf8 word, Iterable<Long> counts, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { long sum = 0; for (long count : counts) sum += count; Pair<Utf8, Long> outputvalue = new Pair<>(word, sum); amos.getCollector("myavro", reporter).collect(outputvalue); amos.collect("myavro1", reporter, outputvalue.toString()); amos.collect("myavro", reporter, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema(), outputvalue, "testavrofile"); amos.collect("myavro", reporter, Schema.create(Schema.Type.STRING), outputvalue.toString(), "testavrofile1"); collector.collect(new Pair<>(word, sum)); }
/** * Returns the named output OutputFormat. * * @param conf job conf * @param namedOutput named output * @return namedOutput OutputFormat */ public static Class<? extends OutputFormat> getNamedOutputFormatClass( JobConf conf, String namedOutput) { checkNamedOutput(conf, namedOutput, false); return conf.getClass(MO_PREFIX + namedOutput + FORMAT, null, OutputFormat.class); }
public void close() throws IOException { amos.close(); } }
/** * Checks if a named output name is valid. * * @param namedOutput named output Name * @throws IllegalArgumentException if the output name is not valid. */ private static void checkNamedOutputName(String namedOutput) { checkTokenName(namedOutput); // name cannot be the name used for the default output if (namedOutput.equals("part")) { throw new IllegalArgumentException( "Named output name cannot be 'part'"); } }
@SuppressWarnings({"unchecked", "deprecation"}) public RecordWriter<Object, Object> getRecordWriter(FileSystem fs,JobConf job, String baseFileName, Progressable arg3) throws IOException { String nameOutput = job.get(CONFIG_NAMED_OUTPUT, null); String fileName = getUniqueName(job, baseFileName); Schema schema = null; String schemastr = job.get(MO_PREFIX+nameOutput+".schema",null); if (schemastr!=null) schema = Schema.parse(schemastr); JobConf outputConf = new JobConf(job); outputConf.setOutputFormat(getNamedOutputFormatClass(job, nameOutput)); boolean isMapOnly = job.getNumReduceTasks() == 0; if (schema != null) { if (isMapOnly) AvroJob.setMapOutputSchema(outputConf, schema); else AvroJob.setOutputSchema(outputConf, schema); } OutputFormat outputFormat = outputConf.getOutputFormat(); return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3); } }
throws IOException { checkNamedOutputName(namedOutput); if (!namedOutputs.contains(namedOutput)) { throw new IllegalArgumentException("Undefined named output '" + namedOutput + "'"); boolean multi = isMultiNamedOutput(conf, namedOutput); checkTokenName(multiName); getRecordWriter(namedOutput, baseFileName, reporter,schema);
/** * Creates and initializes multiple named outputs support, it should be * instantiated in the Mapper/Reducer configure method. * * @param job the job configuration object */ public AvroMultipleOutputs(JobConf job) { this.conf = job; outputFormat = new InternalFileOutputFormat(); namedOutputs = Collections.unmodifiableSet( new HashSet<String>(AvroMultipleOutputs.getNamedOutputsList(job))); recordWriters = new HashMap<String, RecordWriter>(); countersEnabled = getCountersEnabled(job); }
/** * Adds a named output for the job. * <p/> * * @param conf job conf to add the named output * @param namedOutput named output name, it has to be a word, letters * and numbers only, cannot be the word 'part' as * that is reserved for the * default output. * @param multi indicates if the named output is multi * @param outputFormatClass OutputFormat class. * @param schema Schema to used for this namedOutput */ private static void addNamedOutput(JobConf conf, String namedOutput, boolean multi, Class<? extends OutputFormat> outputFormatClass, Schema schema) { checkNamedOutputName(namedOutput); checkNamedOutput(conf, namedOutput, true); boolean isMapOnly = conf.getNumReduceTasks() == 0; if(schema!=null) conf.set(MO_PREFIX+namedOutput+".schema", schema.toString()); conf.set(NAMED_OUTPUTS, conf.get(NAMED_OUTPUTS, "") + " " + namedOutput); conf.setClass(MO_PREFIX + namedOutput + FORMAT, outputFormatClass, OutputFormat.class); conf.setBoolean(MO_PREFIX + namedOutput + MULTI, multi); }
/** * Returns if a named output is multiple. * * @param conf job conf * @param namedOutput named output * @return <code>true</code> if the name output is multi, <code>false</code> * if it is single. If the name output is not defined it returns * <code>false</code> */ public static boolean isMultiNamedOutput(JobConf conf, String namedOutput) { checkNamedOutput(conf, namedOutput, false); return conf.getBoolean(MO_PREFIX + namedOutput + MULTI, false); }
public void close() throws IOException { amos.close(); } }
/** * Checks if a named output name is valid. * * @param namedOutput named output Name * @throws IllegalArgumentException if the output name is not valid. */ private static void checkNamedOutputName(String namedOutput) { checkTokenName(namedOutput); // name cannot be the name used for the default output if (namedOutput.equals("part")) { throw new IllegalArgumentException( "Named output name cannot be 'part'"); } }
@SuppressWarnings({"unchecked", "deprecation"}) public RecordWriter<Object, Object> getRecordWriter(FileSystem fs,JobConf job, String baseFileName, Progressable arg3) throws IOException { String nameOutput = job.get(CONFIG_NAMED_OUTPUT, null); String fileName = getUniqueName(job, baseFileName); Schema schema = null; String schemastr = job.get(MO_PREFIX+nameOutput+".schema",null); if (schemastr!=null) schema = Schema.parse(schemastr); JobConf outputConf = new JobConf(job); outputConf.setOutputFormat(getNamedOutputFormatClass(job, nameOutput)); boolean isMapOnly = job.getNumReduceTasks() == 0; if (schema != null) { if (isMapOnly) AvroJob.setMapOutputSchema(outputConf, schema); else AvroJob.setOutputSchema(outputConf, schema); } OutputFormat outputFormat = outputConf.getOutputFormat(); return outputFormat.getRecordWriter(fs, outputConf, fileName, arg3); } }
@SuppressWarnings("rawtypes") private AvroCollector getCollector(String namedOutput, Reporter reporter, Schema schema) throws IOException{ return getCollector(namedOutput,null,reporter,namedOutput,schema); }
/** * Adds a multi named output for the job. * <p/> * * @param conf job conf to add the named output * @param namedOutput named output name, it has to be a word, letters * and numbers only, cannot be the word 'part' as * that is reserved for the * default output. * @param outputFormatClass OutputFormat class. * @param schema Schema to used for this namedOutput */ public static void addMultiNamedOutput(JobConf conf, String namedOutput, Class<? extends OutputFormat> outputFormatClass, Schema schema) { addNamedOutput(conf, namedOutput, true, outputFormatClass, schema); }