/** * Write key and value to the namedOutput. * * Output path is a unique file generated for the namedOutput. * For example, {namedOutput}-(m|r)-{part-number} * * @param namedOutput the named output name * @param key the key * @param value the value */ @SuppressWarnings("unchecked") public void write(String namedOutput, Object key, Object value) throws IOException, InterruptedException { write(namedOutput, key, value, namedOutput); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ public void write(Object key, Object value, String baseOutputPath) throws IOException, InterruptedException { write(key, value, null, null, baseOutputPath); }
/** * Write key and value to the namedOutput. * * Output path is a unique file generated for the namedOutput. * For example, {namedOutput}-(m|r)-{part-number} * * @param namedOutput the named output name * @param key the key , value is NullWritable */ @SuppressWarnings("unchecked") public void write(String namedOutput, Object key) throws IOException, InterruptedException { write(namedOutput, key, NullWritable.get(), namedOutput); }
@Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { TextStats record = new TextStats(); record.count = 0; for (IntWritable count : counts) { record.count += count.get(); } record.name = line.toString(); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.write("myavro3", mStats, NullWritable.get()); }
@Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { TextStats record = new TextStats(); record.count = 0; for (IntWritable count : counts) { record.count += count.get(); } record.name = line.toString(); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.sync("myavro3","myavro3"); amos.write("myavro3",mStats,NullWritable.get()); } @Override
@Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { GenericData.Record record = new GenericData.Record(STATS_SCHEMA); GenericData.Record record2 = new GenericData.Record(STATS_SCHEMA_2); int sum = 0; for (IntWritable count : counts) { sum += count.get(); } record.put("name", new Utf8(line.toString())); record.put("count", new Integer(sum)); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.write("myavro", mStats, NullWritable.get()); record2.put("name1", new Utf8(line.toString())); record2.put("count1", new Integer(sum)); mStats.datum(record2); amos.write(mStats, NullWritable.get(), STATS_SCHEMA_2, null, "testnewwrite2"); amos.write("myavro1", mStats); amos.write(mStats, NullWritable.get(), STATS_SCHEMA, null, "testnewwrite"); amos.write(mStats, NullWritable.get(), "testwritenonschema"); }
@Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { GenericData.Record record = new GenericData.Record(STATS_SCHEMA); GenericData.Record record2 = new GenericData.Record(STATS_SCHEMA_2); int sum = 0; for (IntWritable count : counts) { sum += count.get(); } record.put("name", new Utf8(line.toString())); record.put("count", sum); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.sync("myavro","myavro"); amos.write("myavro",mStats,NullWritable.get()); record2.put("name1", new Utf8(line.toString())); record2.put("count1", sum); mStats.datum(record2); amos.write(mStats, NullWritable.get(), STATS_SCHEMA_2, null, "testnewwrite2"); amos.sync("myavro1","myavro1"); amos.write("myavro1",mStats); amos.write(mStats, NullWritable.get(), STATS_SCHEMA, null, "testnewwrite"); amos.write(mStats, NullWritable.get(), "testwritenonschema"); }
/** * Write key and value to the namedOutput. * * Output path is a unique file generated for the namedOutput. * For example, {namedOutput}-(m|r)-{part-number} * * @param namedOutput the named output name * @param key the key * @param value the value */ @SuppressWarnings("unchecked") public void write(String namedOutput, Object key, Object value) throws IOException, InterruptedException { write(namedOutput, key, value, namedOutput); }
/** * Write key value to an output file name. * * Gets the record writer from job's output format. * Job's output format should be a FileOutputFormat. * * @param key the key * @param value the value * @param baseOutputPath base-output path to write the record to. * Note: Framework will generate unique filename for the baseOutputPath */ public void write(Object key, Object value, String baseOutputPath) throws IOException, InterruptedException { write(key, value, null, null, baseOutputPath); }
/** * Write key and value to the namedOutput. * * Output path is a unique file generated for the namedOutput. * For example, {namedOutput}-(m|r)-{part-number} * * @param namedOutput the named output name * @param key the key , value is NullWritable */ @SuppressWarnings("unchecked") public void write(String namedOutput, Object key) throws IOException, InterruptedException { write(namedOutput, key, NullWritable.get(), namedOutput); }
@Override protected void reduce(final Text key, final Iterable<AvroValue<GenericRecord>> values, final Context context) throws IOException, InterruptedException { final BaseReducer baseReducer = new BaseReducer(); final URI uri = baseReducer.getReducerKey(key, context); final AvroMultipleOutputs multipleOutputs = new AvroMultipleOutputs(context); try { for (AvroValue value : values) { GenericRecord rec = (GenericRecord) value.datum(); multipleOutputs.write(new AvroKey<GenericRecord>(rec), NullWritable.get(), uri.toString()); } } finally { multipleOutputs.close(); } } }
_multipleOutputs.write(namedOutput, new AvroKey<GenericRecord>(output), (AvroValue<GenericRecord>)null);