@SuppressWarnings("unchecked") public Pair(Object key, Double value) { this((K)key, getSchema(key), (V)value, DOUBLE_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, Long value) { this((K)key, getSchema(key), (V)value, LONG_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Long key, Object value) { this((K)key, LONG_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, CharSequence value) { this((K)key, getSchema(key), (V)value, STRING_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(CharSequence key, Object value) { this((K)key, STRING_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Double key, Object value) { this((K)key, DOUBLE_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, ByteBuffer value) { this((K)key, getSchema(key), (V)value, BYTES_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, Integer value) { this((K)key, getSchema(key), (V)value, INT_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(ByteBuffer key, Object value) { this((K)key, BYTES_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Float key, Object value) { this((K)key, FLOAT_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Void key, Object value) { this((K)key, NULL_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, Float value) { this((K)key, getSchema(key), (V)value, FLOAT_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, Void value) { this((K)key, getSchema(key), (V)value, NULL_SCHEMA); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Integer key, Object value) { this((K)key, INT_SCHEMA, (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(GenericContainer key, Object value) { this((K)key, key.getSchema(), (V)value, getSchema(value)); } @SuppressWarnings("unchecked")
@SuppressWarnings("unchecked") public Pair(Object key, GenericContainer value) { this((K)key, getSchema(key), (V)value, value.getSchema()); } @SuppressWarnings("unchecked")
@SuppressWarnings("deprecation") public void testJobNoreducer() throws Exception { JobConf job = new JobConf(); job.setNumReduceTasks(0); Path outputPath = new Path(OUTPUT_DIR.getRoot().getPath()); outputPath.getFileSystem(job).delete(outputPath); WordCountUtil.writeLinesFile(new File(INPUT_DIR.getRoot(),"lines.avro")); job.setJobName("AvroMultipleOutputs_noreducer"); AvroJob.setInputSchema(job, Schema.create(Schema.Type.STRING)); AvroJob.setOutputSchema(job, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema()); AvroJob.setMapperClass(job, MapImpl.class); FileInputFormat.setInputPaths(job, new Path(INPUT_DIR.getRoot().toString())); FileOutputFormat.setOutputPath(job, outputPath); FileOutputFormat.setCompressOutput(job, false); AvroMultipleOutputs.addNamedOutput(job, "myavro2", AvroOutputFormat.class, Schema.create(Schema.Type.STRING)); JobClient.runJob(job); }
@SuppressWarnings("deprecation") public void testJob(String pathOut) throws Exception { JobConf job = new JobConf(); String pathIn = INPUT_DIR.getRoot().getPath(); WordCountUtil.writeLinesFile(pathIn + "/lines.avro"); Path outputPath = new Path(pathOut); outputPath.getFileSystem(job).delete(outputPath); job.setJobName("wordcount"); AvroJob.setInputSchema(job, Schema.create(Schema.Type.STRING)); AvroJob.setOutputSchema(job, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema()); AvroJob.setMapperClass(job, MapImpl.class); AvroJob.setCombinerClass(job, ReduceImpl.class); AvroJob.setReducerClass(job, ReduceImpl.class); FileInputFormat.setInputPaths(job, new Path(pathIn)); FileOutputFormat.setOutputPath(job, new Path(pathOut)); FileOutputFormat.setCompressOutput(job, true); WordCountUtil.setMeta(job); JobClient.runJob(job); WordCountUtil.validateCountsFile(new File(pathOut, "part-00000.avro")); }
@Override public void reduce(Utf8 word, Iterable<Long> counts, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { long sum = 0; for (long count : counts) sum += count; Pair<Utf8, Long> outputvalue = new Pair<>(word, sum); amos.getCollector("myavro", reporter).collect(outputvalue); amos.collect("myavro1", reporter, outputvalue.toString()); amos.collect("myavro", reporter, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema(), outputvalue, "testavrofile"); amos.collect("myavro", reporter, Schema.create(Schema.Type.STRING), outputvalue.toString(), "testavrofile1"); collector.collect(new Pair<>(word, sum)); }