FileOutputFormat.setOutputPath(job, output); FileOutputFormat.setCompressOutput(job, true); AvroJob.setOutputCodec(job, SNAPPY_CODEC);
public static void main(String... args) throws Exception { JobConf job = new JobConf(); job.setJarByClass(SmallFilesMapReduce.class); Path input = new Path(args[0]); Path output = new Path(args[1]); output.getFileSystem(job).delete(output, true); AvroJob.setInputSchema(job, Stock.SCHEMA$); //<co id="ch03_avro_mr_comment1"/> AvroJob.setMapOutputSchema(job, Pair.getPairSchema(Stock.SCHEMA$, Schema.create(Schema.Type.NULL))); AvroJob.setOutputSchema(job, Stock.SCHEMA$); FileInputFormat.setInputPaths(job, input); FileOutputFormat.setOutputPath(job, output); AvroJob.setMapperClass(job, Mapper.class); //<co id="ch03_smallfilemr_comment2"/> AvroJob.setReducerClass(job, Reducer.class); FileOutputFormat.setCompressOutput(job, true); AvroJob.setOutputCodec(job, SNAPPY_CODEC); JobClient.runJob(job); }