@Override public int run(String[] args) throws Exception { SequenceFileIndexerOptions options = new SequenceFileIndexerOptions(args); options.prepare(); Job job = new Job(getConf()); job.setJarByClass(SequenceFileIndexer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setMapperClass(RecordIndexMapper.class); job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); job.setNumReduceTasks(1); FileInputFormat.setInputPaths(job, options.getInputPaths()); TextOutputFormat.setOutputPath(job, options.getOutputPath()); TextOutputFormat.setCompressOutput(job, false); // SequenceFileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class); // SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); job.waitForCompletion(true); return 0; } public static void main(String [] args) throws Exception {
TextOutputFormat.setCompressOutput(job, true); if (line.hasOption("snappy")) { TextOutputFormat.setOutputCompressorClass(job, SnappyCodec.class);
job.setOutputFormatClass(outputFormatClass); if (outputFormatClass.equals(TextOutputFormat.class) && this.lzoCompress()) { TextOutputFormat.setCompressOutput(job, true); TextOutputFormat.setOutputCompressorClass(job, (Class<? extends CompressionCodec>) Class.forName("com.hadoop.compression.lzo.LzopCodec")); } else { TextOutputFormat.setCompressOutput(job, false);
job.setOutputFormatClass(outputFormatClass); if (outputFormatClass.equals(TextOutputFormat.class) && this.lzoCompress()) { TextOutputFormat.setCompressOutput(job, true); TextOutputFormat.setOutputCompressorClass(job, (Class<? extends CompressionCodec>) Class.forName("com.hadoop.compression.lzo.LzopCodec")); } else { TextOutputFormat.setCompressOutput(job, false);