SequenceFileOutputFormat.setCompressOutput(job, true);
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set(FileOutputFormat.COMPRESS_TYPE, style.toString()); }
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set(FileOutputFormat.COMPRESS_TYPE, style.toString()); }
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set(FileOutputFormat.COMPRESS_TYPE, style.toString()); }
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set(FileOutputFormat.COMPRESS_TYPE, style.toString()); }
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set("mapred.output.compression.type", style.toString()); }
/** * Set the {@link CompressionType} for the output {@link SequenceFile}. * @param job the {@link Job} to modify * @param style the {@link CompressionType} for the output * {@link SequenceFile} */ public static void setOutputCompressionType(Job job, CompressionType style) { setCompressOutput(job, true); job.getConfiguration().set(FileOutputFormat.COMPRESS_TYPE, style.toString()); }
@Override public int run(String[] args) throws Exception { // String clusterFileString = options.getInputString(); Path[] paths = new Path[]{new Path(args[0])}; TextBytesSequenceFileUtility util = new TextBytesSequenceFileUtility(paths[0].toUri() , true); HadoopToolsUtil.removeFile(args[1]); Job job = TextBytesJobUtil.createJob(paths, new Path(args[1]), new HashMap<String,String>(), this.getConf()); job.setJarByClass(this.getClass()); job.setMapperClass(CounterMapper.class); job.setReducerClass(CounterReducer.class); SequenceFileOutputFormat.setCompressOutput(job, false); long start,end; start = System.currentTimeMillis(); job.waitForCompletion(true); end = System.currentTimeMillis(); System.out.println("Took: " + (end - start) + "ms"); return 0; }
@Override public int run(String[] args) throws Exception { // String clusterFileString = options.getInputString(); Path[] paths = new Path[]{new Path(args[0])}; TextBytesSequenceFileUtility util = new TextBytesSequenceFileUtility(paths[0].toUri() , true); HadoopToolsUtil.removeFile(args[1]); Job job = TextBytesJobUtil.createJob(paths, new Path(args[1]), new HashMap<String,String>(), this.getConf()); job.setJarByClass(this.getClass()); job.setMapperClass(CounterMapper.class); job.setReducerClass(CounterReducer.class); SequenceFileOutputFormat.setCompressOutput(job, false); long start,end; start = System.currentTimeMillis(); job.waitForCompletion(true); end = System.currentTimeMillis(); System.out.println("Took: " + (end - start) + "ms"); return 0; }
public static void runJob(String input, String output) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf); job.setJarByClass(SequenceFileStockMapReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(StockPriceWritable.class); job.setInputFormatClass( SequenceFileInputFormat.class); //<co id="ch03_comment_seqfile_mr1"/> job.setOutputFormatClass(SequenceFileOutputFormat.class); //<co id="ch03_comment_seqfile_mr2"/> SequenceFileOutputFormat.setCompressOutput(job, true); //<co id="ch03_comment_seqfile_mr3"/> SequenceFileOutputFormat.setOutputCompressionType(job, //<co id="ch03_comment_seqfile_mr4"/> SequenceFile.CompressionType.BLOCK); SequenceFileOutputFormat.setOutputCompressorClass(job, //<co id="ch03_comment_seqfile_mr5"/> DefaultCodec.class); FileInputFormat.setInputPaths(job, new Path(input)); Path outPath = new Path(output); FileOutputFormat.setOutputPath(job, outPath); outPath.getFileSystem(conf).delete(outPath, true); job.waitForCompletion(true); } }
SequenceFileOutputFormat.setCompressOutput(job, false);
SequenceFileOutputFormat.setCompressOutput(job, false); job.waitForCompletion(true); return 0;
@Override public int run(String[] args) throws Exception { final CmdLineParser parser = new CmdLineParser(this); try { parser.parseArgument(args); } catch (final CmdLineException e) { System.err.println(e.getMessage()); System.err.println("Usage: hadoop jar HadoopImageIndexer.jar [options]"); parser.printUsage(System.err); return -1; } final Path[] paths = SequenceFileUtility.getFilePaths(input, "part"); final Path outputPath = new Path(output); if (outputPath.getFileSystem(this.getConf()).exists(outputPath) && replace) outputPath.getFileSystem(this.getConf()).delete(outputPath, true); final Job job = TextBytesJobUtil.createJob(paths, outputPath, null, this.getConf()); job.setJarByClass(this.getClass()); job.setMapperClass(PcaVladMapper.class); job.setNumReduceTasks(0); DistributedCache.addFileToClassPath(new Path(indexerData), job.getConfiguration()); job.getConfiguration().set(VLAD_INDEXER_DATA_PATH_KEY, new Path(indexerData).getName()); SequenceFileOutputFormat.setCompressOutput(job, !dontcompress); job.waitForCompletion(true); return 0; }
public static Job createJob(Path[] inputPaths, Path outputPath, Map<String, String> metadata, Configuration config) throws IOException { final Job job = new Job(config); job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(BytesWritable.class); job.setOutputFormatClass(MetadataSequenceFileOutputFormat.class); SequenceFileInputFormat.setInputPaths(job, inputPaths); SequenceFileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setCompressOutput(job, true); SequenceFileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); if (metadata != null) MetadataConfiguration.setMetadata(metadata, job.getConfiguration()); return job; } }
public static Job createJob(Path[] inputPaths, Path outputPath, Map<String, String> metadata, Configuration config) throws IOException { final Job job = new Job(config); job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(BytesWritable.class); job.setOutputFormatClass(MetadataSequenceFileOutputFormat.class); SequenceFileInputFormat.setInputPaths(job, inputPaths); SequenceFileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setCompressOutput(job, true); SequenceFileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); if (metadata != null) MetadataConfiguration.setMetadata(metadata, job.getConfiguration()); return job; } }
@Override public int run(String[] args) throws Exception { final CmdLineParser parser = new CmdLineParser(this); try { parser.parseArgument(args); } catch (final CmdLineException e) { System.err.println(e.getMessage()); System.err.println("Usage: hadoop jar HadoopImageIndexer.jar [options]"); parser.printUsage(System.err); return -1; } final Path[] paths = SequenceFileUtility.getFilePaths(input, "part"); final Path outputPath = new Path(output); if (outputPath.getFileSystem(this.getConf()).exists(outputPath) && replace) outputPath.getFileSystem(this.getConf()).delete(outputPath, true); final Job job = TextBytesJobUtil.createJob(paths, outputPath, null, this.getConf()); job.setJarByClass(this.getClass()); job.setMapperClass(PqPcaVladMapper.class); job.setNumReduceTasks(0); MultipleOutputs.addNamedOutput(job, "pcavlad", SequenceFileOutputFormat.class, Text.class, BytesWritable.class); DistributedCache.createSymlink(job.getConfiguration()); DistributedCache.addCacheFile(new URI(indexerData + "#vlad-data.bin"), job.getConfiguration()); SequenceFileOutputFormat.setCompressOutput(job, !dontcompress); job.waitForCompletion(true); return 0; }
@Override public int run(String[] args) throws Exception { HadoopEXIFOptions options = new HadoopEXIFOptions(args,true); options.prepare(); // String clusterFileString = options.getInputString(); Path[] paths = options.getInputPaths(); TextBytesSequenceFileUtility util = new TextBytesSequenceFileUtility(paths[0].toUri() , true); Map<String,String> metadata = new HashMap<String,String>(); if (util.getUUID() != null) metadata.put(MetadataConfiguration.UUID_KEY, util.getUUID()); metadata.put(MetadataConfiguration.CONTENT_TYPE_KEY, "application/imageexif"); Job job = TextBytesJobUtil.createJob(paths, options.getOutputPath(), metadata,this.getConf()); // job.setOutputValueClass(Text.class); job.setJarByClass(this.getClass()); job.setMapperClass(HadoopEXIF.HadoopEXIFMapper.class); job.getConfiguration().setStrings(ARGS_KEY, args); job.setNumReduceTasks(0); SequenceFileOutputFormat.setCompressOutput(job, false); long start,end; start = System.currentTimeMillis(); job.waitForCompletion(true); end = System.currentTimeMillis(); System.out.println("Took: " + (end - start) + "ms"); return 0; }
protected void setupOutput(final Job job, final SampleDataForSplitPoints operation, final Store store) throws IOException { job.setOutputFormatClass(SequenceFileOutputFormat.class); SequenceFileOutputFormat.setOutputPath(job, new Path(operation.getOutputPath())); if (null != operation.getCompressionCodec()) { if (GzipCodec.class.isAssignableFrom(operation.getCompressionCodec()) && !NativeCodeLoader.isNativeCodeLoaded() && !ZlibFactory.isNativeZlibLoaded(job.getConfiguration())) { LOGGER.warn("SequenceFile doesn't work with GzipCodec without native-hadoop code!"); } else { SequenceFileOutputFormat.setCompressOutput(job, true); SequenceFileOutputFormat.setOutputCompressorClass(job, operation.getCompressionCodec()); SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK); } } } }
@Override public int run(String[] args) throws Exception { final HadoopLocalFeaturesToolOptions options = new HadoopLocalFeaturesToolOptions(args, true); options.prepare(); final Path[] paths = options.getInputPaths(); final TextBytesSequenceFileUtility util = new TextBytesSequenceFileUtility(paths[0].toUri(), true); final Map<String, String> metadata = new HashMap<String, String>(); if (util.getUUID() != null) metadata.put(MetadataConfiguration.UUID_KEY, util.getUUID()); metadata.put(MetadataConfiguration.CONTENT_TYPE_KEY, "application/localfeatures-" + options.getMode().name() + "-" + (options.isAsciiMode() ? "ascii" : "bin")); final Job job = TextBytesJobUtil.createJob(paths, options.getOutputPath(), metadata, this.getConf()); job.setJarByClass(this.getClass()); options.mapperModeOp.prepareJobMapper(job, LocalFeaturesMapper.class); job.getConfiguration().setStrings(ARGS_KEY, args); job.setNumReduceTasks(0); SequenceFileOutputFormat.setCompressOutput(job, !options.dontcompress); long start, end; start = System.currentTimeMillis(); job.waitForCompletion(true); end = System.currentTimeMillis(); System.out.println("Took: " + (end - start) + "ms"); options.serialiseExtractor(); return 0; }
@Override public int run(String[] args) throws Exception { final HadoopLocalFeaturesToolOptions options = new HadoopLocalFeaturesToolOptions(args, true); options.prepare(); final Path[] paths = options.getInputPaths(); final TextBytesSequenceFileUtility util = new TextBytesSequenceFileUtility(paths[0].toUri(), true); final Map<String, String> metadata = new HashMap<String, String>(); if (util.getUUID() != null) metadata.put(MetadataConfiguration.UUID_KEY, util.getUUID()); metadata.put(MetadataConfiguration.CONTENT_TYPE_KEY, "application/localfeatures-" + options.getMode().name() + "-" + (options.isAsciiMode() ? "ascii" : "bin")); final Job job = TextBytesJobUtil.createJob(paths, options.getOutputPath(), metadata, this.getConf()); job.setJarByClass(this.getClass()); options.mapperModeOp.prepareJobMapper(job, LocalFeaturesMapper.class); job.getConfiguration().setStrings(ARGS_KEY, args); job.setNumReduceTasks(0); SequenceFileOutputFormat.setCompressOutput(job, !options.dontcompress); long start, end; start = System.currentTimeMillis(); job.waitForCompletion(true); end = System.currentTimeMillis(); System.out.println("Took: " + (end - start) + "ms"); options.serialiseExtractor(); return 0; }