void runMRCreateFail( String dbName, String tableName, Map<String, String> partitionValues, List<HCatFieldSchema> columns) throws Exception { Job job = new Job(mrConf, "hcat mapreduce write fail test"); job.setJarByClass(this.getClass()); job.setMapperClass(TestHCatPartitionPublish.MapFail.class); // input/output settings job.setInputFormatClass(TextInputFormat.class); Path path = new Path(fs.getWorkingDirectory(), "mapred/testHCatMapReduceInput"); // The write count does not matter, as the map will fail in its first // call. createInputFile(path, 5); TextInputFormat.setInputPaths(job, path); job.setOutputFormatClass(HCatOutputFormat.class); OutputJobInfo outputJobInfo = OutputJobInfo.create(dbName, tableName, partitionValues); HCatOutputFormat.setOutput(job, outputJobInfo); job.setMapOutputKeyClass(BytesWritable.class); job.setMapOutputValueClass(DefaultHCatRecord.class); job.setNumReduceTasks(0); HCatOutputFormat.setSchema(job, new HCatSchema(columns)); boolean success = job.waitForCompletion(true); Assert.assertTrue(success == false); }
job.setOutputValueClass(DefaultHCatRecord.class); job.setInputFormatClass(TextInputFormat.class); TextInputFormat.setInputPaths(job, INPUT_FILE_NAME);
TextInputFormat.setInputPaths(job, path); } else { createInputFile(path2, (writeCount - writeCount / 2)); TextInputFormat.setInputPaths(job, path, path2);
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage : need <input path> <output path>"); return 1; } Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Configuration conf = getConf(); Job job = new Job(conf, getClass().getName() + "--<your_name>"); // TODO job.setJarByClass(Combiner.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); /// TODO : set combiner class /// in this case (aggregation) Reducer can be used as Combiner // job.setCombinerClass(cls); job.setMapOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.setInputPaths(job, inputPath); TextOutputFormat.setOutputPath(job, outputPath); return job.waitForCompletion(true) ? 0 : 1; }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage : need <input path> <output path>"); return 1; } Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Configuration conf = getConf(); Job job = new Job(conf, getClass().getName() + "--<your_name>"); // TODO job.setJarByClass(ItemizedBilling.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); job.setMapOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.setInputPaths(job, inputPath); TextOutputFormat.setOutputPath(job, outputPath); return job.waitForCompletion(true) ? 0 : 1; }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage : need <input path> <output path>"); return 1; } Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Configuration conf = getConf(); Job job = new Job(conf, getClass().getName() + "--<your_name>"); // TODO job.setJarByClass(BillingTotal.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); job.setMapOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.setInputPaths(job, inputPath); TextOutputFormat.setOutputPath(job, outputPath); return job.waitForCompletion(true) ? 0 : 1; }
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage : need <input path> <output path>"); return 1; } Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Configuration conf = getConf(); Job job = new Job(conf, getClass().getName()); job.setJarByClass(BillingPolished.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); job.setMapOutputValueClass(IntWritable.class); job.setMapOutputKeyClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.setInputPaths(job, inputPath); TextOutputFormat.setOutputPath(job, outputPath); return job.waitForCompletion(true) ? 0 : 1; }
TextInputFormat.setInputPaths(job, new Path(args[2]));
job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.setInputPaths(job, inputPath); TextOutputFormat.setOutputPath(job, outputPath);
public static Path compressAndIndex(Path file, Configuration conf) throws IOException { Configuration tmpConfig = new Configuration(conf); tmpConfig.setLong("dfs.block.size", 512); tmpConfig.setInt(LzoCodec.LZO_BUFFER_SIZE_KEY, 512); Path compressedFile = LzopFileReadWrite.compress(file, tmpConfig); compressedFile.getFileSystem(tmpConfig).delete(new Path( compressedFile.toString() + LzoIndex.LZO_INDEX_SUFFIX), false); new LzoIndexer(tmpConfig).index(compressedFile); LzoIndex index = LzoIndex .readIndex(compressedFile.getFileSystem(tmpConfig), compressedFile); for (int i = 0; i < index.getNumberOfBlocks(); i++) { System.out.println("block[" + i + "] = " + index.getPosition(i)); } Job job = new Job(conf); job.setInputFormatClass(LzoTextInputFormat.class); LzoTextInputFormat inputFormat = new LzoTextInputFormat(); TextInputFormat.setInputPaths(job, compressedFile); List<InputSplit> is = inputFormat.getSplits(job); System.out.println("input splits = " + is.size()); return compressedFile; }
sampleJob.setOutputKeyClass(Text.class); sampleJob.setOutputValueClass(Text.class); TextInputFormat.setInputPaths(sampleJob, inputPath);
TextInputFormat.setInputPaths(job, new Path(inputDir)); AccumuloFileOutputFormat.setOutputPath(job, new Path(workDir + "/files"));
setVarOrders(sparql, jobConf); TextInputFormat.setInputPaths(job, inputDir); job.setInputFormatClass(TextInputFormat.class);
conf.set("mapreduce.framework.name", "local"); Job job = new Job(conf); TextInputFormat.setInputPaths(job, new Path(in.getPath())); TextOutputFormat.setOutputPath(job, new Path(out.getPath()));