/** * No aggregation, output pairs of (key, record) * @param key * @param values * @param output * @param reporter * @throws IOException */ public void reduce(ImmutableBytesWritable key, Iterator<Put> values, OutputCollector<ImmutableBytesWritable, Put> output, Reporter reporter) throws IOException { while(values.hasNext()) { output.collect(key, values.next()); } } }
@Override public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { while (v.hasNext()) { out.collect(new IntWritable(k.get() % 4), v.next()); } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) ); } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, IntWritable> output, final Reporter reporter) throws IOException { String line_text = value.toString(); final String[] line = line_text.split("\t"); output.collect( new IntWritable(Integer.parseInt(line[1].substring(3))), new IntWritable(1) ); } }
void collectStats(OutputCollector<Text, Text> output, String name, long execTime, Object objSize) throws IOException { long totalSize = ((Long)objSize).longValue(); float ioRateMbSec = (float)totalSize * 1000 / (execTime * MEGA); LOG.info("Number of bytes processed = " + totalSize); LOG.info("Exec time = " + execTime); LOG.info("IO rate = " + ioRateMbSec); output.collect(new Text("l:tasks"), new Text(String.valueOf(1))); output.collect(new Text("l:size"), new Text(String.valueOf(totalSize))); output.collect(new Text("l:time"), new Text(String.valueOf(execTime))); output.collect(new Text("f:rate"), new Text(String.valueOf(ioRateMbSec*1000))); output.collect(new Text("f:sqrate"), new Text(String.valueOf(ioRateMbSec*ioRateMbSec*1000))); } }
@Override public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int sum = 0; while (v.hasNext()) { sum += v.next().get(); } out.collect(k, new IntWritable(sum)); }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) ); } }
@Override public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep) throws IOException { // normalize and split the line String line = v.toString(); String[] tokens = line.toLowerCase().split("\\W+"); // emit the pairs for (String token : tokens) { if (token.length() > 0) { out.collect(new Text(token), new LongWritable(1L)); } } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, IntWritable> output, final Reporter reporter) throws IOException { String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[1])), new IntWritable(1) ); } }
@Override public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int commentCnt = 0; while (vs.hasNext()) { String v = vs.next().toString(); if (v.startsWith(this.countPrefix)) { commentCnt++; } } out.collect(k, new IntWritable(commentCnt)); }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); IntWritable node_key = new IntWritable(Integer.parseInt(line[0])); output.collect(node_key, new Text(line[1]) ); } }
public void map (final LongWritable key, final Text value, final OutputCollector<LongWritable, Text> output, final Reporter reporter) throws IOException { String line_text = value.toString(); int tabpos = line_text.indexOf("\t"); if( tabpos > 0 ) { long out_key = Long.parseLong(line_text.substring(0, tabpos)); output.collect( new LongWritable(out_key) , new Text(line_text.substring(tabpos+1)) ); } else { output.collect( new LongWritable(Long.parseLong(line_text)) , new Text("") ); } } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, IntWritable> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); final String[] tokens = line[1].split(" "); String radius_str = tokens[0].substring(3) ; if( radius_str.length() > 0 ) { String[] radius_info = radius_str.split(":"); double eff_radius = Double.parseDouble(radius_info[1]); output.collect( new IntWritable((int)Math.round(eff_radius)), new IntWritable(1) ); } } }
/** {@inheritDoc} */ @Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { assert wasConfigured : "Mapper should be configured"; String line = val.toString(); StringTokenizer tokenizer = new StringTokenizer(line); while (tokenizer.hasMoreTokens()) { word.set(tokenizer.nextToken()); output.collect(word, one); } HadoopErrorSimulator.instance().onMap(); }