/** {@inheritDoc} */ @Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { assert wasConfigured : "Mapper should be configured"; String line = val.toString(); StringTokenizer tokenizer = new StringTokenizer(line); while (tokenizer.hasMoreTokens()) { word.set(tokenizer.nextToken()); output.collect(word, one); } HadoopErrorSimulator.instance().onMap(); }
public void map(K key, Text value, OutputCollector<Text, LongWritable> output, Reporter reporter) throws IOException { // get input text String text = value.toString(); // value is line of text // tokenize the value StringTokenizer st = new StringTokenizer(text); while (st.hasMoreTokens()) { // output <token,1> pairs output.collect(new Text(st.nextToken()), new LongWritable(1)); } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) ); } }
@Override public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep) throws IOException { // normalize and split the line String line = v.toString(); String[] tokens = line.toLowerCase().split("\\W+"); // emit the pairs for (String token : tokens) { if (token.length() > 0) { out.collect(new Text(token), new LongWritable(1L)); } } }
@Override public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int commentCnt = 0; while (vs.hasNext()) { String v = vs.next().toString(); if (v.startsWith(this.countPrefix)) { commentCnt++; } } out.collect(k, new IntWritable(commentCnt)); }
@Override public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); for (long i=range[0]; i<range[1]; i++) { key.set(i); Text v = new Text(Long.toString(i)); output.collect(key, v); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+v.getLength()); } } }
public void reduce (final IntWritable key, final Iterator<Text> values, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { int i; double next_rank = 0; double previous_rank = 0; while (values.hasNext()) { String cur_value_str = values.next().toString(); if( cur_value_str.charAt(0) == 's' ) previous_rank = Double.parseDouble( cur_value_str.substring(1) ); else next_rank += Double.parseDouble( cur_value_str.substring(1) ) ; } next_rank = next_rank * mixing_c + random_coeff; output.collect( key, new Text("v" + next_rank ) ); if( change_reported == 0 ) { double diff = Math.abs(previous_rank-next_rank); if( diff > converge_threshold ) { reporter.incrCounter(PrCounters.CONVERGE_CHECK, 1); change_reported = 1; } } } }
@Override public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); generator.fireRandom(slotId); rand = new Random(slotId * 1000 + 101); Text k = new Text(); for (long i=range[0]; i<range[1]; i++) { String classname = "/class" + rand.nextInt(groups); k.set(classname); value.set(generator.genBayesWords()); output.collect(k, value); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, k.getLength()+value.getLength()); if (0==(i % 10000)) { log.info("still running: " + (i - range[0]) + " of " + slotpages); } } } }
@Override public void map(LongWritable key, Text value, OutputCollector<LongWritable, JoinBytesInt> output, Reporter reporter) throws IOException { String[] items = value.toString().split("[,\t]"); key.set(Long.parseLong(items[0])); uitem.url= items[1].getBytes(); uitem.ulen = (byte) uitem.url.length; output.collect(key, uitem); } }
@Override public void reduce(LongWritable key, Iterator<JoinBytesInt> values, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { v.clear(); while (values.hasNext()) { v.add(values.next()); } if (0!=v.ulen) { if (v.refs > 0) { Text value = new Text( new String(v.url) + delim + v.refs + delim + (rand.nextInt(99) + 1) ); output.collect( key, value); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+value.getLength()); } else { missed++; } } else { errors++; } } }
Reporter reporter ) throws IOException { String field = key.toString(); reporter.setStatus("starting " + field + " ::host = " + hostName); String sSum = ""; while (values.hasNext()) sSum += values.next().toString() + ";"; output.collect(key, new Text(sSum)); reporter.setStatus("finished " + field + " ::host = " + hostName); return; while (values.hasNext()) fSum += Float.parseFloat(values.next().toString()); output.collect(key, new Text(String.valueOf(fSum))); reporter.setStatus("finished " + field + " ::host = " + hostName); return; lSum += Long.parseLong(values.next().toString()); output.collect(key, new Text(String.valueOf(lSum))); reporter.setStatus("finished " + field + " ::host = " + hostName);
@Override public void map(LongWritable key, Text value, OutputCollector<LongWritable, NullWritable> output, Reporter reporter) throws IOException { String delimiter = "[ \t]"; String[] pair = value.toString().split(delimiter); output.collect( new LongWritable(Long.parseLong(pair[0])), NullWritable.get() ); } }
public void reduce(AvroKey<Long> key, Iterator<AvroValue<Utf8>> values, OutputCollector<LongWritable, Text> out, Reporter reporter) throws IOException { while (values.hasNext()) { AvroValue<Utf8> value = values.next(); out.collect(new LongWritable(key.datum()), new Text(value.datum().toString())); } } }
String[] numbers = value.toString().split("\t"); int i = 0; long numSamples = Long.parseLong(numbers[i++]); Vector p = new RandomAccessSparseVector(dimension); p.assign(vec); output.collect(new LongWritable(count), new VectorWritable(p)); reporter.setStatus(Long.toString(count + 1) + " samples generated"); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8 + p.getNumNondefaultElements() * 8);
@Override public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { while (v.hasNext()) { out.collect(new IntWritable(k.get() % 4), v.next()); } }