/** Called with each map input datum. By default, collects inputs. */ @SuppressWarnings("unchecked") public void map(IN datum, AvroCollector<OUT> collector, Reporter reporter) throws IOException { collector.collect((OUT)datum); }
/** * Output Collector for the default schema. * <p/> * * @param namedOutput the named output name * @param reporter the reporter * @param datum output data * @throws IOException thrown if output collector could not be created */ public void collect(String namedOutput, Reporter reporter,Object datum) throws IOException{ getCollector(namedOutput,reporter).collect(datum); }
@Override public void reduce(Weather w, Iterable<Void> ignore, AvroCollector<Weather> collector, Reporter reporter) throws IOException { collector.collect(w); }
/** * OutputCollector with custom schema. * <p/> * * @param namedOutput the named output name (this will the output file name) * @param reporter the reporter * @param datum output data * @param schema schema to use for this output * @throws IOException thrown if output collector could not be created */ public void collect(String namedOutput, Reporter reporter, Schema schema,Object datum) throws IOException{ getCollector(namedOutput,reporter,schema).collect(datum); }
/** * OutputCollector with custom schema and file name. * <p/> * * @param namedOutput the named output name * @param reporter the reporter * @param baseOutputPath outputfile name to use. * @param datum output data * @param schema schema to use for this output * @throws IOException thrown if output collector could not be created */ public void collect(String namedOutput,Reporter reporter,Schema schema,Object datum,String baseOutputPath) throws IOException{ getCollector(namedOutput,null,reporter,baseOutputPath,schema).collect(datum); }
@Override public void collect(byte[] key, byte[] value) throws IOException { if (!MINIMIZE_ALLOCATIONS || keyBB == null || keyBB.capacity() < key.length) { keyBB = ByteBuffer.wrap(key); } else { keyBB.position(0); keyBB.limit(key.length); keyBB.put(key); keyBB.position(0); } if (!MINIMIZE_ALLOCATIONS || valueBB == null || valueBB.capacity() < value.length) { valueBB = ByteBuffer.wrap(value); } else { valueBB.position(0); valueBB.limit(value.length); valueBB.put(value); valueBB.position(0); } pairToCollect.set(keyBB, valueBB); getCollector().collect(pairToCollect); } }
@Override public void reduce(Text word, Iterable<Count> counts, AvroCollector<WordCount> collector, Reporter reporter) throws IOException { long sum = 0; for (Count count : counts) sum += count.count; collector.collect(new WordCount(word.text, sum)); } }
@Override public void map(Weather w, AvroCollector<Pair<Weather,Void>> collector, Reporter reporter) throws IOException { collector.collect(new Pair<>(w, (Void) null)); }
@Override public void map(String text, AvroCollector<Pair<String,Long>> collector, Reporter reporter) throws IOException { StringTokenizer tokens = new StringTokenizer(text); while (tokens.hasMoreTokens()) collector.collect(new Pair<>(tokens.nextToken(), 1L)); } }
@Override public void reduce(String word, Iterable<Long> counts, AvroCollector<Pair<String,Long>> collector, Reporter reporter) throws IOException { long sum = 0; for (long count : counts) sum += count; collector.collect(new Pair<>(word, sum)); } }
@Override public void reduce(Utf8 word, Iterable<Long> counts, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { long sum = 0; for (long count : counts) sum += count; collector.collect(new Pair<>(word, sum)); } }
/** Called with all map output values with a given key. By default, pairs * key with each value, collecting {@link Pair} instances. */ @SuppressWarnings("unchecked") public void reduce(K key, Iterable<V> values, AvroCollector<OUT> collector, Reporter reporter) throws IOException { if (outputPair == null) outputPair = new Pair<>(AvroJob.getOutputSchema(getConf())); for (V value : values) { outputPair.set(key, value); collector.collect((OUT)outputPair); } }
@Override public void map(Utf8 text, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { StringTokenizer tokens = new StringTokenizer(text.toString()); while (tokens.hasMoreTokens()) collector.collect(new Pair<>(new Utf8(tokens.nextToken()), 1L)); } }
@Override public void reduce(KeyRecord ID, Iterable<JoinableRecord> joinables, AvroCollector<CompleteRecord> collector, Reporter reporter) throws IOException { CompleteRecord rec = new CompleteRecord(); for (JoinableRecord joinable : joinables) { rec.setId(joinable.id); if (joinable.recType.toString().contains("NamesRecord")) { rec.setName(joinable.name); } else { rec.setBalance(joinable.balance); } } collector.collect(rec); }
@Override public void map(Text text, AvroCollector<Pair<Text,Count>> collector, Reporter reporter) throws IOException { StringTokenizer tokens = new StringTokenizer(text.toString()); while (tokens.hasMoreTokens()) collector.collect(new Pair<>(new Text(tokens.nextToken()), new Count(1L))); } }
@Override public void map( NamesRecord nameRecord, AvroCollector<Pair<KeyRecord, JoinableRecord>> collector, Reporter reporter) throws IOException { collector.collect( new Pair<>( new KeyRecord(nameRecord.id), new JoinableRecord(nameRecord.getClass().getName(), nameRecord.id, nameRecord.name, -1L))); }
@Override public void map( BalancesRecord balanceRecord, AvroCollector<Pair<KeyRecord, JoinableRecord>> collector, Reporter reporter) throws IOException { collector.collect( new Pair<>( new KeyRecord(balanceRecord.id), new JoinableRecord(balanceRecord.getClass().getName(), balanceRecord.id, "", balanceRecord.balance))); }
@Override public void map(Utf8 text, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { StringTokenizer tokens = new StringTokenizer(text.toString()); while (tokens.hasMoreTokens()) { String tok = tokens.nextToken(); collector.collect(new Pair<>(new Utf8(tok), 1L)); amos.getCollector("myavro2", reporter).collect(new Pair<Utf8, Long>(new Utf8(tok), 1L).toString()); } }
@Override public void reduce(Utf8 word, Iterable<Long> counts, AvroCollector<Pair<Utf8, Long>> collector, Reporter reporter) throws IOException { long sum = 0; for (long count : counts) sum += count; Pair<Utf8, Long> outputvalue = new Pair<>(word, sum); amos.getCollector("myavro", reporter).collect(outputvalue); amos.collect("myavro1", reporter, outputvalue.toString()); amos.collect("myavro", reporter, new Pair<Utf8, Long>(new Utf8(""), 0L).getSchema(), outputvalue, "testavrofile"); amos.collect("myavro", reporter, Schema.create(Schema.Type.STRING), outputvalue.toString(), "testavrofile1"); collector.collect(new Pair<>(word, sum)); }
/** Called with each map input datum. By default, collects inputs. */ @SuppressWarnings("unchecked") public void map(IN datum, AvroCollector<OUT> collector, Reporter reporter) throws IOException { collector.collect((OUT)datum); }