@Override public int compare(AvroValue<GenericRecord> o1, AvroValue<GenericRecord> o2) { GenericRecord record1= o1.datum(); GenericRecord record2 = o2.datum(); for (String deltaFieldName : this.deltaSchemaProvider.getDeltaFieldNames(record1)) { if (record1.get(deltaFieldName).equals(record2.get(deltaFieldName))) { continue; } return ((Comparable)record1.get(deltaFieldName)).compareTo(record2.get(deltaFieldName)); } return 0; } }
/** * Creates a new empty <code>AvroValue</code> instance. * * @return a new empty AvroValue. */ @Override protected AvroWrapper<D> createAvroWrapper() { return new AvroValue<>(null); } }
AvroValue<GenericRecord> fullRecord1 = new AvroValue<>(); AvroValue<GenericRecord> fullRecord2 = new AvroValue<>(); AvroValue<GenericRecord> fullRecord3 = new AvroValue<>(); AvroValue<GenericRecord> fullRecord4 = new AvroValue<>(); fullRecordBuilder1.set("scn", 123); fullRecordBuilder1.set("scn2", 100); fullRecord1.datum(fullRecordBuilder1.build()); fullRecordBuilder1.set("scn", 125); fullRecordBuilder1.set("scn2", 1); fullRecord2.datum(fullRecordBuilder1.build()); fullRecordBuilder1.set("scn", 124); fullRecordBuilder1.set("scn2", 10); fullRecord3.datum(fullRecordBuilder1.build()); fullRecordBuilder1.set("scn", 122); fullRecordBuilder1.set("scn2", 1000); fullRecord4.datum(fullRecordBuilder1.build()); key.datum(keyRecord); reducer.reduce(key, valueIterable, reducerContext); Assert.assertEquals(reducer.getOutKey().datum(), fullRecord2.datum()); reducer2.setup(reducerContext); reducer2.reduce(key, valueIterable, reducerContext); Assert.assertEquals(reducer2.getOutKey().datum(), fullRecord1.datum()); fullRecordBuilder2.set("scn", 123); fullRecordBuilder2.set("scn2", 100); fullRecord1.datum(fullRecordBuilder2.build());
new AvroKey<CharSequence>("one"), new AvroValue<>(1), new AvroKey<CharSequence>("two"), new AvroValue<>(2)); AvroValue<Integer> value = new AvroValue<>(); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(1, value.datum().intValue()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(2, value.datum().intValue());
@Override protected void map(AvroKey<String> key, AvroValue<Long> value, Context context) throws IOException, InterruptedException { total += value.datum(); } }
@Override protected void setup(Context context) throws IOException, InterruptedException { this.keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration()); this.outKey = new AvroKey<>(); this.outKey.datum(new GenericData.Record(this.keySchema)); this.outValue = new AvroValue<>(); }
new AvroKey<CharSequence>("one"), new AvroValue<>(1), new AvroKey<CharSequence>("two"), new AvroValue<>(2)); AvroValue<Integer> value = new AvroValue<>(); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(1, value.datum().intValue()); value = (AvroValue<Integer>) reader.getCurrentValue(value); assertNotNull(value); assertEquals(2, value.datum().intValue());
@Override public int getPartition(AvroKey<ByteBuffer> key, AvroValue<ByteBuffer> value, int numReduceTasks) { return getPartition(key.datum().array(), value.datum().array(), numReduceTasks); } }
/** * Constructor. * * @param keyReaderSchema The reader schema for the key within the generic record. * @param valueReaderSchema The reader schema for the value within the generic record. */ public AvroKeyValueRecordReader(Schema keyReaderSchema, Schema valueReaderSchema) { super(AvroKeyValue.getSchema(keyReaderSchema, valueReaderSchema)); mCurrentKey = new AvroKey<>(null); mCurrentValue = new AvroValue<>(null); }
@SuppressWarnings("unchecked") @Override public void reduce(AvroKey<ByteBuffer> keyAvro, Iterator<AvroValue<ByteBuffer>> iterator, OutputCollector<Text, Text> collector, Reporter reporter) throws IOException { ByteBuffer keyBuffer = keyAvro.datum(); keyBuffer.rewind(); byte[] keyBytes = null, valueBytes; keyBytes = new byte[keyBuffer.remaining()]; keyBuffer.get(keyBytes); BytesWritable key = new BytesWritable(keyBytes); ArrayList<BytesWritable> valueList = new ArrayList(); while(iterator.hasNext()) { ByteBuffer writable = iterator.next().datum(); writable.rewind(); // BytesWritable writable = iterator.next(); valueBytes = null; valueBytes = new byte[writable.remaining()]; writable.get(valueBytes); BytesWritable value = new BytesWritable(valueBytes); valueList.add(value); } writer.write(key, valueList.iterator(), reporter); }
@Override protected void reduce(Text token, Iterable<IntWritable> docids, Context context) throws IOException, InterruptedException { List<Integer> hitlist = new ArrayList<>(); for (IntWritable docid : docids) { hitlist.add(docid.get()); } context.write(token, new AvroValue<>(hitlist)); } }
GenericRecord value = ((AvroValue<GenericRecord>)valueObj).datum(); if (value.getSchema().getFullName().equals(getSchemas().getIntermediateValueSchema().getFullName())) context.write((AvroKey<GenericRecord>)keyObj,new AvroValue<GenericRecord>(value)); context.write((AvroKey<GenericRecord>)keyObj,new AvroValue<GenericRecord>(value)); if (intermediateValue != null) context.write((AvroKey<GenericRecord>)keyObj,new AvroValue<GenericRecord>(intermediateValue));
@Override protected void map(AvroKey<Integer> docid, AvroValue<CharSequence> body, Context context) throws IOException, InterruptedException { for (String token : body.datum().toString().split(" ")) { context.write(new Text(token), new IntWritable(docid.datum())); } } }
@Override protected void reduce(Text key, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable count : counts) { sum += count.get(); } context.write(new AvroKey<>(key.toString()), new AvroValue<>(sum)); } }
@Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable value: values) { sum += value.get(); } resultKey.datum(key.toString()); resultValue.datum(sum); context.write(resultKey, resultValue); } }
public void map(LongWritable key, Text value, OutputCollector<AvroKey<Long>,AvroValue<Utf8>> out, Reporter reporter) throws IOException { out.collect(new AvroKey<>(key.get()), new AvroValue<>(new Utf8(value.toString()))); } }
/** {@inheritDoc} */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { boolean hasNext = super.nextKeyValue(); if (hasNext) { AvroKeyValue<K, V> avroKeyValue = new AvroKeyValue<>(getCurrentRecord()); mCurrentKey.datum(avroKeyValue.getKey()); mCurrentValue.datum(avroKeyValue.getValue()); } else { mCurrentKey.datum(null); mCurrentValue.datum(null); } return hasNext; }
@Test public void testConvertAvroValue() throws IOException { AvroJob.setOutputValueSchema(mJob, Schema.create(Schema.Type.INT)); AvroValue<Integer> avroValue = new AvroValue<>(42); @SuppressWarnings("unchecked") AvroDatumConverter<AvroValue<Integer>, Integer> converter = mFactory.create( (Class<AvroValue<Integer>>) avroValue.getClass()); assertEquals(42, converter.convert(avroValue).intValue()); }
@Override protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context) throws IOException, InterruptedException { if (context.getNumReduceTasks() == 0) { context.write(key, NullWritable.get()); } else { populateComparableKeyRecord(key.datum(), this.outKey.datum()); this.outValue.datum(key.datum()); try { context.write(this.outKey, this.outValue); } catch (AvroRuntimeException e) { final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths(); throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e); } } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); }