private void setSchema(Job job, Schema keySchema, Schema valSchema) { boolean isMaponly = job.getNumReduceTasks() == 0; if (keySchema != null) { if (isMaponly) AvroJob.setMapOutputKeySchema(job, keySchema); else AvroJob.setOutputKeySchema(job, keySchema); } if (valSchema != null) { if (isMaponly) AvroJob.setMapOutputValueSchema(job, valSchema); else AvroJob.setOutputValueSchema(job, valSchema); } }
public void checkOutputFormat() throws Exception { Job job = new Job(); WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyValueTest", "part-r-00000"); wordCountUtil.writeLinesFile(); AvroJob.setInputKeySchema(job, STRING); AvroJob.setOutputKeySchema(job, STRING); AvroJob.setOutputValueSchema(job, LONG); job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); job.setInputFormatClass(AvroKeyInputFormat.class); job.setOutputFormatClass(AvroTrevniKeyValueOutputFormat.class); job.waitForCompletion(true); wordCountUtil.validateCountsFileGenericRecord(); }
AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));
@Test public void testConvertAvroValue() throws IOException { AvroJob.setOutputValueSchema(mJob, Schema.create(Schema.Type.INT)); AvroValue<Integer> avroValue = new AvroValue<>(42); @SuppressWarnings("unchecked") AvroDatumConverter<AvroValue<Integer>, Integer> converter = mFactory.create( (Class<AvroValue<Integer>>) avroValue.getClass()); assertEquals(42, converter.convert(avroValue).intValue()); }
job.setOutputKeyClass(Text.class); job.setOutputValueClass(AvroValue.class); AvroJob.setOutputValueSchema(job, Schema.createArray(Schema.create(Schema.Type.INT)));
@Test public void testUsingReflection() throws Exception { Job job = new Job(); Schema schema = ReflectData.get().getSchema(R1.class); AvroJob.setOutputValueSchema(job, schema); TaskAttemptContext context = createMock(TaskAttemptContext.class); replay(context);
@Test public void testSyncableWriteRecords() throws IOException { Job job = new Job(); AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$); TaskAttemptContext context = createMock(TaskAttemptContext.class);
@Test public void testWriteRecords() throws IOException { Job job = new Job(); AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$); TaskAttemptContext context = createMock(TaskAttemptContext.class);
private void setSchema(Job job, Schema keySchema, Schema valSchema) { boolean isMaponly = job.getNumReduceTasks() == 0; if (keySchema != null) { if (isMaponly) AvroJob.setMapOutputKeySchema(job, keySchema); else AvroJob.setOutputKeySchema(job, keySchema); } if (valSchema != null) { if (isMaponly) AvroJob.setMapOutputValueSchema(job, valSchema); else AvroJob.setOutputValueSchema(job, valSchema); } }
Schema mapOutputValueSchema = AvroJob.getMapOutputValueSchema(job.getConfiguration()); if (null != mapOutputValueSchema) { AvroJob.setOutputValueSchema(job, mapOutputValueSchema); + "Did you forget to implement AvroValueWriter in your FijiReducer?"); AvroJob.setOutputValueSchema(job, outputValueWriterSchema); } else if (null != outputValueWriterSchema) { throw new JobConfigurationException(reducer.getClass().getName()
@Override public void process(Annotation annotation, Job job, Object target) throws ToolException { AvroJobInfo avroInfo = (AvroJobInfo)annotation; if (avroInfo.inputKeySchema() != AvroDefault.class) { AvroJob.setInputKeySchema(job, getSchema(avroInfo.inputKeySchema())); } if (avroInfo.inputValueSchema() != AvroDefault.class) { AvroJob.setInputValueSchema(job, getSchema(avroInfo.inputValueSchema())); } if (avroInfo.outputKeySchema() != AvroDefault.class) { AvroJob.setOutputKeySchema(job, getSchema(avroInfo.outputKeySchema())); } if (avroInfo.outputValueSchema() != AvroDefault.class) { AvroJob.setOutputValueSchema(job, getSchema(avroInfo.outputValueSchema())); } if (avroInfo.mapOutputKeySchema() != AvroDefault.class) { AvroJob.setMapOutputKeySchema(job, getSchema(avroInfo.mapOutputKeySchema())); } if (avroInfo.mapOutputValueSchema() != AvroDefault.class) { AvroJob.setMapOutputValueSchema(job, getSchema(avroInfo.mapOutputValueSchema())); } AvroSerialization.addToConfiguration(job.getConfiguration()); }