public Pair(K key, Schema keySchema, V value, Schema valueSchema) { this.schema = getPairSchema(keySchema, valueSchema); this.key = key; this.value = value; }
@Override protected void setup(Context context) { result = new AvroKey<>(); result.datum(new Record(Pair.getPairSchema(STRING,LONG))); }
public SequenceFileReader(SequenceFile.Reader reader, Configuration conf) { this.reader = reader; this.schema = Pair.getPairSchema(WritableData.get().getSchema(reader.getKeyClass()), WritableData.get().getSchema(reader.getValueClass())); this.key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.spareKey = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf); if (WRITABLE_CONVERTERS.containsKey(reader.getKeyClass()) ) keyConverter = WRITABLE_CONVERTERS.get(reader.getKeyClass()); if (WRITABLE_CONVERTERS.containsKey(reader.getValueClass()) ) valConverter = WRITABLE_CONVERTERS.get(reader.getValueClass()); }
public void checkOutputFormat() throws Exception { Job job = new Job(); WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyTest", "part-r-00000"); wordCountUtil.writeLinesFile(); AvroJob.setInputKeySchema(job, STRING); AvroJob.setOutputKeySchema(job, Pair.getPairSchema(STRING,LONG)); job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); job.setInputFormatClass(AvroKeyInputFormat.class); job.setOutputFormatClass(AvroTrevniKeyOutputFormat.class); job.waitForCompletion(true); wordCountUtil.validateCountsFile(); }
Pair.getPairSchema(Schema.create(Schema.Type.BYTES), Schema.create(Schema.Type.BYTES)));
@Test public void testJob() throws Exception { JobConf job = new JobConf(); Path outputPath = new Path(DIR.getRoot().getPath() + "/out"); outputPath.getFileSystem(job).delete(outputPath); job.setInputFormat(TextInputFormat.class); FileInputFormat.setInputPaths(job, DIR.getRoot().getPath() + "/in"); job.setMapperClass(AvroTestConverter.class); job.setNumReduceTasks(0); FileOutputFormat.setOutputPath(job, outputPath); System.out.println(createSchema()); AvroJob.setOutputSchema(job, Pair.getPairSchema(Schema.create(Schema.Type.LONG), createSchema())); job.setOutputFormat(AvroOutputFormat.class); JobClient.runJob(job); } }
public void testOutputFormat() throws Exception { JobConf job = new JobConf(); WordCountUtil wordCountUtil = new WordCountUtil("trevniMapredTest"); wordCountUtil.writeLinesFile(); AvroJob.setInputSchema(job, STRING); AvroJob.setOutputSchema(job, Pair.getPairSchema(STRING,LONG)); AvroJob.setMapperClass(job, MapImpl.class); AvroJob.setCombinerClass(job, ReduceImpl.class); AvroJob.setReducerClass(job, ReduceImpl.class); FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in")); FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out")); FileOutputFormat.setCompressOutput(job, true); job.setOutputFormat(AvroTrevniOutputFormat.class); JobClient.runJob(job); wordCountUtil.validateCountsFile(); }
@Test public void testJob() throws Exception { JobConf job = new JobConf(); Path inputPath1 = new Path(INPUT_DIR_1.getRoot().getPath()); Path inputPath2 = new Path(INPUT_DIR_2.getRoot().getPath()); Path outputPath = new Path(OUTPUT_DIR.getRoot().getPath()); outputPath.getFileSystem(job).delete(outputPath); writeNamesFiles(new File(inputPath1.toUri().getPath())); writeBalancesFiles(new File(inputPath2.toUri().getPath())); job.setJobName("multiple-inputs-join"); AvroMultipleInputs.addInputPath(job, inputPath1, NamesMapImpl.class, ReflectData.get().getSchema(NamesRecord.class)); AvroMultipleInputs.addInputPath(job, inputPath2, BalancesMapImpl.class, ReflectData.get().getSchema(BalancesRecord.class)); Schema keySchema = ReflectData.get().getSchema(KeyRecord.class); Schema valueSchema = ReflectData.get().getSchema(JoinableRecord.class); AvroJob.setMapOutputSchema(job, Pair.getPairSchema(keySchema, valueSchema)); AvroJob.setOutputSchema(job, ReflectData.get().getSchema(CompleteRecord.class)); AvroJob.setReducerClass(job, ReduceImpl.class); job.setNumReduceTasks(1); FileOutputFormat.setOutputPath(job, outputPath); AvroJob.setReflect(job); JobClient.runJob(job); validateCompleteFile(new File(OUTPUT_DIR.getRoot(), "part-00000.avro")); }
(job, Pair.getPairSchema(Weather.SCHEMA$, Schema.create(Type.NULL))); AvroJob.setOutputSchema(job, Weather.SCHEMA$);
public Pair(K key, Schema keySchema, V value, Schema valueSchema) { this.schema = getPairSchema(keySchema, valueSchema); this.key = key; this.value = value; }
public Pair(K key, Schema keySchema, V value, Schema valueSchema) { this.schema = getPairSchema(keySchema, valueSchema); this.key = key; this.value = value; }
@Override public void initialize() { keyMapFn.initialize(); valueMapFn.initialize(); pairSchemaJson = org.apache.avro.mapred.Pair.getPairSchema( new Schema.Parser().parse(firstJson), new Schema.Parser().parse(secondJson)).toString(); }
@Override public void initialize() { keyMapFn.initialize(); valueMapFn.initialize(); pairSchemaJson = org.apache.avro.mapred.Pair.getPairSchema( new Schema.Parser().parse(firstJson), Avros.allowNulls(new Schema.Parser().parse(secondJson))).toString(); }
@Override public void initialize() { keyMapFn.setContext(getContext()); valueMapFn.setContext(getContext()); pairSchemaJson = org.apache.avro.mapred.Pair.getPairSchema( new Schema.Parser().parse(firstJson), new Schema.Parser().parse(secondJson)).toString(); }
public Schema getMapOutputSchema() { if (_mapOutputSchema == null) { _mapOutputSchema = Pair.getPairSchema(getMapOutputKeySchema(), getMapOutputValueSchema()); } return _mapOutputSchema; }
public Schema getMapOutputSchema() { if (_mapOutputSchema == null) { _mapOutputSchema = Pair.getPairSchema(getMapOutputKeySchema(), getMapOutputValueSchema()); } return _mapOutputSchema; }
@Override public void run() throws Exception { JobConfig props = super.getJobConfig(); JobConf conf = super.createJobConf(ItemModelTestLoglikMapper.class, ItemModelTestLoglikReducer.class, ItemModelTestLoglikCombiner.class, Pair.getPairSchema(Schema.create(Type.STRING), RegressionTestLoglikOutput.SCHEMA$), RegressionTestLoglikOutput.SCHEMA$); AvroUtils.runAvroJob(conf); } public static class ItemModelTestLoglikMapper extends AvroMapper<GenericData.Record, Pair<String, RegressionTestLoglikOutput>>
public AvroTableType(AvroType<K> keyType, AvroType<V> valueType, Class<Pair<K, V>> pairClass) { super(pairClass, org.apache.avro.mapred.Pair.getPairSchema( keyType.getSchema(), valueType.getSchema()), new IndexedRecordToPair(keyType.getInputMapFn(), valueType.getInputMapFn()), new PairToAvroPair(keyType, valueType), keyType, valueType); this.keyType = keyType; this.valueType = valueType; }
public AvroTableType(AvroType<K> keyType, AvroType<V> valueType, Class<Pair<K, V>> pairClass) { super(pairClass, org.apache.avro.mapred.Pair.getPairSchema(keyType.getSchema(), valueType.getSchema()), new IndexedRecordToPair(keyType.getInputMapFn(), valueType.getInputMapFn()), new PairToAvroPair(keyType, valueType), new TupleDeepCopier( Pair.class, keyType, valueType), keyType, valueType); this.keyType = keyType; this.valueType = valueType; }
public AvroTableType(AvroType<K> keyType, AvroType<V> valueType, Class<Pair<K, V>> pairClass) { super(pairClass, org.apache.avro.mapred.Pair.getPairSchema(keyType.getSchema(), Avros.allowNulls(valueType.getSchema())), new IndexedRecordToPair(keyType.getInputMapFn(), valueType.getInputMapFn()), new PairToAvroPair(keyType, valueType), new TupleDeepCopier(Pair.class, keyType, valueType), null, keyType, valueType); this.keyType = keyType; this.valueType = valueType; }