public static void useSortableBytesAsMapOutputKey(Job job, Class<? extends Partitioner> partitionerClass) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(partitionerClass); }
/** * Sets the map output key schema. * * @param job The job to configure. * @param schema The map output key schema. */ public static void setMapOutputKeySchema(Job job, Schema schema) { job.setMapOutputKeyClass(AvroKey.class); job.setGroupingComparatorClass(AvroKeyComparator.class); job.setSortComparatorClass(AvroKeyComparator.class); AvroSerialization.setKeyWriterSchema(job.getConfiguration(), schema); AvroSerialization.setKeyReaderSchema(job.getConfiguration(), schema); AvroSerialization.addToConfiguration(job.getConfiguration()); }
/** * @param combiner With combiner. * @throws Exception If failed. */ public void doTestGrouping(boolean combiner) throws Exception { HadoopGroupingTestState.values().clear(); Job job = Job.getInstance(); job.setInputFormatClass(InFormat.class); job.setOutputFormatClass(OutFormat.class); job.setOutputKeyClass(YearTemperature.class); job.setOutputValueClass(Text.class); job.setMapperClass(Mapper.class); if (combiner) { job.setCombinerClass(MyReducer.class); job.setNumReduceTasks(0); job.setCombinerKeyGroupingComparatorClass(YearComparator.class); } else { job.setReducerClass(MyReducer.class); job.setNumReduceTasks(4); job.setGroupingComparatorClass(YearComparator.class); } grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration(), null)).get(30000); assertTrue(HadoopGroupingTestState.values().isEmpty()); }
job.setGroupingComparatorClass(NaturalKeyGroupingComparator.class); job.setSortComparatorClass(CompositeKeyComparator.class);
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); Job job = new Job(conf); job.setJarByClass(SecondarySortDriver.class); job.setJobName("SecondarySortDriver"); // args[0] = input directory // args[1] = output directory FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setOutputKeyClass(DateTemperaturePair.class); job.setOutputValueClass(Text.class); job.setMapperClass(SecondarySortMapper.class); job.setReducerClass(SecondarySortReducer.class); job.setPartitionerClass(DateTemperaturePartitioner.class); job.setGroupingComparatorClass(DateTemperatureGroupingComparator.class); boolean status = job.waitForCompletion(true); theLogger.info("run(): status="+status); return status ? 0 : 1; }
job.setGroupingComparatorClass(SecondarySortGroupComparator.class);
job.setGroupingComparatorClass(NaturalKeyGroupingComparator.class); job.setSortComparatorClass(CompositeKeyComparator.class);
/** * Wraps the group comparator defined in the job with this {@link GroupComparatorWrapper} if it is defined. * @param job The MapReduce job */ static void wrap(Job job) { if (WrapperUtil.setIfDefined(job, MRJobConfig.GROUP_COMPARATOR_CLASS, ATTR_CLASS)) { job.setGroupingComparatorClass(GroupComparatorWrapper.class); } }
/** * Wraps the group comparator defined in the job with this {@link GroupComparatorWrapper} if it is defined. * @param job The MapReduce job */ static void wrap(Job job) { if (WrapperUtil.setIfDefined(job, MRJobConfig.GROUP_COMPARATOR_CLASS, ATTR_CLASS)) { job.setGroupingComparatorClass(GroupComparatorWrapper.class); } }
/** * @param job */ public static void setTuplePairSecondarySorting(Job job) { job.setGroupingComparatorClass(SecondarySort.TuplePairGroupComprator.class); job.setPartitionerClass(SecondarySort.TuplePairPartitioner.class); } }
public void configure(Job job) { if (partitionerClass != null) { job.setPartitionerClass(partitionerClass); } if (groupingComparatorClass != null) { job.setGroupingComparatorClass(groupingComparatorClass); } if (sortComparatorClass != null) { job.setSortComparatorClass(sortComparatorClass); } if (numReducers > 0) { job.setNumReduceTasks(numReducers); } for (Map.Entry<String, String> e : extraConf.entrySet()) { job.getConfiguration().set(e.getKey(), e.getValue()); } }
public static void useSortableBytesAsMapOutputKey(Job job) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(SortableBytesPartitioner.class); }
public static void useSortableBytesAsMapOutputKey(Job job, Class<? extends Partitioner> partitionerClass) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(partitionerClass); }
public static void useSortableBytesAsMapOutputKey(Job job) { job.setMapOutputKeyClass(BytesWritable.class); job.setGroupingComparatorClass(SortableBytesGroupingComparator.class); job.setSortComparatorClass(SortableBytesSortingComparator.class); job.setPartitionerClass(SortableBytesPartitioner.class); }
public void configure(Job job) { if (partitionerClass != null) { job.setPartitionerClass(partitionerClass); } if (groupingComparatorClass != null) { job.setGroupingComparatorClass(groupingComparatorClass); } if (sortComparatorClass != null) { job.setSortComparatorClass(sortComparatorClass); } if (numReducers > 0) { job.setNumReduceTasks(numReducers); } }
public void configure(Job job) { if (partitionerClass != null) { job.setPartitionerClass(partitionerClass); } if (groupingComparatorClass != null) { job.setGroupingComparatorClass(groupingComparatorClass); } if (sortComparatorClass != null) { job.setSortComparatorClass(sortComparatorClass); } if (numReducers > 0) { job.setNumReduceTasks(numReducers); } }
/** * Sets the map output key schema. * * @param job The job to configure. * @param schema The map output key schema. */ public static void setMapOutputKeySchema(Job job, Schema schema) { job.setMapOutputKeyClass(AvroKey.class); job.setGroupingComparatorClass(AvroKeyComparator.class); job.setSortComparatorClass(AvroKeyComparator.class); AvroSerialization.setKeyWriterSchema(job.getConfiguration(), schema); AvroSerialization.setKeyReaderSchema(job.getConfiguration(), schema); AvroSerialization.addToConfiguration(job.getConfiguration()); }
@Override protected void configureReasoningJob(String[] args) throws Exception { distributeSchema(); Configuration conf = job.getConfiguration(); // We can ignore irrelevant triples, unless the schema has just changed // and therefore we can't rely on previous determinations of relevance. configureMultipleInput(TableMapper.class, RdfMapper.class, FileMapper.class, !MRReasoningUtils.isSchemaNew(conf)); job.setMapOutputKeyClass(ResourceWritable.class); job.setMapOutputValueClass(Fact.class); job.setReducerClass(ReasoningReducer.class); job.setSortComparatorClass(ResourceWritable.SecondaryComparator.class); job.setGroupingComparatorClass(ResourceWritable.PrimaryComparator.class); configureDerivationOutput(true); }
@Override public void process(Annotation annotation, Job job, Object target) { Grouping grouping = (Grouping)annotation; if (grouping != null && grouping.value() != null && grouping.value() != NULLCOMPARATOR.class) { job.setGroupingComparatorClass(grouping.value()); } }
@Override public void initialize() throws Exception { String inputPath = Bytes.toString(table.read(Bytes.toBytes("inputPath"))); String outputPath = Bytes.toString(table.read(Bytes.toBytes("outputPath"))); Job hadoopJob = getContext().getHadoopJob(); WordCount.configureJob(hadoopJob, inputPath, outputPath); hadoopJob.setPartitionerClass(SimplePartitioner.class); hadoopJob.setNumReduceTasks(2); hadoopJob.setGroupingComparatorClass(SimpleComparator.class); hadoopJob.setSortComparatorClass(SimpleComparator.class); hadoopJob.setCombinerKeyGroupingComparatorClass(SimpleComparator.class); } }