/** * See {@link #setInput(MapReduceContext, String, DatasetStatePersistor, ConsumerConfiguration)}, * but using {@link ConsumerConfiguration#DEFAULT}. */ public static BatchPartitionCommitter setInput(MapReduceContext mapreduceContext, String partitionedFileSetName, DatasetStatePersistor statePersistor) { return setInput(mapreduceContext, partitionedFileSetName, statePersistor, ConsumerConfiguration.DEFAULT); }
@Override public void initialize() throws Exception { MapReduceContext context = getContext(); batchPartitionCommitter = PartitionBatchInput.setInput(context, "lines", new KVTableStatePersistor("consumingState", "state.key")); Map<String, String> outputArgs = new HashMap<>(); PartitionKey partitionKey = PartitionKey.builder().addLongField("time", context.getLogicalStartTime()).build(); PartitionedFileSetArguments.setOutputPartitionKey(outputArgs, partitionKey); context.addOutput(Output.ofDataset("outputLines", outputArgs)); context.addOutput(Output.ofDataset("counts")); Job job = context.getHadoopJob(); job.setMapperClass(Tokenizer.class); job.setReducerClass(Counter.class); job.setNumReduceTasks(1); }