@Override public byte[] readState(DatasetContext datasetContext) { return getKVTable(datasetContext).read(rowKey); }
@Override public void initialize() throws Exception { MapReduceContext context = getContext(); batchPartitionCommitter = PartitionBatchInput.setInput(context, "lines", new KVTableStatePersistor("consumingState", "state.key")); Map<String, String> outputArgs = new HashMap<>(); PartitionKey partitionKey = PartitionKey.builder().addLongField("time", context.getLogicalStartTime()).build(); PartitionedFileSetArguments.setOutputPartitionKey(outputArgs, partitionKey); context.addOutput(Output.ofDataset("outputLines", outputArgs)); context.addOutput(Output.ofDataset("counts")); Job job = context.getHadoopJob(); job.setMapperClass(Tokenizer.class); job.setReducerClass(Counter.class); job.setNumReduceTasks(1); }
TransactionalPartitionConsumer partitionConsumer = new TransactionalPartitionConsumer(getContext(), "lines", new KVTableStatePersistor("consumingState", "state.key")); final List<PartitionDetail> partitions = partitionConsumer.consumePartitions().getPartitions();
@Override public void persistState(DatasetContext datasetContext, byte[] state) { getKVTable(datasetContext).write(rowKey, state); }