@Override public Object[] createAccumulator() { Object[] accumsArray = new Object[combineFnCount]; for (int i = 0; i < combineFnCount; ++i) { accumsArray[i] = combineFns.get(i).createAccumulator(); } return accumsArray; }
private static <InputT, AccumT, OutputT> List<AccumT> combineInputs( CombineFn<InputT, AccumT, OutputT> fn, Iterable<? extends Iterable<InputT>> shards) { List<AccumT> accumulators = new ArrayList<>(); int maybeCompact = 0; for (Iterable<InputT> shard : shards) { AccumT accumulator = fn.createAccumulator(); for (InputT elem : shard) { accumulator = fn.addInput(accumulator, elem); } if (maybeCompact++ % 2 == 0) { accumulator = fn.compact(accumulator); } accumulators.add(accumulator); } return accumulators; }
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) { WindowedStructuralKey<K> key = WindowedStructuralKey.create(keyCoder, context.element().getKey(), window); AccumT accumulator = accumulators.get(key); Instant assignedTs = timestampCombiner.assign(window, context.timestamp()); if (accumulator == null) { accumulator = combineFn.createAccumulator(); accumulators.put(key, accumulator); timestamps.put(key, assignedTs); } accumulators.put(key, combineFn.addInput(accumulator, context.element().getValue())); timestamps.put(key, timestampCombiner.combine(assignedTs, timestamps.get(key))); }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state." , e); } }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state.", e); } }
@Override public OutputT read() { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT accum = state.value(); if (accum != null) { return combineFn.extractOutput(accum); } else { return combineFn.extractOutput(combineFn.createAccumulator()); } } catch (Exception e) { throw new RuntimeException("Error reading state.", e); } }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state.", e); } }
private static <InputT, AccumT, OutputT> void checkCombineFnShardsIncrementalMerging( CombineFn<InputT, AccumT, OutputT> fn, List<? extends Iterable<InputT>> shards, Matcher<? super OutputT> matcher) { AccumT accumulator = shards.isEmpty() ? fn.createAccumulator() : null; for (AccumT inputAccum : combineInputs(fn, shards)) { if (accumulator == null) { accumulator = inputAccum; } else { accumulator = fn.mergeAccumulators(Arrays.asList(accumulator, inputAccum)); } fn.extractOutput(accumulator); // Extract output to simulate multiple firings } assertThat(fn.extractOutput(accumulator), matcher); }
/** * Applies this {@code CombineFn} to a collection of input values to produce a combined output * value. * * <p>Useful when using a {@code CombineFn} separately from a {@code Combine} transform. Does * not invoke the {@link #mergeAccumulators} operation. */ public OutputT apply(Iterable<? extends InputT> inputs) { AccumT accum = createAccumulator(); for (InputT input : inputs) { accum = addInput(accum, input); } return extractOutput(accum); }
@Override public OutputT read() { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT accum = state.value(); if (accum != null) { return combineFn.extractOutput(accum); } else { return combineFn.extractOutput(combineFn.createAccumulator()); } } catch (Exception e) { throw new RuntimeException("Error reading state.", e); } }
@Override public AccumT createAccumulator() { return fn.createAccumulator(); }
@Override public AccumT createAccumulator(Context c) { return combineFn.createAccumulator(); }
@Override public Object createAccumulator() { return combineFn.createAccumulator(); }
@Override public AccumT createAccumulator() { return fn.createAccumulator(); }
@Override public OutputT read() { return combineFn.extractOutput( combineFn.mergeAccumulators(Arrays.asList(combineFn.createAccumulator(), accum))); }
@Override public void clear() { // Even though we're clearing we can't remove this from the in-memory state map, since // other users may already have a handle on this CombiningValue. accum = combineFn.createAccumulator(); isCleared = true; }
/** * {@inheritDoc} * * <p>By default returns the extract output of an empty accumulator. */ @Override public OutputT defaultValue() { return extractOutput(createAccumulator()); }
@Override public AccumT createAccumulator( PipelineOptions options, SideInputReader sideInputReader, Collection<? extends BoundedWindow> windows) { return combineFn.createAccumulator(); }
private static <InputT, AccumT, OutputT> void checkCombineFnShardsWithEmptyAccumulators( CombineFn<InputT, AccumT, OutputT> fn, Iterable<? extends Iterable<InputT>> shards, Matcher<? super OutputT> matcher) { List<AccumT> accumulators = combineInputs(fn, shards); accumulators.add(0, fn.createAccumulator()); accumulators.add(fn.createAccumulator()); AccumT merged = fn.mergeAccumulators(accumulators); assertThat(fn.extractOutput(merged), matcher); }
public InMemoryCombiningState( CombineFn<InputT, AccumT, OutputT> combineFn, Coder<AccumT> accumCoder) { this.combineFn = combineFn; accum = combineFn.createAccumulator(); this.accumCoder = accumCoder; }