/** Create a simple state tag for values of type {@code T}. */ public static <T> StateTag<ValueState<T>> value(String id, Coder<T> valueCoder) { return new SimpleStateTag<>(new StructuredId(id), StateSpecs.value(valueCoder)); }
GroupIntoBatchesDoFn( long batchSize, Duration allowedLateness, Coder<K> inputKeyCoder, Coder<InputT> inputValueCoder) { this.batchSize = batchSize; this.allowedLateness = allowedLateness; this.batchSpec = StateSpecs.bag(inputValueCoder); this.numElementsInBatchSpec = StateSpecs.combining( new Combine.BinaryCombineLongFn() { @Override public long identity() { return 0L; } @Override public long apply(long left, long right) { return left + right; } }); this.keySpec = StateSpecs.value(inputKeyCoder); // prefetch every 20% of batchSize elements. Do not prefetch if batchSize is too little this.prefetchFrequency = ((batchSize / 5) <= 1) ? Long.MAX_VALUE : (batchSize / 5); }
StatefulPredicateCheck( Coder<T> coder, SerializableFunction<T, String> formatter, SerializableFunction<Set<T>, Boolean> successPredicate) { this.seenEvents = StateSpecs.bag(coder); this.formatter = formatter; this.successPredicate = successPredicate; }
private final StateSpec<SetState<MyInteger>> setState = StateSpecs.set(); StateSpecs.combiningFromInputInternal(VarIntCoder.of(), Sum.ofIntegers());
private final StateSpec<MapState<String, MyInteger>> mapState = StateSpecs.map(); StateSpecs.combiningFromInputInternal(VarIntCoder.of(), Sum.ofIntegers());
/** * Create a state tag for values that use a {@link CombineFnWithContext} to automatically merge * multiple {@code InputT}s into a single {@code OutputT}. */ public static <InputT, AccumT, OutputT> StateTag<CombiningState<InputT, AccumT, OutputT>> combiningValueWithContext( String id, Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) { return new SimpleStateTag<>(new StructuredId(id), StateSpecs.combining(accumCoder, combineFn)); }
/** Create a state spec that supporting for {@link java.util.Map} like access patterns. */ public static <K, V> StateTag<MapState<K, V>> map( String id, Coder<K> keyCoder, Coder<V> valueCoder) { return new SimpleStateTag<>(new StructuredId(id), StateSpecs.map(keyCoder, valueCoder)); }
/** Create a state spec that supporting for {@link java.util.Set} like access patterns. */ public static <T> StateTag<SetState<T>> set(String id, Coder<T> elemCoder) { return new SimpleStateTag<>(new StructuredId(id), StateSpecs.set(elemCoder)); }
/** * Create a state tag for values that use a {@link CombineFn} to automatically merge multiple * {@code InputT}s into a single {@code OutputT}. * * <p>This determines the {@code Coder<AccumT>} from the given {@code Coder<InputT>}, and should * only be used to initialize static values. */ public static <InputT, AccumT, OutputT> StateTag<CombiningState<InputT, AccumT, OutputT>> combiningValueFromInputInternal( String id, Coder<InputT> inputCoder, CombineFn<InputT, AccumT, OutputT> combineFn) { return new SimpleStateTag<>( new StructuredId(id), StateSpecs.combiningFromInputInternal(inputCoder, combineFn)); }
ExactlyOnceWriter(Write<K, V> spec, Coder<KV<K, V>> elemCoder) { this.spec = spec; this.outOfOrderBufferSpec = StateSpecs.bag(KvCoder.of(BigEndianLongCoder.of(), TimestampedValueCoder.of(elemCoder))); }
private final StateSpec<SetState<Integer>> setState = StateSpecs.set(VarIntCoder.of()); StateSpecs.combiningFromInputInternal(VarIntCoder.of(), Sum.ofIntegers());
StateSpecs.map(StringUtf8Coder.of(), VarIntCoder.of()); StateSpecs.combiningFromInputInternal(VarIntCoder.of(), Sum.ofIntegers());
/** * Create a state tag for values that use a {@link CombineFn} to automatically merge multiple * {@code InputT}s into a single {@code OutputT}. */ public static <InputT, AccumT, OutputT> StateTag<CombiningState<InputT, AccumT, OutputT>> combiningValue( String id, Coder<AccumT> accumCoder, CombineFn<InputT, AccumT, OutputT> combineFn) { return new SimpleStateTag<>(new StructuredId(id), StateSpecs.combining(accumCoder, combineFn)); }
private void verifyMapStateUnsupported(PipelineOptions options) throws Exception { Pipeline p = Pipeline.create(options); p.apply(Create.of(KV.of(13, 42))) .apply( ParDo.of( new DoFn<KV<Integer, Integer>, Void>() { @StateId("fizzle") private final StateSpec<MapState<Void, Void>> voidState = StateSpecs.map(); @ProcessElement public void process() {} })); thrown.expectMessage("MapState"); thrown.expect(UnsupportedOperationException.class); p.run(); }
private void verifySetStateUnsupported(PipelineOptions options) throws Exception { Pipeline p = Pipeline.create(options); p.apply(Create.of(KV.of(13, 42))) .apply( ParDo.of( new DoFn<KV<Integer, Integer>, Void>() { @StateId("fizzle") private final StateSpec<SetState<Void>> voidState = StateSpecs.set(); @ProcessElement public void process() {} })); thrown.expectMessage("SetState"); thrown.expect(UnsupportedOperationException.class); p.run(); }
@Test public void testSimpleStateIdAnonymousDoFn() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<KV<String, Integer>, Long>() { @StateId("foo") private final StateSpec<ValueState<Integer>> bizzle = StateSpecs.value(VarIntCoder.of()); @ProcessElement public void foo(ProcessContext context) {} }.getClass()); assertThat(sig.stateDeclarations().size(), equalTo(1)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat(decl.field().getName(), equalTo("bizzle")); assertThat( decl.stateType(), Matchers.<TypeDescriptor<?>>equalTo(new TypeDescriptor<ValueState<Integer>>() {})); }
switch (stateSpec.getSpecCase()) { case VALUE_SPEC: return StateSpecs.value(components.getCoder(stateSpec.getValueSpec().getCoderId())); case BAG_SPEC: return StateSpecs.bag(components.getCoder(stateSpec.getBagSpec().getElementCoderId())); case COMBINING_SPEC: FunctionSpec combineFnSpec = stateSpec.getCombiningSpec().getCombineFn().getSpec(); return StateSpecs.combining( (Coder) components.getCoder(stateSpec.getCombiningSpec().getAccumulatorCoderId()), combineFn); return StateSpecs.map( components.getCoder(stateSpec.getMapSpec().getKeyCoderId()), components.getCoder(stateSpec.getMapSpec().getValueCoderId())); return StateSpecs.set(components.getCoder(stateSpec.getSetSpec().getElementCoderId()));
ExactlyOnceWriter(Write<K, V> spec, Coder<KV<K, V>> elemCoder) { this.spec = spec; this.outOfOrderBufferSpec = StateSpecs.bag(KvCoder.of(BigEndianLongCoder.of(), TimestampedValueCoder.of(elemCoder))); }
private final StateSpec<SetState<MyInteger>> setState = StateSpecs.set(); StateSpecs.combiningFromInputInternal(VarIntCoder.of(), Sum.ofIntegers());