@Override public void encode(KeyedWorkItem<K, ElemT> value, OutputStream outStream) throws CoderException, IOException { keyCoder.encode(value.key(), outStream); timersCoder.encode(value.timersIterable(), outStream); elemsCoder.encode(value.elementsIterable(), outStream); }
private static <T> BoundedWindow getUnderlyingWindow(KeyedWorkItem<byte[], T> kwi) { if (Iterables.isEmpty(kwi.elementsIterable())) { // ProcessFn sets only a single timer. TimerData timer = Iterables.getOnlyElement(kwi.timersIterable()); return ((WindowNamespace) timer.getNamespace()).getWindow(); } else { // KWI must have a single element in elementsIterable, because it follows a GBK by a // uniquely generated key. // Additionally, windows must be exploded before GBKIntoKeyedWorkItems, so there's also // only a single window. WindowedValue<T> value = Iterables.getOnlyElement(kwi.elementsIterable()); return Iterables.getOnlyElement(value.getWindows()); } }
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>> createTestHarness(WindowDoFnOperator<Long, Long, Long> windowDoFnOperator) throws Exception { return new KeyedOneInputStreamOperatorTestHarness<>( windowDoFnOperator, (KeySelector<WindowedValue<KeyedWorkItem<Long, Long>>, ByteBuffer>) o -> { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { VarLongCoder.of().encode(o.getValue().key(), baos); return ByteBuffer.wrap(baos.toByteArray()); } }, new GenericTypeInfo<>(ByteBuffer.class)); }
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>> createTestHarness(WindowDoFnOperator<Long, Long, Long> windowDoFnOperator) throws Exception { return new KeyedOneInputStreamOperatorTestHarness<>( windowDoFnOperator, (KeySelector<WindowedValue<KeyedWorkItem<Long, Long>>, ByteBuffer>) o -> { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { VarLongCoder.of().encode(o.getValue().key(), baos); return ByteBuffer.wrap(baos.toByteArray()); } }, new GenericTypeInfo<>(ByteBuffer.class)); }
@Override public boolean equals(Object other) { if (other == null || !(other instanceof ComposedKeyedWorkItem)) { return false; } KeyedWorkItem<?, ?> that = (KeyedWorkItem<?, ?>) other; return Objects.equals(this.key, that.key()) && Iterables.elementsEqual(this.timersIterable(), that.timersIterable()) && Iterables.elementsEqual(this.elementsIterable(), that.elementsIterable()); }
WindowedValueCoder<InputT> wvCoder = (WindowedValueCoder) windowedInputCoder; if (value instanceof KeyedWorkItem) { key = ((KeyedWorkItem) value).key(); @SuppressWarnings({"rawtypes", "unchecked"}) KeyedWorkItemCoder<Object, ?> kwiCoder = (KeyedWorkItemCoder) wvCoder.getValueCoder();
@Override public void processElement(WindowedValue<KeyedWorkItem<K, InputT>> elem) { Iterable<WindowedValue<InputT>> nonLateElements = lateDataFilter.filter(elem.getValue().key(), elem.getValue().elementsIterable()); KeyedWorkItem<K, InputT> keyedWorkItem = KeyedWorkItems.workItem( elem.getValue().key(), elem.getValue().timersIterable(), nonLateElements); doFnRunner.processElement(elem.withValue(keyedWorkItem)); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { KeyedWorkItem<K, InputT> keyedWorkItem = c.element(); K key = keyedWorkItem.key(); StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key); TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key); ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner = new ReduceFnRunner<>( key, windowingStrategy, ExecutableTriggerStateMachine.create( TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger()))), stateInternals, timerInternals, outputWindowedValue(), sideInputReader, reduceFn, c.getPipelineOptions()); reduceFnRunner.processElements(keyedWorkItem.elementsIterable()); reduceFnRunner.onTimers(keyedWorkItem.timersIterable()); reduceFnRunner.persist(); } }
@ProcessElement public void processElement(final ProcessContext c) { byte[] key = c.element().key(); StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key); TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key); TimerInternals.TimerData timer = Iterables.getOnlyElement(c.element().timersIterable(), null); boolean isSeedCall = (timer == null); StateNamespace stateNamespace; if (isSeedCall) { WindowedValue<KV<InputT, RestrictionT>> windowedValue = Iterables.getOnlyElement(c.element().elementsIterable()); BoundedWindow window = Iterables.getOnlyElement(windowedValue.getWindows()); stateNamespace = if (isSeedCall) { WindowedValue<KV<InputT, RestrictionT>> windowedValue = Iterables.getOnlyElement(c.element().elementsIterable()); WindowedValue<InputT> element = windowedValue.withValue(windowedValue.getValue().getKey()); elementState.write(element);