/** * Retrieve the value as written by {@link #addSideInputValue(PCollectionView, WindowedValue)}, * without applying the SDK specific {@link ViewFn}. * * @param view * @param window * @param <T> * @return */ public <T> Iterable<?> getIterable(PCollectionView<T> view, BoundedWindow window) { @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) view.getWindowingStrategyInternal().getWindowFn().windowCoder(); StateTag<ValueState<Iterable<?>>> stateTag = sideInputContentsTags.get(view); ValueState<Iterable<?>> state = stateInternals.state(StateNamespaces.window(windowCoder, window), stateTag); Iterable<?> elements = state.read(); // return empty collection when no side input was received for ready window return (elements != null) ? elements : Collections.emptyList(); }
/** * When a callAfterWindowCloses with the specified view's producing transform, window, and * windowing strategy is invoked, immediately execute the callback. */ private void immediatelyInvokeCallback(PCollectionView<?> view, BoundedWindow window) { doAnswer( invocation -> { Object callback = invocation.getArguments()[3]; Runnable callbackRunnable = (Runnable) callback; callbackRunnable.run(); return null; }) .when(context) .scheduleAfterOutputWouldBeProduced( Mockito.eq(view), Mockito.eq(window), Mockito.eq(view.getWindowingStrategyInternal()), Mockito.any(Runnable.class)); }
/** * Create SideInputs as Broadcast variables. * * @param views The {@link PCollectionView}s. * @param context The {@link JavaSparkContext}. * @param pviews The {@link SparkPCollectionView}. * @return a map of tagged {@link SideInputBroadcast}s and their {@link WindowingStrategy}. */ public static Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> getSideInputs( List<PCollectionView<?>> views, JavaSparkContext context, SparkPCollectionView pviews) { if (views == null) { return ImmutableMap.of(); } else { Map<TupleTag<?>, KV<WindowingStrategy<?, ?>, SideInputBroadcast<?>>> sideInputs = Maps.newHashMap(); for (PCollectionView<?> view : views) { SideInputBroadcast helper = pviews.getPCollectionView(view, context); WindowingStrategy<?, ?> windowingStrategy = view.getWindowingStrategyInternal(); sideInputs.put(view.getTagInternal(), KV.of(windowingStrategy, helper)); } return sideInputs; } }
@Override public AtomicReference<Iterable<? extends WindowedValue<?>>> load( PCollectionViewWindow<?> view) { AtomicReference<Iterable<? extends WindowedValue<?>>> contents = new AtomicReference<>(); WindowingStrategy<?, ?> windowingStrategy = view.getView().getWindowingStrategyInternal(); context.scheduleAfterOutputWouldBeProduced( view.getView(), view.getWindow(), windowingStrategy, new WriteEmptyViewContents(view.getView(), view.getWindow(), contents)); return contents; } }
Mockito.eq(view), Mockito.eq(window), Mockito.eq(view.getWindowingStrategyInternal()), Mockito.any(Runnable.class)); return runLatch;
/** * Add the given value to the internal side-input store of the given side input. This might change * the result of {@link #isReady(PCollectionView, BoundedWindow)} for that side input. */ public void addSideInputValue(PCollectionView<?> sideInput, WindowedValue<Iterable<?>> value) { @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder(); StateTag<ValueState<Iterable<?>>> stateTag = sideInputContentsTags.get(sideInput); for (BoundedWindow window : value.getWindows()) { stateInternals .state(StateNamespaces.window(windowCoder, window), stateTag) .write(value.getValue()); stateInternals .state(StateNamespaces.global(), availableWindowsTags.get(sideInput)) .add(window); } }
addGBK( keyedSideInputStream, sideInput.getValue().getWindowingStrategyInternal(), kvCoder, viewName,
addGBK( keyedSideInputStream, sideInput.getValue().getWindowingStrategyInternal(), kvCoder, viewName,
sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());
sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());
sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());
Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder();
restoredView.getWindowMappingFn(), instanceOf(view.getWindowMappingFn().getClass())); assertThat( restoredView.getWindowingStrategyInternal(), equalTo(view.getWindowingStrategyInternal().fixDefaults())); assertThat(restoredView.getCoderInternal(), equalTo(view.getCoderInternal()));
for (PCollectionView<?> sideInput : (List<PCollectionView<?>>) ((Combine.PerKey) transform).getSideInputs()) { sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());
for (PCollectionView<?> sideInput : (List<PCollectionView<?>>) ((Combine.PerKey) transform).getSideInputs()) { sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());
sideInputStrategies.put(sideInput, sideInput.getWindowingStrategyInternal());