@Override public <T, V, W extends BoundedWindow> SideInputHandler<V, W> forSideInput( String transformId, String sideInputId, RunnerApi.FunctionSpec accessPattern, Coder<T> elementCoder, Coder<W> windowCoder) { PCollectionView collectionNode = sideInputToCollection.get( SideInputId.newBuilder().setTransformId(transformId).setLocalName(sideInputId).build()); checkArgument(collectionNode != null, "No side input for %s/%s", transformId, sideInputId); if (PTransformTranslation.ITERABLE_SIDE_INPUT.equals(accessPattern.getUrn())) { @SuppressWarnings("unchecked") // T == V Coder<V> outputCoder = (Coder<V>) elementCoder; return forIterableSideInput(collectionNode, outputCoder); } else if (PTransformTranslation.MULTIMAP_SIDE_INPUT.equals(accessPattern.getUrn()) || Materializations.MULTIMAP_MATERIALIZATION_URN.equals(accessPattern.getUrn())) { // TODO: Remove non standard URN. // Using non standard version of multimap urn as dataflow uses the non standard urn. @SuppressWarnings("unchecked") // T == KV<?, V> KvCoder<?, V> kvCoder = (KvCoder<?, V>) elementCoder; return forMultimapSideInput(collectionNode, kvCoder.getKeyCoder(), kvCoder.getValueCoder()); } else { throw new IllegalArgumentException( String.format("Unknown side input access pattern: %s", accessPattern)); } }
@Override public <T, V, W extends BoundedWindow> SideInputHandler<V, W> forSideInput( String transformId, String sideInputId, RunnerApi.FunctionSpec accessPattern, Coder<T> elementCoder, Coder<W> windowCoder) { PCollectionView collectionNode = sideInputToCollection.get( SideInputId.newBuilder().setTransformId(transformId).setLocalName(sideInputId).build()); checkArgument(collectionNode != null, "No side input for %s/%s", transformId, sideInputId); if (PTransformTranslation.ITERABLE_SIDE_INPUT.equals(accessPattern.getUrn())) { @SuppressWarnings("unchecked") // T == V Coder<V> outputCoder = (Coder<V>) elementCoder; return forIterableSideInput(collectionNode, outputCoder); } else if (PTransformTranslation.MULTIMAP_SIDE_INPUT.equals(accessPattern.getUrn()) || Materializations.MULTIMAP_MATERIALIZATION_URN.equals(accessPattern.getUrn())) { // TODO: Remove non standard URN. // Using non standard version of multimap urn as dataflow uses the non standard urn. @SuppressWarnings("unchecked") // T == KV<?, V> KvCoder<?, V> kvCoder = (KvCoder<?, V>) elementCoder; return forMultimapSideInput(collectionNode, kvCoder.getKeyCoder(), kvCoder.getValueCoder()); } else { throw new IllegalArgumentException( String.format("Unknown side input access pattern: %s", accessPattern)); } }
@Override protected boolean matchesSafely(ExecutableStage item) { return item.getInputPCollection().getId().equals(inputPCollectionId) && containsInAnyOrder(sideInputIds.toArray()) .matches( item.getSideInputs() .stream() .map( ref -> SideInputId.newBuilder() .setTransformId(ref.transform().getId()) .setLocalName(ref.localName()) .build()) .collect(Collectors.toSet())) && materializedPCollection.matches( item.getOutputPCollections() .stream() .map(PCollectionNode::getId) .collect(Collectors.toSet())) && containsInAnyOrder(fusedTransforms.toArray(new String[0])) .matches( item.getTransforms() .stream() .map(PTransformNode::getId) .collect(Collectors.toSet())); }
.withSideInputs( RunnerApi.ExecutableStagePayload.SideInputId.newBuilder() .setTransformId("sideParDo") .setLocalName("side") .build())
payload.addSideInputs( SideInputId.newBuilder() .setTransformId(sideInput.transform().getId()) .setLocalName(sideInput.localName()));
SideInputId.newBuilder().setTransformId(transformId).setLocalName(sideInputId).build()); checkArgument(collectionNode != null, "No side input for %s/%s", transformId, sideInputId);
/** * Creates a new state handler for the given stage. Note that this requires a traversal of the * stage itself, so this should only be called once per stage rather than once per bundle. */ public static FlinkStreamingSideInputHandlerFactory forStage( ExecutableStage stage, Map<SideInputId, PCollectionView<?>> viewMapping, org.apache.beam.runners.core.SideInputHandler runnerHandler) { ImmutableMap.Builder<SideInputId, PCollectionView<?>> sideInputBuilder = ImmutableMap.builder(); for (SideInputReference sideInput : stage.getSideInputs()) { SideInputId sideInputId = SideInputId.newBuilder() .setTransformId(sideInput.transform().getId()) .setLocalName(sideInput.localName()) .build(); sideInputBuilder.put( sideInputId, checkNotNull( viewMapping.get(sideInputId), "No side input for %s/%s", sideInputId.getTransformId(), sideInputId.getLocalName())); } FlinkStreamingSideInputHandlerFactory factory = new FlinkStreamingSideInputHandlerFactory(sideInputBuilder.build(), runnerHandler); return factory; }
SideInputId.newBuilder().setTransformId(transformId).setLocalName(sideInputId).build()); checkArgument(collectionNode != null, "No side input for %s/%s", transformId, sideInputId);
/** * Creates a new state handler for the given stage. Note that this requires a traversal of the * stage itself, so this should only be called once per stage rather than once per bundle. */ public static FlinkStreamingSideInputHandlerFactory forStage( ExecutableStage stage, Map<SideInputId, PCollectionView<?>> viewMapping, org.apache.beam.runners.core.SideInputHandler runnerHandler) { ImmutableMap.Builder<SideInputId, PCollectionView<?>> sideInputBuilder = ImmutableMap.builder(); for (SideInputReference sideInput : stage.getSideInputs()) { SideInputId sideInputId = SideInputId.newBuilder() .setTransformId(sideInput.transform().getId()) .setLocalName(sideInput.localName()) .build(); sideInputBuilder.put( sideInputId, checkNotNull( viewMapping.get(sideInputId), "No side input for %s/%s", sideInputId.getTransformId(), sideInputId.getLocalName())); } FlinkStreamingSideInputHandlerFactory factory = new FlinkStreamingSideInputHandlerFactory(sideInputBuilder.build(), runnerHandler); return factory; }
/** * Creates a new state handler for the given stage. Note that this requires a traversal of the * stage itself, so this should only be called once per stage rather than once per bundle. */ static FlinkBatchSideInputHandlerFactory forStage( ExecutableStage stage, RuntimeContext runtimeContext) { ImmutableMap.Builder<SideInputId, PCollectionNode> sideInputBuilder = ImmutableMap.builder(); for (SideInputReference sideInput : stage.getSideInputs()) { sideInputBuilder.put( SideInputId.newBuilder() .setTransformId(sideInput.transform().getId()) .setLocalName(sideInput.localName()) .build(), sideInput.collection()); } return new FlinkBatchSideInputHandlerFactory(sideInputBuilder.build(), runtimeContext); }
/** * Creates a new state handler for the given stage. Note that this requires a traversal of the * stage itself, so this should only be called once per stage rather than once per bundle. */ static FlinkBatchSideInputHandlerFactory forStage( ExecutableStage stage, RuntimeContext runtimeContext) { ImmutableMap.Builder<SideInputId, PCollectionNode> sideInputBuilder = ImmutableMap.builder(); for (SideInputReference sideInput : stage.getSideInputs()) { sideInputBuilder.put( SideInputId.newBuilder() .setTransformId(sideInput.transform().getId()) .setLocalName(sideInput.localName()) .build(), sideInput.collection()); } return new FlinkBatchSideInputHandlerFactory(sideInputBuilder.build(), runtimeContext); }