sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName(); } else if (transform instanceof Read.Unbounded) { sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(Read.Unbounded.class) private static void unboundedReadTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final Read.Unbounded<?> transform) { final IRVertex vertex = new BeamUnboundedSourceVertex<>(transform.getSource(), DisplayData.from(transform)); ctx.addVertex(vertex); beamNode.getInputs().values().forEach(input -> ctx.addEdgeTo(vertex, input)); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testTranslate() { ReadUnboundedTranslator translator = new ReadUnboundedTranslator(); GearpumpPipelineOptions options = PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class); Read.Unbounded transform = mock(Read.Unbounded.class); UnboundedSource source = mock(UnboundedSource.class); when(transform.getSource()).thenReturn(source); TranslationContext translationContext = mock(TranslationContext.class); when(translationContext.getPipelineOptions()).thenReturn(options); JavaStream stream = mock(JavaStream.class); PValue mockOutput = mock(PValue.class); when(translationContext.getOutput()).thenReturn(mockOutput); when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream); translator.translate(transform, translationContext); verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher())); verify(translationContext).setOutputStream(mockOutput, stream); } }
context.getCurrentTransform().getFullName(), context.getPipelineOptions(), transform.getSource(), context.getExecutionEnvironment().getParallelism()); nonDedupSource = context .addSource(sourceWrapper).name(transform.getName()).returns(withIdTypeInfo); if (transform.getSource().requiresDeduping()) { source = nonDedupSource.keyBy( new ValueWithRecordIdKeySelector<T>()) "Error while translating UnboundedSource: " + transform.getSource(), e);
public StreamingUnboundedRead(Read.Unbounded<T> transform) { this.source = transform.getSource(); }
@Override public void translate(Read.Unbounded<T> transform, TranslationContext context) { UnboundedSource<T, ?> unboundedSource = transform.getSource(); ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(unboundedSource, context.getPipelineOptions()); context.addOperator(operator, operator.output); } }
public static ReadPayload toProto(Unbounded<?> read, SdkComponents components) { return ReadPayload.newBuilder() .setIsBounded(IsBounded.Enum.UNBOUNDED) .setSource(toProto(read.getSource(), components)) .build(); }
@Override public void translate(Read.Unbounded<T> transform, TranslationContext context) { UnboundedSource<T, ?> unboundedSource = transform.getSource(); UnboundedSourceWrapper<T, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(unboundedSource, context.getPipelineOptions()); JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper); context.setOutputStream(context.getOutput(), sourceStream); } }