String sourceName = "..."; if (transform instanceof Read.Bounded) { sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName(); } else if (transform instanceof Read.Unbounded) { sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
@Override public void translateNode( Read.Bounded<T> transform, FlinkStreamingTranslationContext context) { PCollection<T> output = context.getOutput(transform); TypeInformation<WindowedValue<T>> outputTypeInfo = context.getTypeInfo(context.getOutput(transform)); DataStream<WindowedValue<T>> source; try { BoundedSourceWrapper<T> sourceWrapper = new BoundedSourceWrapper<>( context.getCurrentTransform().getFullName(), context.getPipelineOptions(), transform.getSource(), context.getExecutionEnvironment().getParallelism()); source = context .getExecutionEnvironment() .addSource(sourceWrapper).name(transform.getName()).returns(outputTypeInfo); } catch (Exception e) { throw new RuntimeException( "Error while translating BoundedSource: " + transform.getSource(), e); } context.setOutputDataStream(output, source); } }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testTranslate() { ReadBoundedTranslator translator = new ReadBoundedTranslator(); GearpumpPipelineOptions options = PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class); Read.Bounded transform = mock(Read.Bounded.class); BoundedSource source = mock(BoundedSource.class); when(transform.getSource()).thenReturn(source); TranslationContext translationContext = mock(TranslationContext.class); when(translationContext.getPipelineOptions()).thenReturn(options); JavaStream stream = mock(JavaStream.class); PValue mockOutput = mock(PValue.class); when(translationContext.getOutput()).thenReturn(mockOutput); when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream); translator.translate(transform, translationContext); verify(translationContext).getSourceStream(argThat(new BoundedSourceWrapperMatcher())); verify(translationContext).setOutputStream(mockOutput, stream); } }
public StreamingBoundedRead(Read.Bounded<T> transform) { this.source = transform.getSource(); }
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(Read.Bounded.class) private static void boundedReadTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final Read.Bounded<?> transform) { final IRVertex vertex = new BeamBoundedSourceVertex<>(transform.getSource(), DisplayData.from(transform)); ctx.addVertex(vertex); beamNode.getInputs().values().forEach(input -> ctx.addEdgeTo(vertex, input)); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }
@Override public void translateNode(Read.Bounded<T> transform, FlinkBatchTranslationContext context) { String name = transform.getName(); BoundedSource<T> source = transform.getSource(); PCollection<T> output = context.getOutput(transform); TypeInformation<WindowedValue<T>> typeInformation = context.getTypeInfo(output); DataSource<WindowedValue<T>> dataSource = new DataSource<>( context.getExecutionEnvironment(), new SourceInputFormat<>( context.getCurrentTransform().getFullName(), source, context.getPipelineOptions()), typeInformation, name); context.setOutputDataSet(output, dataSource); } }
@Override public void translate(Read.Bounded<?> transform, TranslationContext context) { translateReadHelper(transform.getSource(), transform, context); }
@Override public void translate(Read.Bounded<T> transform, TranslationContext context) { // TODO: adapter is visibleForTesting BoundedToUnboundedSourceAdapter unboundedSource = new BoundedToUnboundedSourceAdapter<>(transform.getSource()); ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(unboundedSource, true, context.getPipelineOptions()); context.addOperator(operator, operator.output); } }
public static ReadPayload toProto(Read.Bounded<?> read, SdkComponents components) { return ReadPayload.newBuilder() .setIsBounded(IsBounded.Enum.BOUNDED) .setSource(toProto(read.getSource(), components)) .build(); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform) { return PTransformReplacement.of( transform.getPipeline().begin(), JavaReadViaImpulse.bounded(transform.getTransform().getSource())); }
/** * Returns a new {@code Read.Bounded} {@code PTransform} reading from the given {@code * BoundedSource}. */ public <T> Bounded<T> from(BoundedSource<T> source) { return new Bounded<>(name, source); }
/** * Returns a new {@code Read.Bounded} {@code PTransform} reading from the given {@code * BoundedSource}. */ public static <T> Bounded<T> from(BoundedSource<T> source) { return new Bounded<>(null, source); }
@Override public void translate(Read.Bounded<T> transform, TranslationContext context) { BoundedSource<T> boundedSource = transform.getSource(); BoundedSourceWrapper<T> sourceWrapper = new BoundedSourceWrapper<>(boundedSource, context.getPipelineOptions()); JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(sourceWrapper); context.setOutputStream(context.getOutput(), sourceStream); } }