try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getTransform().getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn());
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString()) .build()) .build();
.setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build(); PTransform read2Transform = .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build(); .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py"))
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build()) .putPcollections(
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("py.out", pc("py.out"))
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("rare")) .build() .toByteString())) .build()) .putPcollections(
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("py.out", PCollection.newBuilder().setUniqueName("py.out").build())
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("rare")) .build() .toByteString())) .build()) .putPcollections(
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("py.out", pc("py.out"))
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("window.out", pc("window.out"))
.setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build();
components.registerEnvironment(Environments.createDockerEnvironment("java")); WindowIntoPayload payload = WindowIntoPayload.newBuilder() .setWindowFn( WindowingStrategyTranslation.toProto( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload(payload.toByteString()) .build()) .build(); components .toComponents() .getEnvironmentsOrThrow(payload.getWindowFn().getEnvironmentId())));
try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getTransform().getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn());
public static WindowIntoPayload getWindowIntoPayload(AppliedPTransform<?, ?, ?> application) { RunnerApi.PTransform transformProto; try { SdkComponents components = SdkComponents.create(application.getPipeline().getOptions()); transformProto = PTransformTranslation.toProto(application, Collections.emptyList(), components); } catch (IOException exc) { throw new RuntimeException(exc); } checkArgument( PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN.equals( transformProto.getSpec().getUrn()), "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"", Window.Assign.class.getSimpleName(), application.getTransform(), application.getFullName(), transformProto.getSpec().getUrn()); try { return WindowIntoPayload.parseFrom(transformProto.getSpec().getPayload()); } catch (InvalidProtocolBufferException exc) { throw new IllegalStateException( String.format( "%s translated %s with URN '%s' but payload was not a %s", PTransformTranslation.class.getSimpleName(), application, PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, WindowIntoPayload.class.getSimpleName()), exc); } }
private <T> void translateAssignWindows( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); RunnerApi.WindowIntoPayload payload; try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); } //TODO: https://issues.apache.org/jira/browse/BEAM-4296 // This only works for well known window fns, we should defer this execution to the SDK // if the WindowFn can't be parsed or just defer it all the time. WindowFn<T, ? extends BoundedWindow> windowFn = (WindowFn<T, ? extends BoundedWindow>) WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn()); String inputCollectionId = Iterables.getOnlyElement(transform.getInputsMap().values()); String outputCollectionId = Iterables.getOnlyElement(transform.getOutputsMap().values()); Coder<WindowedValue<T>> outputCoder = instantiateCoder(outputCollectionId, components); TypeInformation<WindowedValue<T>> resultTypeInfo = new CoderTypeInformation<>(outputCoder); DataStream<WindowedValue<T>> inputDataStream = context.getDataStreamOrThrow(inputCollectionId); FlinkAssignWindows<T, ? extends BoundedWindow> assignWindowsFunction = new FlinkAssignWindows<>(windowFn); DataStream<WindowedValue<T>> resultDataStream = inputDataStream .flatMap(assignWindowsFunction) .name(transform.getUniqueName()) .returns(resultTypeInfo); context.addDataStream(outputCollectionId, resultDataStream); }
@Test public void getEnvironmentWindowIntoKnown() throws IOException { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); WindowIntoPayload payload = WindowIntoPayload.newBuilder() .setWindowFn( WindowingStrategyTranslation.toProto( FixedWindows.of(Duration.standardMinutes(5L)), components)) .build(); RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(components.toComponents()); PTransform builder = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload(payload.toByteString()) .build()) .build(); Environment env = Environments.getEnvironment(builder, rehydratedComponents).get(); assertThat( env, equalTo( components .toComponents() .getEnvironmentsOrThrow(payload.getWindowFn().getEnvironmentId()))); }
private <T> void translateAssignWindows( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); RunnerApi.WindowIntoPayload payload; try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); } //TODO: https://issues.apache.org/jira/browse/BEAM-4296 // This only works for well known window fns, we should defer this execution to the SDK // if the WindowFn can't be parsed or just defer it all the time. WindowFn<T, ? extends BoundedWindow> windowFn = (WindowFn<T, ? extends BoundedWindow>) WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn()); String inputCollectionId = Iterables.getOnlyElement(transform.getInputsMap().values()); String outputCollectionId = Iterables.getOnlyElement(transform.getOutputsMap().values()); Coder<WindowedValue<T>> outputCoder = instantiateCoder(outputCollectionId, components); TypeInformation<WindowedValue<T>> resultTypeInfo = new CoderTypeInformation<>(outputCoder); DataStream<WindowedValue<T>> inputDataStream = context.getDataStreamOrThrow(inputCollectionId); FlinkAssignWindows<T, ? extends BoundedWindow> assignWindowsFunction = new FlinkAssignWindows<>(windowFn); DataStream<WindowedValue<T>> resultDataStream = inputDataStream .flatMap(assignWindowsFunction) .name(transform.getUniqueName()) .returns(resultTypeInfo); context.addDataStream(outputCollectionId, resultDataStream); }