public static WindowIntoPayload getWindowIntoPayload(AppliedPTransform<?, ?, ?> application) { RunnerApi.PTransform transformProto; try { SdkComponents components = SdkComponents.create(application.getPipeline().getOptions()); transformProto = PTransformTranslation.toProto(application, Collections.emptyList(), components); } catch (IOException exc) { throw new RuntimeException(exc); } checkArgument( PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN.equals( transformProto.getSpec().getUrn()), "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"", Window.Assign.class.getSimpleName(), application.getTransform(), application.getFullName(), transformProto.getSpec().getUrn()); try { return WindowIntoPayload.parseFrom(transformProto.getSpec().getPayload()); } catch (InvalidProtocolBufferException exc) { throw new IllegalStateException( String.format( "%s translated %s with URN '%s' but payload was not a %s", PTransformTranslation.class.getSimpleName(), application, PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN, WindowIntoPayload.class.getSimpleName()), exc); } }
private <T> void translateAssignWindows( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); RunnerApi.WindowIntoPayload payload; try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); } //TODO: https://issues.apache.org/jira/browse/BEAM-4296 // This only works for well known window fns, we should defer this execution to the SDK // if the WindowFn can't be parsed or just defer it all the time. WindowFn<T, ? extends BoundedWindow> windowFn = (WindowFn<T, ? extends BoundedWindow>) WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn()); String inputCollectionId = Iterables.getOnlyElement(transform.getInputsMap().values()); String outputCollectionId = Iterables.getOnlyElement(transform.getOutputsMap().values()); Coder<WindowedValue<T>> outputCoder = instantiateCoder(outputCollectionId, components); TypeInformation<WindowedValue<T>> resultTypeInfo = new CoderTypeInformation<>(outputCoder); DataStream<WindowedValue<T>> inputDataStream = context.getDataStreamOrThrow(inputCollectionId); FlinkAssignWindows<T, ? extends BoundedWindow> assignWindowsFunction = new FlinkAssignWindows<>(windowFn); DataStream<WindowedValue<T>> resultDataStream = inputDataStream .flatMap(assignWindowsFunction) .name(transform.getUniqueName()) .returns(resultTypeInfo); context.addDataStream(outputCollectionId, resultDataStream); }
private <T> void translateAssignWindows( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); RunnerApi.WindowIntoPayload payload; try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e); } //TODO: https://issues.apache.org/jira/browse/BEAM-4296 // This only works for well known window fns, we should defer this execution to the SDK // if the WindowFn can't be parsed or just defer it all the time. WindowFn<T, ? extends BoundedWindow> windowFn = (WindowFn<T, ? extends BoundedWindow>) WindowingStrategyTranslation.windowFnFromProto(payload.getWindowFn()); String inputCollectionId = Iterables.getOnlyElement(transform.getInputsMap().values()); String outputCollectionId = Iterables.getOnlyElement(transform.getOutputsMap().values()); Coder<WindowedValue<T>> outputCoder = instantiateCoder(outputCollectionId, components); TypeInformation<WindowedValue<T>> resultTypeInfo = new CoderTypeInformation<>(outputCoder); DataStream<WindowedValue<T>> inputDataStream = context.getDataStreamOrThrow(inputCollectionId); FlinkAssignWindows<T, ? extends BoundedWindow> assignWindowsFunction = new FlinkAssignWindows<>(windowFn); DataStream<WindowedValue<T>> resultDataStream = inputDataStream .flatMap(assignWindowsFunction) .name(transform.getUniqueName()) .returns(resultTypeInfo); context.addDataStream(outputCollectionId, resultDataStream); }
try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getTransform().getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e);
try { payload = RunnerApi.WindowIntoPayload.parseFrom(transform.getTransform().getSpec().getPayload()); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException(e);
private static void validateAssignWindows(String id, PTransform transform, Components components) throws Exception { WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); }
private static String windowExtractor(PTransform transform) throws InvalidProtocolBufferException { return WindowIntoPayload.parseFrom(transform.getSpec().getPayload()) .getWindowFn() .getEnvironmentId(); }