@Override public FunctionSpec translate( AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) { ReadPayload payload = toProto(transform.getTransform(), components); return RunnerApi.FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payload.toByteString()) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) { ReadPayload payload = toProto(transform.getTransform(), components); return RunnerApi.FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payload.toByteString()) .build(); } }
private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_UNBOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
@Override public FunctionSpec migrate(SdkComponents components) throws IOException { return FunctionSpec.newBuilder() .setUrn(WRITE_FILES_TRANSFORM_URN) .setPayload(payloadForWriteFilesLike(this, components).toByteString()) .build(); }
@Override public RunnerApi.FunctionSpec translate( AppliedPTransform<?, ?, ParDoSingle<?, ?>> transform, SdkComponents components) throws IOException { RunnerApi.ParDoPayload payload = payloadForParDoSingle(transform.getTransform(), components); return RunnerApi.FunctionSpec.newBuilder() .setUrn(PAR_DO_TRANSFORM_URN) .setPayload(payload.toByteString()) .build(); }
private <T> RunnerApi.FunctionSpec translateTyped( final TestStream<T> testStream, SdkComponents components) throws IOException { return RunnerApi.FunctionSpec.newBuilder() .setUrn(TEST_STREAM_TRANSFORM_URN) .setPayload(payloadForTestStream(testStream, components).toByteString()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Flatten.PCollections<?>> transform, SdkComponents components) { return RunnerApi.FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build(); }
public static SideInput translateView(PCollectionView<?> view, SdkComponents components) { Builder builder = SideInput.newBuilder(); builder.setAccessPattern( FunctionSpec.newBuilder().setUrn(view.getViewFn().getMaterialization().getUrn()).build()); builder.setViewFn(translateViewFn(view.getViewFn(), components)); builder.setWindowMappingFn(translateWindowMappingFn(view.getWindowMappingFn(), components)); return builder.build(); }
private static SdkFunctionSpec toProto(BoundedSource<?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
public static SdkFunctionSpec translateViewFn(ViewFn<?, ?> viewFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_VIEW_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn))) .build()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) { WindowIntoPayload payload = toProto(transform.getTransform(), components); return RunnerApi.FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payload.toByteString()) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Impulse> application, SdkComponents components) throws IOException { return FunctionSpec.newBuilder().setUrn(getUrn(application.getTransform())).build(); } }
private static SdkFunctionSpec toProto(String urn, Serializable serializable) { return SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(urn) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable))) .build()) .build(); }
public static SdkFunctionSpec toProto( GlobalCombineFn<?, ?, ?> combineFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn))) .build()) .build(); } }
@Test public void testMatch() { Assert.assertTrue( NativeTransforms.isNative( RunnerApi.PTransform.newBuilder() .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn("test").build()) .build())); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, WriteFiles<?, ?, ?>> transform, SdkComponents components) throws IOException { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payloadForWriteFiles(transform.getTransform(), components).toByteString()) .build(); } }
public static SdkFunctionSpec translateWindowMappingFn( WindowMappingFn<?> windowMappingFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn))) .build()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(transform.getTransform().getView()))) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn("urn:beam:transform:window:v1") .setPayload( WindowIntoTranslation.toProto(transform.getTransform(), components).toByteString()) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, GroupByKey<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build(); } }