private static RunnerApi.Coder toKnownCoder(Coder<?> coder, SdkComponents components) throws IOException { CoderTranslator translator = KNOWN_TRANSLATORS.get(coder.getClass()); List<String> componentIds = registerComponents(coder, translator, components); return RunnerApi.Coder.newBuilder() .addAllComponentCoderIds(componentIds) .setSpec( SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(KNOWN_CODER_URNS.get(coder.getClass())) .setPayload(ByteString.copyFrom(translator.getPayload(coder))))) .build(); }
private <T> RunnerApi.FunctionSpec translateTyped( final TestStream<T> testStream, SdkComponents components) throws IOException { return RunnerApi.FunctionSpec.newBuilder() .setUrn(TEST_STREAM_TRANSFORM_URN) .setPayload(payloadForTestStream(testStream, components).toByteString()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Flatten.PCollections<?>> transform, SdkComponents components) { return RunnerApi.FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build(); }
public static SideInput translateView(PCollectionView<?> view, SdkComponents components) { Builder builder = SideInput.newBuilder(); builder.setAccessPattern( FunctionSpec.newBuilder().setUrn(view.getViewFn().getMaterialization().getUrn()).build()); builder.setViewFn(translateViewFn(view.getViewFn(), components)); builder.setWindowMappingFn(translateWindowMappingFn(view.getWindowMappingFn(), components)); return builder.build(); }
@Test public void windowedValueCoderComponentsWrongUrn() { thrown.expect(IllegalArgumentException.class); ModelCoders.getWindowedValueCoderComponents( Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.LENGTH_PREFIX_CODER_URN))) .build()); }
private static SdkFunctionSpec toProto(BoundedSource<?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
public static SdkFunctionSpec translateViewFn(ViewFn<?, ?> viewFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_VIEW_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn))) .build()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) { WindowIntoPayload payload = toProto(transform.getTransform(), components); return RunnerApi.FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payload.toByteString()) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Impulse> application, SdkComponents components) throws IOException { return FunctionSpec.newBuilder().setUrn(getUrn(application.getTransform())).build(); } }
private static SdkFunctionSpec toProto(String urn, Serializable serializable) { return SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(urn) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable))) .build()) .build(); }
public static SdkFunctionSpec toProto( GlobalCombineFn<?, ?, ?> combineFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn))) .build()) .build(); } }
public static Coder kvCoder(String keyCoderId, String valueCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(KV_CODER_URN))) .addComponentCoderIds(keyCoderId) .addComponentCoderIds(valueCoderId) .build(); }
@Test public void testMatch() { Assert.assertTrue( NativeTransforms.isNative( RunnerApi.PTransform.newBuilder() .setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn("test").build()) .build())); }
@Test public void kvCoderComponentsWrongUrn() { thrown.expect(IllegalArgumentException.class); ModelCoders.getKvCoderComponents( Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.LENGTH_PREFIX_CODER_URN))) .build()); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, WriteFiles<?, ?, ?>> transform, SdkComponents components) throws IOException { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload(payloadForWriteFiles(transform.getTransform(), components).toByteString()) .build(); } }
public static Coder windowedValueCoder(String elementCoderId, String windowCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(WINDOWED_VALUE_CODER_URN))) .addComponentCoderIds(elementCoderId) .addComponentCoderIds(windowCoderId) .build(); }
public static SdkFunctionSpec translateWindowMappingFn( WindowMappingFn<?> windowMappingFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn))) .build()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(transform.getTransform().getView()))) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn("urn:beam:transform:window:v1") .setPayload( WindowIntoTranslation.toProto(transform.getTransform(), components).toByteString()) .build(); } }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, GroupByKey<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder().setUrn(getUrn(transform.getTransform())).build(); } }