private static RunnerApi.Coder toKnownCoder(Coder<?> coder, SdkComponents components) throws IOException { CoderTranslator translator = KNOWN_TRANSLATORS.get(coder.getClass()); List<String> componentIds = registerComponents(coder, translator, components); return RunnerApi.Coder.newBuilder() .addAllComponentCoderIds(componentIds) .setSpec( SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(KNOWN_CODER_URNS.get(coder.getClass())) .setPayload(ByteString.copyFrom(translator.getPayload(coder))))) .build(); }
ByteString serializedFn = ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowFn)); if (windowFn instanceof GlobalWindows) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec(FunctionSpec.newBuilder().setUrn(GLOBAL_WINDOWS_URN)) .setOffset(Timestamps.fromMillis(((FixedWindows) windowFn).getOffset().getMillis())) .build(); return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( .setPeriod(Durations.fromMillis(((SlidingWindows) windowFn).getPeriod().getMillis())) .build(); return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( .setGapSize(Durations.fromMillis(((Sessions) windowFn).getGapDuration().getMillis())) .build(); return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( .build(); } else { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec(
extractAccumulatorCoder(combineFn, (AppliedPTransform) combine))) .setCombineFn( SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec(
.setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putTimerSpecs("timer", TimerSpec.getDefaultInstance()) .build()
ParDoPayload.newBuilder() .setDoFn( SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString())) WindowIntoPayload.newBuilder() .setWindowFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString()))
.setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString()))
.setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString()))
.setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putStateSpecs("state", StateSpec.getDefaultInstance()) .putTimerSpecs("timer", TimerSpec.getDefaultInstance())
public static SdkFunctionSpec translateDoFn( DoFn<?, ?> fn, TupleTag<?> tag, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_DO_FN_URN) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(DoFnAndMainOutput.of(fn, tag)))) .build()) .build(); }
private static RunnerApi.Coder toCustomCoder(Coder<?> coder) throws IOException { RunnerApi.Coder.Builder coderBuilder = RunnerApi.Coder.newBuilder(); return coderBuilder .setSpec( SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_CODER_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(coder))) .build())) .build(); }
private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_UNBOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
@Test public void windowedValueCoderComponentsWrongUrn() { thrown.expect(IllegalArgumentException.class); ModelCoders.getWindowedValueCoderComponents( Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.LENGTH_PREFIX_CODER_URN))) .build()); }
private static SdkFunctionSpec toProto(BoundedSource<?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
public static SdkFunctionSpec toProto( GlobalCombineFn<?, ?, ?> combineFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn))) .build()) .build(); } }
public static Coder kvCoder(String keyCoderId, String valueCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(KV_CODER_URN))) .addComponentCoderIds(keyCoderId) .addComponentCoderIds(valueCoderId) .build(); }
@Test public void kvCoderComponentsWrongUrn() { thrown.expect(IllegalArgumentException.class); ModelCoders.getKvCoderComponents( Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.LENGTH_PREFIX_CODER_URN))) .build()); }
public static SdkFunctionSpec translateViewFn(ViewFn<?, ?> viewFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_VIEW_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn))) .build()) .build(); }
private static SdkFunctionSpec toProto(String urn, Serializable serializable) { return SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(urn) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable))) .build()) .build(); }
public static Coder windowedValueCoder(String elementCoderId, String windowCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(WINDOWED_VALUE_CODER_URN))) .addComponentCoderIds(elementCoderId) .addComponentCoderIds(windowCoderId) .build(); }
public static SdkFunctionSpec translateWindowMappingFn( WindowMappingFn<?> windowMappingFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn))) .build()) .build(); }