ParDoPayload payload = ParDoPayload.parseFrom(parDo.getTransform().getSpec().getPayload()); if (Maps.filterKeys( parDo.getTransform().getInputsMap(), s -> payload.getTimerSpecsMap().containsKey(s)) .values() .contains(candidate.getId())) { } else if (payload.getStateSpecsCount() > 0 || payload.getTimerSpecsCount() > 0) {
private static void validateParDo(String id, PTransform transform, Components components) throws Exception { ParDoPayload payload = ParDoPayload.parseFrom(transform.getSpec().getPayload()); // side_inputs for (String sideInputId : payload.getSideInputsMap().keySet()) { checkArgument( transform.containsInputs(sideInputId), "Transform %s side input %s is not listed in the transform's inputs", id, sideInputId); } // TODO: Validate state_specs and timer_specs if (!payload.getRestrictionCoderId().isEmpty()) { checkArgument(components.containsCoders(payload.getRestrictionCoderId())); } }
@Test public void testToProto() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); ParDoPayload payload = ParDoTranslation.translateParDo(parDo, p, components); assertThat(ParDoTranslation.getDoFn(payload), equalTo(parDo.getFn())); assertThat(ParDoTranslation.getMainOutputTag(payload), equalTo(parDo.getMainOutputTag())); for (PCollectionView<?> view : parDo.getSideInputs()) { payload.getSideInputsOrThrow(view.getTagInternal().getId()); } }
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("sideRead.out", pc("sideRead.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString()) .build()) .build()) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload(
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString())) .build()) .putPcollections("goRead.out", pc("goRead.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("pyParDo.out", pc("pyParDo.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload(
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString()) .build()) .build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString()) .build()) .build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString()) .build()) .build();
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString())) .build()) .putPcollections("goRead.out", pc("goRead.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString())) .build()) .putPcollections("goParDo.out", pc("goParDo.out"))
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); PTransform otherEnvRead = .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("rare")) .build() .toByteString())) .build(); PTransform flattenTransform =
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString())) .build()) .putPcollections("go.out", pc("go.out"))
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putSideInputs("side_input", SideInput.getDefaultInstance()) .build() .toByteString())) .build(); PCollection sideInputPCollection =
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("parDo.out", pc("parDo.out"))
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("foo")) .putSideInputs("side_input", SideInput.getDefaultInstance()) .putTimerSpecs("timer", TimerSpec.getDefaultInstance()) .build() .toByteString())) .build(); PCollection input = PCollection.newBuilder().setUniqueName("input.out").build();
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn( SdkFunctionSpec.newBuilder().setEnvironmentId("py").build()) .build() .toByteString())) .build()) .putPcollections("parDo.out", pc("parDo.out"))
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); PTransform processMain = .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putSideInputs("side", SideInput.getDefaultInstance()) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); PTransform windowTransform =
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("read.out", pc(name)) .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString())) .build()) .putPcollections("parDo.out", pc("parDo.out"))
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn( SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putSideInputs("side_input", SideInput.getDefaultInstance()) .build() .toByteString())) .build()) .putPcollections(
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); PTransform statefulTransform = .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putStateSpecs("state", StateSpec.getDefaultInstance()) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build();
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .build() .toByteString())) .build(); PTransform statefulTransform = .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .putStateSpecs("state", StateSpec.getDefaultInstance()) .build() .toByteString())) .build();