public static Coder windowedValueCoder(String elementCoderId, String windowCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(WINDOWED_VALUE_CODER_URN))) .addComponentCoderIds(elementCoderId) .addComponentCoderIds(windowCoderId) .build(); }
private RunnerApi.Trigger convertSpecific(ReshuffleTrigger v) { return RunnerApi.Trigger.newBuilder() .setAlways(RunnerApi.Trigger.Always.getDefaultInstance()) .build(); }
private static String windowExtractor(PTransform transform) throws InvalidProtocolBufferException { return WindowIntoPayload.parseFrom(transform.getSpec().getPayload()) .getWindowFn() .getEnvironmentId(); }
private void removeDescendants(RunnerApi.Pipeline.Builder pipeline, String parentId) { RunnerApi.PTransform parentProto = pipeline.getComponents().getTransformsOrDefault(parentId, null); if (parentProto != null) { for (String childId : parentProto.getSubtransformsList()) { removeDescendants(pipeline, childId); pipeline.getComponentsBuilder().removeTransforms(childId); } pipeline .getComponentsBuilder() .putTransforms(parentId, parentProto.toBuilder().clearSubtransforms().build()); } }
private static String parDoExtractor(PTransform pTransform) throws InvalidProtocolBufferException { return ParDoPayload.parseFrom(pTransform.getSpec().getPayload()).getDoFn().getEnvironmentId(); }
private static String combineExtractor(PTransform pTransform) throws InvalidProtocolBufferException { return CombinePayload.parseFrom(pTransform.getSpec().getPayload()) .getCombineFn() .getEnvironmentId(); }
private static void validateCombine(String id, PTransform transform, Components components) throws Exception { CombinePayload payload = CombinePayload.parseFrom(transform.getSpec().getPayload()); checkArgument( components.containsCoders(payload.getAccumulatorCoderId()), "Transform %s uses unknown accumulator coder id %s", payload.getAccumulatorCoderId()); }
private static void validateAssignWindows(String id, PTransform transform, Components components) throws Exception { WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); }
private static PCollection pc(String name) { return PCollection.newBuilder() .setUniqueName(name) .setCoderId("coder") .setWindowingStrategyId("ws") .build(); }
private static void validateTestStream(String id, PTransform transform, Components components) throws Exception { TestStreamPayload.parseFrom(transform.getSpec().getPayload()); }
public RawWriteFiles( RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents) throws IOException { this.rehydratedComponents = rehydratedComponents; this.protoTransform = protoTransform; this.spec = protoTransform.getSpec(); this.payload = RunnerApi.WriteFilesPayload.parseFrom(spec.getPayload()); }
private ExecutableGraphBuilder addPCollection(String name) { components.putPcollections(name, PCollection.newBuilder().setUniqueName(name).build()); return this; }
private RunnerApi.Trigger convertSpecific(AfterPane v) { return RunnerApi.Trigger.newBuilder() .setElementCount( RunnerApi.Trigger.ElementCount.newBuilder().setElementCount(v.getElementCount())) .build(); }
@Override public MessageWithComponents getReplacement( String transformId, ComponentsOrBuilder existingComponents) { return MessageWithComponents.newBuilder() .setPtransform(extraTransform) .setComponents(extraComponents) .build(); } }
private RunnerApi.Trigger convertSpecific(DefaultTrigger v) { return RunnerApi.Trigger.newBuilder() .setDefault(RunnerApi.Trigger.Default.getDefaultInstance()) .build(); }
private RunnerApi.Trigger convertSpecific(NeverTrigger v) { return RunnerApi.Trigger.newBuilder() .setNever(RunnerApi.Trigger.Never.getDefaultInstance()) .build(); }
private RunnerApi.Trigger convertSpecific(FromEndOfWindow v) { return RunnerApi.Trigger.newBuilder() .setAfterEndOfWindow(RunnerApi.Trigger.AfterEndOfWindow.newBuilder()) .build(); }
private RunnerApi.Trigger convertSpecific(AfterSynchronizedProcessingTime v) { return RunnerApi.Trigger.newBuilder() .setAfterSynchronizedProcessingTime( RunnerApi.Trigger.AfterSynchronizedProcessingTime.getDefaultInstance()) .build(); }
public static Coder kvCoder(String keyCoderId, String valueCoderId) { return Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(KV_CODER_URN))) .addComponentCoderIds(keyCoderId) .addComponentCoderIds(valueCoderId) .build(); }
@Test public void kvCoderComponentsWrongUrn() { thrown.expect(IllegalArgumentException.class); ModelCoders.getKvCoderComponents( Coder.newBuilder() .setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.LENGTH_PREFIX_CODER_URN))) .build()); }