.putCoders(kwiCoderId, kwiCoder) .putPcollections(kwiCollectionId, kwi) .putTransforms(gbkoId, gbko);
.putCoders(kwiCoderId, kwiCoder) .putPcollections(kwiCollectionId, kwiCollection) .putTransforms(rawGbkId, rawGbk);
"intermediatePc", PCollection.newBuilder().setUniqueName("intermediate").build()) .putCoders( "coder", Coder.newBuilder().setSpec(SdkFunctionSpec.getDefaultInstance()).build()))
"intermediatePc", PCollection.newBuilder().setUniqueName("intermediate").build()) .putCoders( "coder", Coder.newBuilder().setSpec(SdkFunctionSpec.getDefaultInstance()).build()))
/** This method doesn't do any pruning for reachability, but this may not require a test. */ @Test public void retainOnlyPrimitivesIgnoresUnreachableNodes() { Pipeline p = Pipeline.create(); p.apply( new org.apache.beam.sdk.transforms.PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return input .apply(GenerateSequence.from(2L)) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(5L)))) .apply(MapElements.into(TypeDescriptors.longs()).via(l -> l + 1)); } }); Components augmentedComponents = PipelineTranslation.toProto(p) .getComponents() .toBuilder() .putCoders("extra-coder", RunnerApi.Coder.getDefaultInstance()) .putWindowingStrategies( "extra-windowing-strategy", RunnerApi.WindowingStrategy.getDefaultInstance()) .putEnvironments("extra-env", RunnerApi.Environment.getDefaultInstance()) .putPcollections("extra-pc", RunnerApi.PCollection.getDefaultInstance()) .build(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(augmentedComponents); } }
/** * Registers the provided {@link Coder} into this {@link SdkComponents}, returning a unique ID for * the {@link Coder}. Multiple registrations of the same {@link Coder} will return the same unique * ID. * * <p>Coders are stored by identity to ensure that coders with implementations of {@link * #equals(Object)} and {@link #hashCode()} but incompatible binary formats are not considered the * same coder. */ public String registerCoder(Coder<?> coder) throws IOException { String existing = coderIds.get(Equivalence.identity().wrap(coder)); if (existing != null) { return existing; } String baseName = NameUtils.approximateSimpleName(coder); String name = uniqify(baseName, coderIds.values()); coderIds.put(Equivalence.identity().wrap(coder), name); RunnerApi.Coder coderProto = CoderTranslation.toProto(coder, this); componentsBuilder.putCoders(name, coderProto); return name; }
@Before public void setup() { partialComponents = Components.newBuilder() .putTransforms( "impulse", PTransform.newBuilder() .setUniqueName("Impulse") .putOutputs("output", "impulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("impulse.out", pc("impulse.out")) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .build(); }