public <OutputT extends POutput> Map<TupleTag<?>, PValue> getOutputs( PTransform<?, OutputT> transform) { return currentTransform.getOutputs(); } }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { return (application.getTransform() instanceof Flatten.PCollections) && application.getInputs().isEmpty(); }
@Override public AppliedPTransform<?, ?, ?> getProducer(PValue value) { return checkNotNull( producers.get(value), "Unknown producer for value %s while translating step %s", value, currentTransform.getFullName()); }
public Map<TupleTag<?>, PValue> getOutputs(PTransform<?, ?> transform) { checkArgument(currentTransform != null, "can only be called with non-null currentTransform"); checkArgument( currentTransform.getTransform() == transform, "can only be called with current transform"); return currentTransform.getOutputs(); }
@Override public PTransformReplacement<PCollectionList<T>, PCollection<T>> getReplacementTransform( AppliedPTransform<PCollectionList<T>, PCollection<T>, PCollections<T>> transform) { checkArgument( transform.getInputs().isEmpty(), "Unexpected nonempty input %s for %s", transform.getInputs(), getClass().getSimpleName()); return PTransformReplacement.of( PCollectionList.empty(transform.getPipeline()), new CreateEmptyFromList<T>()); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { return application.getTransform().getClass().equals(Combine.GroupedValues.class) && ((Combine.GroupedValues<?, ?, ?>) application.getTransform()) .getSideInputs() .isEmpty(); }
@Test public void toAndFromProto() throws IOException { SdkComponents components = SdkComponents.create(spec.getTransform().getPipeline().getOptions()); RunnerApi.PTransform converted = convert(spec, components); Components protoComponents = components.toComponents(); // Sanity checks assertThat(converted.getInputsCount(), equalTo(spec.getTransform().getInputs().size())); assertThat(converted.getOutputsCount(), equalTo(spec.getTransform().getOutputs().size())); assertThat(converted.getSubtransformsCount(), equalTo(spec.getChildren().size())); assertThat(converted.getUniqueName(), equalTo(spec.getTransform().getFullName())); for (PValue inputValue : spec.getTransform().getInputs().values()) { PCollection<?> inputPc = (PCollection<?>) inputValue; protoComponents.getPcollectionsOrThrow(components.registerPCollection(inputPc)); } for (PValue outputValue : spec.getTransform().getOutputs().values()) { PCollection<?> outputPc = (PCollection<?>) outputValue; protoComponents.getPcollectionsOrThrow(components.registerPCollection(outputPc)); } }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) { Create.Values<T> original = transform.getTransform(); PCollection<T> output = (PCollection) Iterables.getOnlyElement(transform.getOutputs().values()); return PTransformReplacement.of( transform.getPipeline().begin(), new StreamingFnApiCreate<>(original, output)); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform) { return PTransformReplacement.of( transform.getPipeline().begin(), new StreamingBoundedRead<>(transform.getTransform())); }
public <T extends PInput> Map<TupleTag<?>, PValue> getInputs(PTransform<T, ?> transform) { return currentTransform.getInputs(); }
private AppliedPTransform<?, ?, ?> appliedWrite(WriteFiles<Integer, Void, Integer> write) { return AppliedPTransform.of( "WriteFiles", Collections.emptyMap(), Collections.emptyMap(), write, p); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) { try { return PTransformReplacement.of( transform.getPipeline().begin(), new DirectTestStream<>(runner, TestStreamTranslation.getTestStream(transform))); } catch (IOException exc) { throw new RuntimeException( String.format( "Transform could not be converted to %s", TestStream.class.getSimpleName()), exc); } }
private AppliedPTransform<?, ?, ?> getCurrentTransform(PTransform<?, ?> transform) { checkArgument( currentTransform != null && currentTransform.getTransform() == transform, "can only be called with current transform"); return currentTransform; }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Values<T>> transform) { return PTransformReplacement.of( transform.getPipeline().begin(), new PrimitiveCreate<>( transform.getTransform(), ((PCollection<T>) Iterables.getOnlyElement(transform.getOutputs().values())) .getCoder())); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, Read.Unbounded<T>> transform) { return PTransformReplacement.of( transform.getPipeline().begin(), new StreamingUnboundedRead<>(transform.getTransform())); }
Map<TupleTag<?>, PValue> getInputs(PTransform<?, ?> transform) { return currentTransform.getInputs(); }
@Override public PTransformReplacement<PCollectionList<T>, PCollection<T>> getReplacementTransform( AppliedPTransform<PCollectionList<T>, PCollection<T>, PCollections<T>> transform) { return PTransformReplacement.of( getInput(transform.getInputs(), transform.getPipeline()), new FlattenWithoutDuplicateInputs<>()); }
/** Returns the {@link AppliedPTransform} representing this {@link Node}. */ public AppliedPTransform<?, ?, ?> toAppliedPTransform(Pipeline pipeline) { return AppliedPTransform.of( getFullName(), inputs, outputs, (PTransform) getTransform(), pipeline); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) { PBegin input = PBegin.in(transform.getPipeline()); BoundedSource<T> source; try { source = ReadTranslation.boundedSourceFromTransform(transform); } catch (IOException e) { throw new RuntimeException(e); } return PTransformReplacement.of(input, bounded(source)); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { if (application.getTransform() instanceof Flatten.PCollections) { Set<PValue> observed = new HashSet<>(); for (PValue pvalue : application.getInputs().values()) { boolean firstInstance = observed.add(pvalue); if (!firstInstance) { return true; } } } return false; }