!DoFnSignatures.signatureForDoFn(doFn).processElement().isSplittable(), "Not expected to directly translate splittable DoFn, should have been overridden: %s", doFn);
@Test public void testBasicDoFnAllParameters() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process( @Element String element, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo paneInfo, OutputReceiver<String> receiver, PipelineOptions options) {} }.getClass()); assertThat(sig.processElement().extraParameters().size(), equalTo(6)); assertThat(sig.processElement().extraParameters().get(0), instanceOf(ElementParameter.class)); assertThat(sig.processElement().extraParameters().get(1), instanceOf(TimestampParameter.class)); assertThat(sig.processElement().extraParameters().get(2), instanceOf(WindowParameter.class)); assertThat(sig.processElement().extraParameters().get(3), instanceOf(PaneInfoParameter.class)); assertThat( sig.processElement().extraParameters().get(4), instanceOf(OutputReceiverParameter.class)); assertThat( sig.processElement().extraParameters().get(5), instanceOf(PipelineOptionsParameter.class)); }
@Test public void testFieldAccess() throws IllegalAccessException { FieldAccessDescriptor descriptor = FieldAccessDescriptor.withFieldNames("foo", "bar"); DoFn<String, String> doFn = new DoFn<String, String>() { @FieldAccess("foo") final FieldAccessDescriptor fieldAccess = descriptor; @ProcessElement public void process(@FieldAccess("foo") Row row) {} }; DoFnSignature sig = DoFnSignatures.getSignature(doFn.getClass()); assertThat(sig.fieldAccessDeclarations().get("foo"), notNullValue()); Field field = sig.fieldAccessDeclarations().get("foo").field(); assertThat(field.getName(), equalTo("fieldAccess")); assertThat(field.get(doFn), equalTo(descriptor)); assertThat(sig.processElement().getRowParameter(), notNullValue()); }
this.fn = fn; this.signature = DoFnSignatures.getSignature(fn.getClass()); this.observesWindow = signature.processElement().observesWindow() || !sideInputReader.isEmpty(); this.invoker = DoFnInvokers.invokerFor(fn); this.sideInputReader = sideInputReader; RowParameter rowParameter = processElementMethod.getRowParameter(); FieldAccessDescriptor fieldAccessDescriptor = null; if (rowParameter != null) {
if (processElement.isSplittable()) { verifySplittableMethods(signature, errors); } else {
!DoFnSignatures.signatureForDoFn(doFn).processElement().isSplittable(), "Not expected to directly translate splittable DoFn, should have been overridden: %s", doFn);
errors.forMethod(DoFn.ProcessElement.class, processElement.targetMethod()); TypeDescriptor<?> restrictionT = getInitialRestriction.restrictionT(); processElementErrors.checkArgument( processElement.trackerT().getRawType().equals(RestrictionTracker.class), "Has tracker type %s, but the DoFn's tracker type must be of type RestrictionTracker.", formatType(processElement.trackerT()));
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); final String restrictionCoderId; if (signature.processElement().isSplittable()) { final Coder<?> restrictionCoder = DoFnInvokers.invokerFor(doFn).invokeGetRestrictionCoder(pipeline.getCoderRegistry());
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); checkArgument( !signature.processElement().isSplittable(), String.format( "Not expecting a splittable %s: should have been overridden",
DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format(
if (processElement.isSplittable()) { if (isBounded == null) { isBounded = processElement.hasReturnValue() ? PCollection.IsBounded.UNBOUNDED : PCollection.IsBounded.BOUNDED; ? DoFn.BoundedPerElement.class.getSimpleName() : DoFn.UnboundedPerElement.class.getSimpleName())); checkState(!processElement.hasReturnValue(), "Should have been inferred splittable"); isBounded = PCollection.IsBounded.BOUNDED;
private DoFnTester(DoFn<InputT, OutputT> origFn) { this.origFn = origFn; DoFnSignature signature = DoFnSignatures.signatureForDoFn(origFn); for (DoFnSignature.Parameter param : signature.processElement().extraParameters()) { param.match( new DoFnSignature.Parameter.Cases.WithDefault<Void>() {
RowParameter rowParameter = processElementMethod.getRowParameter();
}.getClass()); assertThat(sig.processElement().extraParameters().size(), equalTo(2)); .extraParameters() .get(1) .match(
assertEquals(RestrictionTracker.class, signature.processElement().trackerT().getRawType()); assertTrue(signature.processElement().isSplittable()); assertTrue(signature.processElement().hasReturnValue()); assertEquals( SomeRestriction.class, signature.getInitialRestriction().restrictionT().getRawType());
SomeRestriction, RestrictionTracker<SomeRestriction, ?>, SomeRestrictionCoder>() {}.getClass()); assertEquals(RestrictionTracker.class, signature.processElement().trackerT().getRawType()); assertTrue(signature.processElement().isSplittable()); assertTrue(signature.processElement().hasReturnValue()); assertEquals( SomeRestriction.class, signature.getInitialRestriction().restrictionT().getRawType());
/** * Perform common validations of the {@link DoFn}, for example ensuring that state is used * correctly and that its features can be supported. */ private static <InputT, OutputT> void validate(DoFn<InputT, OutputT> fn) { DoFnSignature signature = DoFnSignatures.getSignature((Class) fn.getClass()); // State is semantically incompatible with splitting if (!signature.stateDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses state, but these are not compatible", fn.getClass().getName())); } // Timers are semantically incompatible with splitting if (!signature.timerDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses timers, but these are not compatible", fn.getClass().getName())); } }
@SuppressWarnings("unchecked") private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication( AppliedPTransform< PCollection<? extends InputT>, PCollectionTuple, PTransform<PCollection<? extends InputT>, PCollectionTuple>> application) throws IOException { DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application); DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); if (signature.processElement().isSplittable()) { return SplittableParDo.forAppliedParDo((AppliedPTransform) application); } else if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) { return new GbkThenStatefulParDo( fn, ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application), ParDoTranslation.getSideInputs(application)); } else { return application.getTransform(); } }
@Test public void testDeclAndUsageOfStateInSuperclass() throws Exception { class DoFnOverridingAbstractStateUse extends DoFnDeclaringStateAndAbstractUse { @Override public void processWithState(ProcessContext c, ValueState<String> state) {} } DoFnSignature sig = DoFnSignatures.getSignature(new DoFnOverridingAbstractStateUse().getClass()); assertThat(sig.stateDeclarations().size(), equalTo(1)); assertThat(sig.processElement().extraParameters().size(), equalTo(2)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get(DoFnOverridingAbstractStateUse.STATE_ID); StateParameter stateParam = (StateParameter) sig.processElement().extraParameters().get(1); assertThat( decl.field(), equalTo(DoFnDeclaringStateAndAbstractUse.class.getDeclaredField("myStateSpec"))); // The method we pull out is the superclass method; this is what allows validation to remain // simple. The later invokeDynamic instruction causes it to invoke the actual implementation. assertThat(stateParam.referent(), equalTo(decl)); }
@Test public void testHasRestrictionTracker() throws Exception { DoFnSignature.ProcessElementMethod signature = analyzeProcessElementMethod( new AnonymousMethod() { private void method( DoFn<Integer, String>.ProcessContext context, SomeRestrictionTracker tracker) {} }); assertTrue(signature.isSplittable()); assertTrue( signature .extraParameters() .stream() .anyMatch( Predicates.instanceOf(DoFnSignature.Parameter.RestrictionTrackerParameter.class) ::apply)); assertEquals(SomeRestrictionTracker.class, signature.trackerT().getRawType()); }