private static void rejectSplittable(DoFn<?, ?> doFn) { DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s does not currently support splittable DoFn: %s", FlinkRunner.class.getSimpleName(), doFn)); } }
/** * Perform common validations of the {@link DoFn}, for example ensuring that state is used * correctly and that its features can be supported. */ private static <InputT, OutputT> void validate(DoFn<InputT, OutputT> fn) { DoFnSignature signature = DoFnSignatures.getSignature((Class) fn.getClass()); // State is semantically incompatible with splitting if (!signature.stateDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses state, but these are not compatible", fn.getClass().getName())); } // Timers are semantically incompatible with splitting if (!signature.timerDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses timers, but these are not compatible", fn.getClass().getName())); } }
DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format(
!DoFnSignatures.signatureForDoFn(doFn).processElement().isSplittable(), "Not expected to directly translate splittable DoFn, should have been overridden: %s", doFn);
!DoFnSignatures.signatureForDoFn(doFn).processElement().isSplittable(), "Not expected to directly translate splittable DoFn, should have been overridden: %s", doFn);
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); checkArgument( !signature.processElement().isSplittable(), String.format( "Not expecting a splittable %s: should have been overridden",
final DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); final String restrictionCoderId; if (signature.processElement().isSplittable()) { final Coder<?> restrictionCoder = DoFnInvokers.invokerFor(doFn).invokeGetRestrictionCoder(pipeline.getCoderRegistry());
if (processElement.isSplittable()) { if (isBounded == null) { isBounded =
assertTrue(signature.processElement().isSplittable()); assertTrue(signature.processElement().hasReturnValue()); assertEquals(
SomeRestrictionCoder>() {}.getClass()); assertEquals(RestrictionTracker.class, signature.processElement().trackerT().getRawType()); assertTrue(signature.processElement().isSplittable()); assertTrue(signature.processElement().hasReturnValue()); assertEquals(
@SuppressWarnings("unchecked") private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication( AppliedPTransform< PCollection<? extends InputT>, PCollectionTuple, PTransform<PCollection<? extends InputT>, PCollectionTuple>> application) throws IOException { DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application); DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); if (signature.processElement().isSplittable()) { return SplittableParDo.forAppliedParDo((AppliedPTransform) application); } else if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) { return new GbkThenStatefulParDo( fn, ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application), ParDoTranslation.getSideInputs(application)); } else { return application.getTransform(); } }
@Test public void testHasRestrictionTracker() throws Exception { DoFnSignature.ProcessElementMethod signature = analyzeProcessElementMethod( new AnonymousMethod() { private void method( DoFn<Integer, String>.ProcessContext context, SomeRestrictionTracker tracker) {} }); assertTrue(signature.isSplittable()); assertTrue( signature .extraParameters() .stream() .anyMatch( Predicates.instanceOf(DoFnSignature.Parameter.RestrictionTrackerParameter.class) ::apply)); assertEquals(SomeRestrictionTracker.class, signature.trackerT().getRawType()); }
private SplittableParDo( DoFn<InputT, OutputT> doFn, List<PCollectionView<?>> sideInputs, TupleTag<OutputT> mainOutputTag, TupleTagList additionalOutputTags, Map<TupleTag<?>, Coder<?>> outputTagsToCoders) { checkArgument( DoFnSignatures.getSignature(doFn.getClass()).processElement().isSplittable(), "fn must be a splittable DoFn"); this.doFn = doFn; this.sideInputs = sideInputs; this.mainOutputTag = mainOutputTag; this.additionalOutputTags = additionalOutputTags; this.outputTagsToCoders = outputTagsToCoders; }
@Override public boolean isSplittable() { return signature.processElement().isSplittable(); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof SplittableParDo.ProcessKeyedElements) { DoFn<?, ?> fn = ((SplittableParDo.ProcessKeyedElements) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable() && signature.isBoundedPerElement() == IsBounded.UNBOUNDED; } return false; }
private static Implementation getRestrictionCoderDelegation( TypeDescription doFnType, DoFnSignature signature) { if (signature.processElement().isSplittable()) { if (signature.getRestrictionCoder() == null) { return MethodDelegation.to( new DefaultRestrictionCoder(signature.getInitialRestriction().restrictionT())); } else { return new DowncastingParametersMethodDelegation( doFnType, signature.getRestrictionCoder().targetMethod()); } } else { return ExceptionMethod.throwing(UnsupportedOperationException.class); } }
private static boolean isSplittable(DoFn<?, ?> fn) { return DoFnSignatures.signatureForDoFn(fn).processElement().isSplittable(); } }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof SplittableParDo.ProcessKeyedElements) { DoFn<?, ?> fn = ((SplittableParDo.ProcessKeyedElements) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable() && signature.isBoundedPerElement() == IsBounded.BOUNDED; } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; }