/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeClass * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTypeClass) { return withFeedbackType(TypeInformation.of(feedbackTypeClass)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeHint * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeHint<F> feedbackTypeHint) { return withFeedbackType(TypeInformation.of(feedbackTypeHint)); }
@Test(expected = UnsupportedOperationException.class) public void testCoIterClosingFromOutOfLoop() throws Exception { // this test verifies that we cannot close an iteration with a DataStream that does not // have the iteration in its predecessors StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // introduce dummy mapper to get to correct parallelism DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); IterativeStream<Integer> iter1 = source.iterate(); ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType( Integer.class); coIter.closeWith(iter1.map(noOpIntMap)); }
@Test(expected = UnsupportedOperationException.class) public void testCoDifferingParallelism() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // introduce dummy mapper to get to correct parallelism DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType( Integer.class); coIter.closeWith(coIter.map(noOpIntCoMap).setParallelism(parallelism / 2)); }
@Test public void testImmutabilityWithCoiteration() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance IterativeStream<Integer> iter1 = source.iterate(); // Calling withFeedbackType should create a new iteration ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class); iter1.closeWith(iter1.map(noOpIntMap)).print(); iter2.closeWith(iter2.map(noOpCoMap)).print(); StreamGraph graph = env.getStreamGraph(); assertEquals(2, graph.getIterationSourceSinkPairs().size()); for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) { assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)), graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0))); } }
.map(noOpIntMap).name("ParallelizeMap") .iterate(2000 * timeoutScale) .withFeedbackType(Types.STRING);
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeHint * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeHint<F> feedbackTypeHint) { return withFeedbackType(TypeInformation.of(feedbackTypeHint)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeClass * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTypeClass) { return withFeedbackType(TypeExtractor.getForClass(feedbackTypeClass)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeString * String describing the type information of the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(String feedbackTypeString) { return withFeedbackType(TypeInfoParser.<F> parse(feedbackTypeString)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeClass * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTypeClass) { return withFeedbackType(TypeInformation.of(feedbackTypeClass)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeClass * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(Class<F> feedbackTypeClass) { return withFeedbackType(TypeInformation.of(feedbackTypeClass)); }
/** * Changes the feedback type of the iteration and allows the user to apply * co-transformations on the input and feedback stream, as in a * {@link ConnectedStreams}. * * <p>For type safety the user needs to define the feedback type * * @param feedbackTypeHint * Class of the elements in the feedback stream. * @return A {@link ConnectedIterativeStreams}. */ public <F> ConnectedIterativeStreams<T, F> withFeedbackType(TypeHint<F> feedbackTypeHint) { return withFeedbackType(TypeInformation.of(feedbackTypeHint)); }