/** * Gets the type of the stream. * * @return The type of the datastream. */ public TypeInformation<T> getType() { return transformation.getOutputType(); }
/** * Returns the {@code TypeInformation} for the elements from the first input. */ public TypeInformation<IN2> getInputType2() { return input2.getOutputType(); }
/** * Returns the {@code TypeInformation} for the elements of the input. */ public TypeInformation<IN> getInputType() { return input.getOutputType(); }
/** * Returns the {@code TypeInformation} for the elements from the first input. */ public TypeInformation<IN1> getInputType1() { return input1.getOutputType(); }
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code StreamTransformation} * @param outputSelector The output selector */ public SplitTransformation(StreamTransformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
/** * Creates a new {@code PartitionTransformation} from the given input and * {@link StreamPartitioner}. * * @param input The input {@code StreamTransformation} * @param partitioner The {@code StreamPartitioner} */ public PartitionTransformation(StreamTransformation<T> input, StreamPartitioner<T> partitioner) { super("Partition", input.getOutputType(), input.getParallelism()); this.input = input; this.partitioner = partitioner; }
/** * Creates a new {@code SelectionTransformation} from the given input that only selects * the streams with the selected names. * * @param input The input {@code StreamTransformation} * @param selectedNames The names from the upstream {@code SplitTransformation} that this * {@code SelectTransformation} selects. */ public SelectTransformation(StreamTransformation<T> input, List<String> selectedNames) { super("Select", input.getOutputType(), input.getParallelism()); this.input = input; this.selectedNames = selectedNames; }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
/** * Creates a new {@code FeedbackTransformation} from the given input. * * @param input The input {@code StreamTransformation} * @param waitTime The wait time of the feedback operator. After the time expires * the operation will close and not receive any more feedback elements. */ public FeedbackTransformation(StreamTransformation<T> input, Long waitTime) { super("Feedback", input.getOutputType(), input.getParallelism()); this.input = input; this.waitTime = waitTime; this.feedbackEdges = Lists.newArrayList(); }
/** * Extracts a timestamp from an element and assigns it as the internal timestamp of that element. * The internal timestamps are, for example, used to to event-time window operations. * * <p>If you know that the timestamps are strictly increasing you can use an * {@link AscendingTimestampExtractor}. Otherwise, * you should provide a {@link TimestampExtractor} that also implements * {@link TimestampExtractor#getCurrentWatermark()} to keep track of watermarks. * * @param extractor The TimestampExtractor that is called for each element of the DataStream. * * @deprecated Please use {@link #assignTimestampsAndWatermarks(AssignerWithPeriodicWatermarks)} * of {@link #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks)} * instead. * @see #assignTimestampsAndWatermarks(AssignerWithPeriodicWatermarks) * @see #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks) */ @Deprecated public SingleOutputStreamOperator<T> assignTimestamps(TimestampExtractor<T> extractor) { // match parallelism to input, otherwise dop=1 sources could lead to some strange // behaviour: the watermark will creep along very slowly because the elements // from the source go to each extraction operator round robin. int inputParallelism = getTransformation().getParallelism(); ExtractTimestampsOperator<T> operator = new ExtractTimestampsOperator<>(clean(extractor)); return transform("ExtractTimestamps", getTransformation().getOutputType(), operator) .setParallelism(inputParallelism); }
/** * Method for passing user defined operators along with the type * information that will transform the DataStream. * * @param operatorName * name of the operator, for logging purposes * @param outTypeInfo * the output type of the operator * @param operator * the object containing the transformation logic * @param <R> * type of the return stream * @return the data stream constructed */ @PublicEvolving public <R> SingleOutputStreamOperator<R> transform(String operatorName, TypeInformation<R> outTypeInfo, OneInputStreamOperator<T, R> operator) { // read the output type of the input Transform to coax out errors about MissingTypeInfo transformation.getOutputType(); OneInputTransformation<T, R> resultTransform = new OneInputTransformation<>( this.transformation, operatorName, operator, outTypeInfo, environment.getParallelism()); @SuppressWarnings({ "unchecked", "rawtypes" }) SingleOutputStreamOperator<R> returnStream = new SingleOutputStreamOperator(environment, resultTransform); getExecutionEnvironment().addOperator(resultTransform); return returnStream; }
new TimestampsAndPunctuatedWatermarksOperator<>(cleanedAssigner); return transform("Timestamps/Watermarks", getTransformation().getOutputType(), operator) .setParallelism(inputParallelism);
new TimestampsAndPeriodicWatermarksOperator<>(cleanedAssigner); return transform("Timestamps/Watermarks", getTransformation().getOutputType(), operator) .setParallelism(inputParallelism);
/** * Adds the given sink to this DataStream. Only streams with sinks added * will be executed once the {@link StreamExecutionEnvironment#execute()} * method is called. * * @param sinkFunction * The object containing the sink's invoke function. * @return The closed DataStream. */ public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) { // read the output type of the input Transform to coax out errors about MissingTypeInfo transformation.getOutputType(); // configure the type if needed if (sinkFunction instanceof InputTypeConfigurable) { ((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig()); } StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction)); DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator); getExecutionEnvironment().addOperator(sink.getTransformation()); return sink; }
transform.getOutputType();
/** * Transforms a {@code SourceTransformation}. */ private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) { Collection<Integer> inputIds = transform(sink.getInput()); String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds); streamGraph.addSink(sink.getId(), slotSharingGroup, sink.getCoLocationGroupKey(), sink.getOperator(), sink.getInput().getOutputType(), null, "Sink: " + sink.getName()); streamGraph.setParallelism(sink.getId(), sink.getParallelism()); streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism()); for (Integer inputId: inputIds) { streamGraph.addEdge(inputId, sink.getId(), 0 ); } if (sink.getStateKeySelector() != null) { TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(env.getConfig()); streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer); } return Collections.emptyList(); }
/** * Gets the type of the stream. * * @return The type of the datastream. */ public TypeInformation<T> getType() { return transformation.getOutputType(); }
/** * Gets the type of the stream. * * @return The type of the datastream. */ public TypeInformation<T> getType() { return transformation.getOutputType(); }
/** * Returns the {@code TypeInformation} for the elements of the input. */ public TypeInformation<IN> getInputType() { return input.getOutputType(); }
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code StreamTransformation} * @param outputSelector The output selector */ public SplitTransformation(StreamTransformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }