private DataStream<OUT> selectOutput(String[] outputNames) { for (String outName : outputNames) { if (outName == null) { throw new RuntimeException("Selected names must not be null"); } } SelectTransformation<OUT> selectTransform = new SelectTransformation<OUT>(this.getTransformation(), Lists.newArrayList(outputNames)); return new DataStream<OUT>(this.getExecutionEnvironment(), selectTransform); }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { if (wasSplitApplied) { throw new UnsupportedOperationException("getSideOutput() and split() may not be called on the same DataStream. " + "As a work-around, please add a no-op map function before the split() call."); } sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
public ConnectedIterativeStreams(DataStream<I> input, TypeInformation<F> feedbackType, long waitTime) { super(input.getExecutionEnvironment(), input, new DataStream<>(input.getExecutionEnvironment(), new CoFeedbackTransformation<>(input.getParallelism(), feedbackType, waitTime))); this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation(); }
producerProperties.setProperty("retries", "3"); StreamTransformation<String> mockTransform = new MockStreamTransformation(); DataStream<String> stream = new DataStream<>(new DummyStreamExecutionEnvironment(), mockTransform);
private DataStream<OUT> selectOutput(String[] outputNames) { for (String outName : outputNames) { if (outName == null) { throw new RuntimeException("Selected names must not be null"); } } SelectTransformation<OUT> selectTransform = new SelectTransformation<OUT>(this.getTransformation(), Lists.newArrayList(outputNames)); return new DataStream<OUT>(this.getExecutionEnvironment(), selectTransform); }
private DataStream<OUT> selectOutput(String[] outputNames) { for (String outName : outputNames) { if (outName == null) { throw new RuntimeException("Selected names must not be null"); } } SelectTransformation<OUT> selectTransform = new SelectTransformation<OUT>(this.getTransformation(), Lists.newArrayList(outputNames)); return new DataStream<OUT>(this.getExecutionEnvironment(), selectTransform); }
private DataStream<OUT> selectOutput(String[] outputNames) { for (String outName : outputNames) { if (outName == null) { throw new RuntimeException("Selected names must not be null"); } } SelectTransformation<OUT> selectTransform = new SelectTransformation<OUT>(this.getTransformation(), Lists.newArrayList(outputNames)); return new DataStream<OUT>(this.getExecutionEnvironment(), selectTransform); }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Creates a new {@link DataStream} by merging {@link DataStream} outputs of * the same type with each other. The DataStreams merged using this operator * will be transformed simultaneously. * * @param streams * The DataStreams to union output with. * @return The {@link DataStream}. */ @SafeVarargs public final DataStream<T> union(DataStream<T>... streams) { List<StreamTransformation<T>> unionedTransforms = new ArrayList<>(); unionedTransforms.add(this.transformation); for (DataStream<T> newStream : streams) { if (!getType().equals(newStream.getType())) { throw new IllegalArgumentException("Cannot union streams of different types: " + getType() + " and " + newStream.getType()); } unionedTransforms.add(newStream.getTransformation()); } return new DataStream<>(this.environment, new UnionTransformation<>(unionedTransforms)); }
/** * Gets the {@link DataStream} that contains the elements that are emitted from an operation * into the side output with the given {@link OutputTag}. * * @see org.apache.flink.streaming.api.functions.ProcessFunction.Context#output(OutputTag, Object) */ public <X> DataStream<X> getSideOutput(OutputTag<X> sideOutputTag) { sideOutputTag = clean(requireNonNull(sideOutputTag)); // make a defensive copy sideOutputTag = new OutputTag<X>(sideOutputTag.getId(), sideOutputTag.getTypeInfo()); TypeInformation<?> type = requestedSideOutputs.get(sideOutputTag); if (type != null && !type.equals(sideOutputTag.getTypeInfo())) { throw new UnsupportedOperationException("A side output with a matching id was " + "already requested with a different type. This is not allowed, side output " + "ids need to be unique."); } requestedSideOutputs.put(sideOutputTag, sideOutputTag.getTypeInfo()); SideOutputTransformation<X> sideOutputTransformation = new SideOutputTransformation<>(this.getTransformation(), sideOutputTag); return new DataStream<>(this.getExecutionEnvironment(), sideOutputTransformation); } }
public ConnectedIterativeStreams(DataStream<I> input, TypeInformation<F> feedbackType, long waitTime) { super(input.getExecutionEnvironment(), input, new DataStream<>(input.getExecutionEnvironment(), new CoFeedbackTransformation<>(input.getParallelism(), feedbackType, waitTime))); this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation(); }
public ConnectedIterativeStreams(DataStream<I> input, TypeInformation<F> feedbackType, long waitTime) { super(input.getExecutionEnvironment(), input, new DataStream<>(input.getExecutionEnvironment(), new CoFeedbackTransformation<>(input.getParallelism(), feedbackType, waitTime))); this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation(); }
public ConnectedIterativeStreams(DataStream<I> input, TypeInformation<F> feedbackType, long waitTime) { super(input.getExecutionEnvironment(), input, new DataStream<>(input.getExecutionEnvironment(), new CoFeedbackTransformation<>(input.getParallelism(), feedbackType, waitTime))); this.coFeedbackTransformation = (CoFeedbackTransformation<F>) getSecondInput().getTransformation(); }
/** * Siddhi Continuous Query Language (CQL) * * @return ExecutionSiddhiStream context */ public ExecutionSiddhiStream cql(DataStream<ControlEvent> controlStream) { DataStream<Tuple2<StreamRoute, Object>> unionStream = controlStream .map(new NamedControlStream(ControlEvent.DEFAULT_INTERNAL_CONTROL_STREAM)) .broadcast() .union(this.toDataStream()) .transform("add route transform", SiddhiTypeFactory.getStreamTupleTypeInformation(TypeInformation.of(Object.class)), new AddRouteOperator(getCepEnvironment().getDataStreamSchemas())); DataStream<Tuple2<StreamRoute, Object>> partitionedStream = new DataStream<>( unionStream.getExecutionEnvironment(), new PartitionTransformation<>(unionStream.getTransformation(), new DynamicPartitioner())); return new ExecutionSiddhiStream(partitionedStream, null, getCepEnvironment()); }