/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Transforms a {@code PartitionTransformation}. * * <p>For this we create a virtual node in the {@code StreamGraph} that holds the partition * property. @see StreamGraphGenerator */ private <T> Collection<Integer> transformPartition(PartitionTransformation<T> partition) { StreamTransformation<T> input = partition.getInput(); List<Integer> resultIds = new ArrayList<>(); Collection<Integer> transformedIds = transform(input); for (Integer transformedId: transformedIds) { int virtualId = StreamTransformation.getNewNodeId(); streamGraph.addVirtualPartitionNode(transformedId, virtualId, partition.getPartitioner()); resultIds.add(virtualId); } return resultIds; }
private <T> void validateSplitTransformation(StreamTransformation<T> input) { if (input instanceof SelectTransformation || input instanceof SplitTransformation) { throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof SideOutputTransformation) { throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof UnionTransformation) { for (StreamTransformation<T> transformation : ((UnionTransformation<T>) input).getInputs()) { validateSplitTransformation(transformation); } } else if (input instanceof PartitionTransformation) { validateSplitTransformation(((PartitionTransformation) input).getInput()); } else { return; } } }
private <T> void validateSplitTransformation(StreamTransformation<T> input) { if (input instanceof SelectTransformation || input instanceof SplitTransformation) { throw new IllegalStateException("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof SideOutputTransformation) { throw new IllegalStateException("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs."); } else if (input instanceof UnionTransformation) { for (StreamTransformation<T> transformation : ((UnionTransformation<T>) input).getInputs()) { validateSplitTransformation(transformation); } } else if (input instanceof PartitionTransformation) { validateSplitTransformation(((PartitionTransformation) input).getInput()); } else { return; } } }
/** * Creates a new {@link KeyedStream} using the given {@link KeySelector} * to partition operator state by key. * * @param dataStream * Base stream of data * @param keySelector * Function for determining state partitions */ public KeyedStream(DataStream<T> dataStream, KeySelector<T, KEY> keySelector, TypeInformation<KEY> keyType) { this( dataStream, new PartitionTransformation<>( dataStream.getTransformation(), new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM)), keySelector, keyType); }
/** * Transforms a {@code PartitionTransformation}. * * <p>For this we create a virtual node in the {@code StreamGraph} that holds the partition * property. @see StreamGraphGenerator */ private <T> Collection<Integer> transformPartition(PartitionTransformation<T> partition) { StreamTransformation<T> input = partition.getInput(); List<Integer> resultIds = new ArrayList<>(); Collection<Integer> transformedIds = transform(input); for (Integer transformedId: transformedIds) { int virtualId = StreamTransformation.getNewNodeId(); streamGraph.addVirtualPartitionNode(transformedId, virtualId, partition.getPartitioner()); resultIds.add(virtualId); } return resultIds; }
/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param typeInfo Explicit type information about the key type. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector, TypeInformation<K> typeInfo) { PartitionTransformation<T> partitionTransformation = new PartitionTransformation<>( stream.getTransformation(), new ForwardPartitioner<>()); return new KeyedStream<>( stream, partitionTransformation, keySelector, typeInfo); }
/** * Transforms a {@code PartitionTransformation}. * * <p>For this we create a virtual node in the {@code StreamGraph} that holds the partition * property. @see StreamGraphGenerator */ private <T> Collection<Integer> transformPartition(PartitionTransformation<T> partition) { StreamTransformation<T> input = partition.getInput(); List<Integer> resultIds = new ArrayList<>(); Collection<Integer> transformedIds = transform(input); for (Integer transformedId: transformedIds) { int virtualId = StreamTransformation.getNewNodeId(); streamGraph.addVirtualPartitionNode(transformedId, virtualId, partition.getPartitioner()); resultIds.add(virtualId); } return resultIds; }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Transforms a {@code PartitionTransformation}. * * <p>For this we create a virtual node in the {@code StreamGraph} that holds the partition * property. @see StreamGraphGenerator */ private <T> Collection<Integer> transformPartition(PartitionTransformation<T> partition) { StreamTransformation<T> input = partition.getInput(); List<Integer> resultIds = new ArrayList<>(); Collection<Integer> transformedIds = transform(input); for (Integer transformedId: transformedIds) { int virtualId = StreamTransformation.getNewNodeId(); streamGraph.addVirtualPartitionNode(transformedId, virtualId, partition.getPartitioner()); resultIds.add(virtualId); } return resultIds; }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Internal function for setting the partitioner for the DataStream. * * @param partitioner * Partitioner to set. * @return The modified DataStream. */ protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new DataStream<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
@Override protected DataStream<T> setConnectionType(StreamPartitioner<T> partitioner) { return new SingleOutputStreamOperator<>(this.getExecutionEnvironment(), new PartitionTransformation<>(this.getTransformation(), partitioner)); }
/** * Creates a new {@link KeyedStream} using the given {@link KeySelector} * to partition operator state by key. * * @param dataStream * Base stream of data * @param keySelector * Function for determining state partitions */ public KeyedStream(DataStream<T> dataStream, KeySelector<T, KEY> keySelector, TypeInformation<KEY> keyType) { this( dataStream, new PartitionTransformation<>( dataStream.getTransformation(), new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM)), keySelector, keyType); }
/** * Creates a new {@link KeyedStream} using the given {@link KeySelector} * to partition operator state by key. * * @param dataStream * Base stream of data * @param keySelector * Function for determining state partitions */ public KeyedStream(DataStream<T> dataStream, KeySelector<T, KEY> keySelector, TypeInformation<KEY> keyType) { this( dataStream, new PartitionTransformation<>( dataStream.getTransformation(), new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM)), keySelector, keyType); }
/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param typeInfo Explicit type information about the key type. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector, TypeInformation<K> typeInfo) { PartitionTransformation<T> partitionTransformation = new PartitionTransformation<>( stream.getTransformation(), new ForwardPartitioner<>()); return new KeyedStream<>( stream, partitionTransformation, keySelector, typeInfo); }
/** * Creates a new {@link KeyedStream} using the given {@link KeySelector} * to partition operator state by key. * * @param dataStream * Base stream of data * @param keySelector * Function for determining state partitions */ public KeyedStream(DataStream<T> dataStream, KeySelector<T, KEY> keySelector, TypeInformation<KEY> keyType) { super( dataStream.getExecutionEnvironment(), new PartitionTransformation<>( dataStream.getTransformation(), new KeyGroupStreamPartitioner<>(keySelector, StreamGraphGenerator.DEFAULT_LOWER_BOUND_MAX_PARALLELISM))); this.keySelector = keySelector; this.keyType = validateKeyType(keyType); }
/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param typeInfo Explicit type information about the key type. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector, TypeInformation<K> typeInfo) { PartitionTransformation<T> partitionTransformation = new PartitionTransformation<>( stream.getTransformation(), new ForwardPartitioner<>()); return new KeyedStream<>( stream, partitionTransformation, keySelector, typeInfo); }
/** * Siddhi Continuous Query Language (CQL) * * @return ExecutionSiddhiStream context */ public ExecutionSiddhiStream cql(DataStream<ControlEvent> controlStream) { DataStream<Tuple2<StreamRoute, Object>> unionStream = controlStream .map(new NamedControlStream(ControlEvent.DEFAULT_INTERNAL_CONTROL_STREAM)) .broadcast() .union(this.toDataStream()) .transform("add route transform", SiddhiTypeFactory.getStreamTupleTypeInformation(TypeInformation.of(Object.class)), new AddRouteOperator(getCepEnvironment().getDataStreamSchemas())); DataStream<Tuple2<StreamRoute, Object>> partitionedStream = new DataStream<>( unionStream.getExecutionEnvironment(), new PartitionTransformation<>(unionStream.getTransformation(), new DynamicPartitioner())); return new ExecutionSiddhiStream(partitionedStream, null, getCepEnvironment()); }