/** * Tests that the max parallelism is properly set for connected * streams. */ @Test public void testMaxParallelismWithConnectedKeyedStream() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128); DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult = input1 .connect(input2) .keyBy(value -> value, value -> value) .map(new NoOpIntCoMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner(); StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner(); }
.addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource") .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap") .keyBy(0)
.addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource") .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap") .keyBy(0)
public static void createTableServiceJob(StreamExecutionEnvironment env, ServiceDescriptor serviceDescriptor) { ResourceSpec resourceSpec = ResourceSpec.newBuilder() .setCpuCores(serviceDescriptor.getServiceCpuCores()) .setHeapMemoryInMB(serviceDescriptor.getServiceHeapMemoryMb()) .setDirectMemoryInMB(serviceDescriptor.getServiceDirectMemoryMb()) .setNativeMemoryInMB(serviceDescriptor.getServiceNativeMemoryMb()) .build(); DataStream<BaseRow> ds = env.addSource(new FlinkTableServiceFunction(serviceDescriptor)) .setParallelism(serviceDescriptor.getServiceParallelism()) .setMaxParallelism(serviceDescriptor.getServiceParallelism()); ds.addSink(new SinkFunction<BaseRow>() { @Override public void invoke(BaseRow value, Context context) { } }).setParallelism(serviceDescriptor.getServiceParallelism()); }