@Before public void setUp() { defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>()); inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName"); intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName"); input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde); input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde); outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde); intermediateInputDescriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); intermediateOutputDescriptor = intermediateSystemDescriptor.getOutputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); broadcastInputDesriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-broadcast-b1", defaultSerde) .withPhysicalName("jobName-jobId-broadcast-b1"); Map<String, String> configs = new HashMap<>(); configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.putAll(input1Descriptor.toConfig()); configs.putAll(input2Descriptor.toConfig()); configs.putAll(outputDescriptor.toConfig()); configs.putAll(inputSystemDescriptor.toConfig()); configs.putAll(outputSystemDescriptor.toConfig()); configs.putAll(intermediateSystemDescriptor.toConfig()); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystemDescriptor.getSystemName()); mockConfig = spy(new MapConfig(configs)); mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); }
@Test public void testStreamApplicationWithTableAndSideInput() { mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); // add table to the RepartitionJoinStreamApplication GenericInputDescriptor<KV<String, Object>> sideInput1 = inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde); BaseTableDescriptor mockTableDescriptor = new MockLocalTableDescriptor("testTable", defaultSerde) .withSideInputs(Arrays.asList(sideInput1.getStreamId())) .withSideInputsProcessor(mock(SideInputsProcessor.class, withSettings().serializable())) .withConfig("mock.table.provider.config", "mock.config.value"); // add side input and terminate at table in the appplication mockStreamAppDesc.getInputStream(sideInput1).sendTo(mockStreamAppDesc.getTable(mockTableDescriptor)); StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig); // need to put the sideInput related stream configuration to the original config // TODO: this is confusing since part of the system and stream related configuration is generated outside the JobGraphConfigureGenerator // It would be nice if all system and stream related configuration is generated in one place and only intermediate stream // configuration is generated by JobGraphConfigureGenerator Map<String, String> configs = new HashMap<>(mockConfig); configs.putAll(sideInputEdge.generateConfig()); mockConfig = spy(new MapConfig(configs)); configureJobNode(mockStreamAppDesc); // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator(); JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson"); Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges()); validateJobConfig(expectedJobConfig, jobConfig); Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 5); validateTableConfigure(jobConfig, deserializedSerdes, mockTableDescriptor); }
.withConfig("mock.table.provider.config", "mock.config.value"); StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig);