@Test(expected = IllegalArgumentException.class) public void testGetInputDescriptorWithNullSerde() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name"); mySystem.getInputDescriptor("streamId", null); // should throw an exception }
@Test public void testAPIUsage() { // does not assert anything, but acts as a compile-time check on expected descriptor type parameters // and validates that the method calls can be chained. GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); GenericInputDescriptor<Integer> input1 = mySystem.getInputDescriptor("input1", new IntegerSerde()); GenericOutputDescriptor<Integer> output1 = mySystem.getOutputDescriptor("output1", new IntegerSerde()); input1 .withPhysicalName("input-1") .shouldBootstrap() .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST) .withPriority(1) .shouldResetOffset() .isBounded() .shouldDeleteCommittedMessages() .withStreamConfigs(Collections.emptyMap()); output1 .withPhysicalName("output-1") .withStreamConfigs(Collections.emptyMap()); }
@Before public void setUp() { defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>()); inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName"); intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName"); input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde); input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde); outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde); intermediateInputDescriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); intermediateOutputDescriptor = intermediateSystemDescriptor.getOutputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); broadcastInputDesriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-broadcast-b1", defaultSerde) .withPhysicalName("jobName-jobId-broadcast-b1"); Map<String, String> configs = new HashMap<>(); configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.putAll(input1Descriptor.toConfig()); configs.putAll(input2Descriptor.toConfig()); configs.putAll(outputDescriptor.toConfig()); configs.putAll(inputSystemDescriptor.toConfig()); configs.putAll(outputSystemDescriptor.toConfig()); configs.putAll(intermediateSystemDescriptor.toConfig()); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystemDescriptor.getSystemName()); mockConfig = spy(new MapConfig(configs)); mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); }
@Test public void testSDConfigs() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(ImmutableMap.of("custom-config-key", "custom-config-value")) .withDefaultStreamConfigs(ImmutableMap.of("custom-stream-config-key", "custom-stream-config-value")) .withDefaultStreamOffsetDefault(SystemStreamMetadata.OffsetType.UPCOMING); Map<String, String> generatedConfigs = mySystem.toConfig(); Map<String, String> expectedConfigs = ImmutableMap.of( "systems.input-system.samza.factory", "factory.class.name", "systems.input-system.custom-config-key", "custom-config-value", "systems.input-system.default.stream.custom-stream-config-key", "custom-stream-config-value", "systems.input-system.default.stream.samza.offset.default", "upcoming" ); assertEquals(expectedConfigs, generatedConfigs); }
@Test public void testISDObjectsWithOverrides() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); IntegerSerde streamSerde = new IntegerSerde(); GenericInputDescriptor<Integer> isd = mySystem.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); } }
@Test public void testMultipleSystemDescriptorForSameSystemName() { GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class)); GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class)); GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); try { appDesc.getInputStream(isd2); fail("Adding input stream with the same system name but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } try { appDesc.getOutputStream(osd1); fail("adding output stream with the same system name but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } }, getConfig()); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.withDefaultSystem(sd2); try { appDesc.getInputStream(isd1); fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } }, getConfig()); }
@Test(expected = IllegalStateException.class) public void testGetSameOutputStreamTwice() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd1 = sd.getOutputDescriptor(streamId, mock(Serde.class)); GenericOutputDescriptor osd2 = sd.getOutputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd1); appDesc.getOutputStream(osd2); // should throw exception }, getConfig()); }
String intermediateBroadcast = "intermediate-broadcast1"; GenericInputDescriptor<KV<Object, Object>> intermediateInput1 = system1Descriptor.getInputDescriptor( intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor( intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); GenericInputDescriptor<KV<Object, Object>> intermediateBroacastInput1 = system1Descriptor.getInputDescriptor( intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor( intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); inputDescriptors.put(intermediateStream1, intermediateInput1);
@Test public void testTaskApplicationWithTableAndSideInput() { GenericInputDescriptor<KV<String, Object>> sideInput1 = inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde); BaseTableDescriptor mockTableDescriptor = new MockLocalTableDescriptor("testTable", defaultSerde) .withSideInputs(Arrays.asList(sideInput1.getStreamId())) .withConfig("mock.table.provider.config", "mock.config.value"); StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig); taskAppDesc.withInputStream(inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde)); configureJobNode(taskAppDesc);
@Test(expected = IllegalArgumentException.class) public void testGetSystemDescriptorWithEmptySystemName() { new GenericSystemDescriptor(" ", "factory.class.name"); } }
inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName"); intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName"); input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde); input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde); outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde); table1Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table1", defaultSerde); table2Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table2", defaultSerde);
@Test(expected = IllegalArgumentException.class) public void testGetOutputStreamWithNullSerde() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, null); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); }, getConfig()); }
@Test public void testStreamApplicationWithTableAndSideInput() { mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); // add table to the RepartitionJoinStreamApplication GenericInputDescriptor<KV<String, Object>> sideInput1 = inputSystemDescriptor.getInputDescriptor("sideInput1", defaultSerde); BaseTableDescriptor mockTableDescriptor = new MockLocalTableDescriptor("testTable", defaultSerde) .withSideInputs(Arrays.asList(sideInput1.getStreamId())) .withSideInputsProcessor(mock(SideInputsProcessor.class, withSettings().serializable())) .withConfig("mock.table.provider.config", "mock.config.value"); // add side input and terminate at table in the appplication mockStreamAppDesc.getInputStream(sideInput1).sendTo(mockStreamAppDesc.getTable(mockTableDescriptor)); StreamEdge sideInputEdge = new StreamEdge(new StreamSpec(sideInput1.getStreamId(), "sideInput1", inputSystemDescriptor.getSystemName()), false, false, mockConfig); // need to put the sideInput related stream configuration to the original config // TODO: this is confusing since part of the system and stream related configuration is generated outside the JobGraphConfigureGenerator // It would be nice if all system and stream related configuration is generated in one place and only intermediate stream // configuration is generated by JobGraphConfigureGenerator Map<String, String> configs = new HashMap<>(mockConfig); configs.putAll(sideInputEdge.generateConfig()); mockConfig = spy(new MapConfig(configs)); configureJobNode(mockStreamAppDesc); // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator(); JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson"); Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges()); validateJobConfig(expectedJobConfig, jobConfig); Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 5); validateTableConfigure(jobConfig, deserializedSerdes, mockTableDescriptor); }
@Test(expected = IllegalArgumentException.class) public void testGetSystemDescriptorWithNullSystemName() { new GenericSystemDescriptor(null, "factory.class.name"); }
@Test(expected = IllegalStateException.class) public void testGetSameInputStreamTwice() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); // should throw exception appDesc.getInputStream(isd2); }, getConfig()); }
system1Descriptor = new GenericSystemDescriptor("system1", mockSystemFactoryClass); system2Descriptor = new GenericSystemDescriptor("system2", mockSystemFactoryClass); input1Descriptor = system1Descriptor.getInputDescriptor("input1", kvSerde); input2Descriptor = system2Descriptor.getInputDescriptor("input2", kvSerde); input3Descriptor = system2Descriptor.getInputDescriptor("input3", kvSerde); input4Descriptor = system1Descriptor.getInputDescriptor("input4", kvSerde); output1Descriptor = system1Descriptor.getOutputDescriptor("output1", kvSerde); output2Descriptor = system2Descriptor.getOutputDescriptor("output2", kvSerde);
@Test public void testISDConfigsWithDefaults() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); DoubleSerde streamSerde = new DoubleSerde(); GenericInputDescriptor<Double> isd = mySystem.getInputDescriptor("input-stream", streamSerde); Map<String, String> generatedConfigs = isd.toConfig(); Map<String, String> expectedConfigs = ImmutableMap.of("streams.input-stream.samza.system", "input-system"); assertEquals(expectedConfigs, generatedConfigs); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); }
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingOutputStream() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); appDesc.withDefaultSystem(sd); // should throw exception }, getConfig()); }
@Test public void testGetIntermediateStreamWithDefaultSystemDescriptor() { Config mockConfig = getConfig(); String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig); GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory"); streamAppDesc.withDefaultSystem(sd); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); }
@Test(expected = IllegalArgumentException.class) public void testGetInputStreamWithNullSerde() { GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor("mockStreamId", null); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); }