@Test(expected = IllegalArgumentException.class) public void testGetInputDescriptorWithNullSerde() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name"); mySystem.getInputDescriptor("streamId", null); // should throw an exception }
@Test(expected = IllegalStateException.class) public void testGetSameInputStreamTwice() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); // should throw exception appDesc.getInputStream(isd2); }, getConfig()); }
@Test public void testGetInputStreamPreservesInsertionOrder() { Config mockConfig = getConfig(); String testStreamId1 = "test-stream-1"; String testStreamId2 = "test-stream-2"; String testStreamId3 = "test-stream-3"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(sd.getInputDescriptor(testStreamId1, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId2, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId3, mock(Serde.class))); }, mockConfig); List<InputOperatorSpec> inputSpecs = new ArrayList<>(streamAppDesc.getInputOperators().values()); assertEquals(inputSpecs.size(), 3); assertEquals(inputSpecs.get(0).getStreamId(), testStreamId1); assertEquals(inputSpecs.get(1).getStreamId(), testStreamId2); assertEquals(inputSpecs.get(2).getStreamId(), testStreamId3); }
@Test public void testISDObjectsWithOverrides() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); IntegerSerde streamSerde = new IntegerSerde(); GenericInputDescriptor<Integer> isd = mySystem.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); } }
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingInputStream() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); appDesc.withDefaultSystem(sd); // should throw exception }, getConfig()); }
@Test public void testISDConfigsWithDefaults() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); DoubleSerde streamSerde = new DoubleSerde(); GenericInputDescriptor<Double> isd = mySystem.getInputDescriptor("input-stream", streamSerde); Map<String, String> generatedConfigs = isd.toConfig(); Map<String, String> expectedConfigs = ImmutableMap.of("streams.input-stream.samza.system", "input-system"); assertEquals(expectedConfigs, generatedConfigs); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); }
@Test(expected = IllegalArgumentException.class) public void testGetInputStreamWithNullSerde() { GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor("mockStreamId", null); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); }
@Test public void testMultipleGetInputStreams() { String streamId1 = "test-stream-1"; String streamId2 = "test-stream-2"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId1, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId2, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); appDesc.getInputStream(isd2); }, getConfig()); InputOperatorSpec inputOpSpec1 = streamAppDesc.getInputOperators().get(streamId1); InputOperatorSpec inputOpSpec2 = streamAppDesc.getInputOperators().get(streamId2); assertEquals(2, streamAppDesc.getInputOperators().size()); assertEquals(streamId1, inputOpSpec1.getStreamId()); assertEquals(streamId2, inputOpSpec2.getStreamId()); assertEquals(2, streamAppDesc.getInputDescriptors().size()); assertEquals(isd1, streamAppDesc.getInputDescriptors().get(streamId1)); assertEquals(isd2, streamAppDesc.getInputDescriptors().get(streamId2)); }
@Test public void testGetInputStreamWithKeyValueSerde() { String streamId = "test-stream-1"; KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockKVSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertEquals(mockKeySerde, inputOpSpec.getKeySerde()); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
@Test public void testGetInputStreamWithRelaxedTypes() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); }
@Test public void testBroadcastChain() { String inputStreamId = "input"; String inputSystem = "input-system"; String inputPhysicalName = "input-stream"; HashMap<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "test-job"); configMap.put(JobConfig.JOB_ID(), "1"); StreamTestUtils.addStreamConfigs(configMap, inputStreamId, inputSystem, inputPhysicalName); Config config = new MapConfig(configMap); when(this.context.getJobContext().getConfig()).thenReturn(config); StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { GenericSystemDescriptor sd = new GenericSystemDescriptor(inputSystem, "mockFactoryClass"); GenericInputDescriptor inputDescriptor = sd.getInputDescriptor(inputStreamId, mock(Serde.class)); MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor); inputStream.filter(mock(FilterFunction.class)); inputStream.map(mock(MapFunction.class)); }, config); OperatorImplGraph opImplGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class)); InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream(inputSystem, inputPhysicalName)); assertEquals(2, inputOpImpl.registeredOperators.size()); assertTrue(inputOpImpl.registeredOperators.stream() .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER)); assertTrue(inputOpImpl.registeredOperators.stream() .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP)); }
private StreamApplicationDescriptorImpl getTestJoinStreamGraph(TestJoinFunction joinFn) throws IOException { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream2", "insystem", "instream2"); Config config = new MapConfig(mapConfig); return new StreamApplicationDescriptorImpl(appDesc -> { IntegerSerde integerSerde = new IntegerSerde(); KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde); GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor1 = sd.getInputDescriptor("inStream", kvSerde); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor2 = sd.getInputDescriptor("inStream2", kvSerde); MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor1); MessageStream<KV<Integer, Integer>> inStream2 = appDesc.getInputStream(inputDescriptor2); inStream .join(inStream2, joinFn, integerSerde, kvSerde, kvSerde, JOIN_TTL, "j1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }, config); }
@Test(expected = SamzaException.class) public void joinWithSelfThrowsException() throws Exception { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); Config config = new MapConfig(mapConfig); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { IntegerSerde integerSerde = new IntegerSerde(); KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde); GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("inStream", kvSerde); MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor); inStream.join(inStream, new TestJoinFunction(), integerSerde, kvSerde, kvSerde, JOIN_TTL, "join"); }, config); createStreamOperatorTask(new SystemClock(), streamAppDesc); // should throw an exception }
@Test public void testGetInputStreamWithExpandingSystem() { String streamId = "test-stream-1"; String expandedStreamId = "expanded-stream"; AtomicInteger expandCallCount = new AtomicInteger(); StreamExpander expander = (sg, isd) -> { expandCallCount.incrementAndGet(); InputDescriptor expandedISD = new GenericSystemDescriptor("expanded-system", "mockFactoryClass") .getInputDescriptor(expandedStreamId, new IntegerSerde()); return sg.getInputStream(expandedISD); }; MockExpandingSystemDescriptor sd = new MockExpandingSystemDescriptor("mock-system", expander); MockInputDescriptor isd = sd.getInputDescriptor(streamId, new IntegerSerde()); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(expandedStreamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(1, expandCallCount.get()); assertFalse(streamAppDesc.getInputOperators().containsKey(streamId)); assertFalse(streamAppDesc.getInputDescriptors().containsKey(streamId)); assertTrue(streamAppDesc.getInputDescriptors().containsKey(expandedStreamId)); assertEquals(expandedStreamId, inputOpSpec.getStreamId()); }
@Test public void testGetInputStreamWithValueSerde() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
@Test public void testAPIUsage() { // does not assert anything, but acts as a compile-time check on expected descriptor type parameters // and validates that the method calls can be chained. GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); GenericInputDescriptor<Integer> input1 = mySystem.getInputDescriptor("input1", new IntegerSerde()); GenericOutputDescriptor<Integer> output1 = mySystem.getOutputDescriptor("output1", new IntegerSerde()); input1 .withPhysicalName("input-1") .shouldBootstrap() .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST) .withPriority(1) .shouldResetOffset() .isBounded() .shouldDeleteCommittedMessages() .withStreamConfigs(Collections.emptyMap()); output1 .withPhysicalName("output-1") .withStreamConfigs(Collections.emptyMap()); }
private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
private StreamApplicationDescriptorImpl getAggregateTumblingWindowStreamGraph(AccumulationMode mode, Duration timeDuration, Trigger<IntegerEnvelope> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); MessageStream<KV<Integer, Integer>> integers = appDesc.getInputStream(inputDescriptor); integers .map(new KVMapFunction()) .window(Windows.<IntegerEnvelope, Integer>tumblingWindow(timeDuration, () -> 0, (m, c) -> c + 1, new IntegerSerde()) .setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }