@Test(expected = IllegalStateException.class) public void testGetSameIntermediateStreamTwice() { StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false); // should throw exception streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false); }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingIntermediateStream() { String streamId = "test-stream-1"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); streamAppDesc.withDefaultSystem(mock(SystemDescriptor.class)); // should throw exception }
@Override public <K, V> MessageStream<KV<K, V>> partitionBy(MapFunction<? super M, ? extends K> keyExtractor, MapFunction<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.PARTITION_BY, userDefinedId); IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, false); if (!intermediateStream.isKeyed()) { // this can only happen when the default serde partitionBy variant is being used throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde."); } PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec = OperatorSpecs.createPartitionByOperatorSpec( intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId); this.operatorSpec.registerNextOperatorSpec(partitionByOperatorSpec); return intermediateStream; }
@Override public <K, V> MessageStream<KV<K, V>> partitionBy(MapFunction<? super M, ? extends K> keyExtractor, MapFunction<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.PARTITION_BY, userDefinedId); IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, false); if (!intermediateStream.isKeyed()) { // this can only happen when the default serde partitionBy variant is being used throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde."); } PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec = OperatorSpecs.createPartitionByOperatorSpec( intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId); this.operatorSpec.registerNextOperatorSpec(partitionByOperatorSpec); return intermediateStream; }
@Override public <K, V> MessageStream<KV<K, V>> partitionBy(MapFunction<? super M, ? extends K> keyExtractor, MapFunction<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.PARTITION_BY, userDefinedId); IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, false); if (!intermediateStream.isKeyed()) { // this can only happen when the default serde partitionBy variant is being used throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde."); } PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec = OperatorSpecs.createPartitionByOperatorSpec( intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId); this.operatorSpec.registerNextOperatorSpec(partitionByOperatorSpec); return intermediateStream; }
@Override public <K, V> MessageStream<KV<K, V>> partitionBy(MapFunction<? super M, ? extends K> keyExtractor, MapFunction<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.PARTITION_BY, userDefinedId); IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, false); if (!intermediateStream.isKeyed()) { // this can only happen when the default serde partitionBy variant is being used throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde."); } PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec = OperatorSpecs.createPartitionByOperatorSpec( intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId); this.operatorSpec.registerNextOperatorSpec(partitionByOperatorSpec); return intermediateStream; }
@Override public <K, V> MessageStream<KV<K, V>> partitionBy(MapFunction<? super M, ? extends K> keyExtractor, MapFunction<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.PARTITION_BY, userDefinedId); IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, false); if (!intermediateStream.isKeyed()) { // this can only happen when the default serde partitionBy variant is being used throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde."); } PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec = OperatorSpecs.createPartitionByOperatorSpec( intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId); this.operatorSpec.registerNextOperatorSpec(partitionByOperatorSpec); return intermediateStream; }
@Test(expected = NullPointerException.class) public void testGetIntermediateStreamWithNoSerde() { Config mockConfig = getConfig(); String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, null, false); // should throw }
@Test public void testPartitionBy() throws IOException { StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class); OperatorSpec mockOpSpec = mock(OperatorSpec.class); String mockOpName = "mockName"; when(mockGraph.getNextOpId(anyObject(), anyObject())).thenReturn(mockOpName); OutputStreamImpl mockOutputStreamImpl = mock(OutputStreamImpl.class); KVSerde mockKVSerde = mock(KVSerde.class); IntermediateMessageStreamImpl mockIntermediateStream = mock(IntermediateMessageStreamImpl.class); when(mockGraph.getIntermediateStream(eq(mockOpName), eq(mockKVSerde), eq(false))) .thenReturn(mockIntermediateStream); when(mockIntermediateStream.getOutputStream()) .thenReturn(mockOutputStreamImpl); when(mockIntermediateStream.isKeyed()).thenReturn(true); MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec); MapFunction mockKeyFunction = mock(MapFunction.class); MapFunction mockValueFunction = mock(MapFunction.class); inputStream.partitionBy(mockKeyFunction, mockValueFunction, mockKVSerde, "p1"); ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class); verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture()); OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue(); assertTrue(registeredOpSpec instanceof PartitionByOperatorSpec); assertEquals(OpCode.PARTITION_BY, registeredOpSpec.getOpCode()); assertEquals(mockOutputStreamImpl, ((PartitionByOperatorSpec) registeredOpSpec).getOutputStream()); assertEquals(mockKeyFunction, ((PartitionByOperatorSpec) registeredOpSpec).getKeyFunction()); assertEquals(mockValueFunction, ((PartitionByOperatorSpec) registeredOpSpec).getValueFunction()); }
IntermediateMessageStreamImpl mockPartitionedStream = new IntermediateMessageStreamImpl(mockAppDesc, mockInputOp, mockOutputStream); when(mockAppDesc.getIntermediateStream(any(String.class), any(Serde.class), eq(false))).thenReturn(mockPartitionedStream); doAnswer(this.getRegisteredTableAnswer()).when(mockAppDesc).getTable(any(RocksDbTableDescriptor.class));
@Test public void testGetIntermediateStreamWithDefaultSystemDescriptor() { Config mockConfig = getConfig(); String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig); GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory"); streamAppDesc.withDefaultSystem(sd); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); }
@Test public void testGetIntermediateStreamWithKeyValueSerde() { String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockKVSerde, false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); assertEquals(mockKeySerde, intermediateStreamImpl.getOutputStream().getKeySerde()); assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde()); assertEquals(mockKeySerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde()); assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde()); }
@Test public void testGetIntermediateStreamWithValueSerde() { String streamId = "stream-1"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); Serde mockValueSerde = mock(Serde.class); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde()); assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde()); }