public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
@Test public void testMultipleGetInputStreams() { String streamId1 = "test-stream-1"; String streamId2 = "test-stream-2"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId1, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId2, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); appDesc.getInputStream(isd2); }, getConfig()); InputOperatorSpec inputOpSpec1 = streamAppDesc.getInputOperators().get(streamId1); InputOperatorSpec inputOpSpec2 = streamAppDesc.getInputOperators().get(streamId2); assertEquals(2, streamAppDesc.getInputOperators().size()); assertEquals(streamId1, inputOpSpec1.getStreamId()); assertEquals(streamId2, inputOpSpec2.getStreamId()); assertEquals(2, streamAppDesc.getInputDescriptors().size()); assertEquals(isd1, streamAppDesc.getInputDescriptors().get(streamId1)); assertEquals(isd2, streamAppDesc.getInputDescriptors().get(streamId2)); }
@Test public void testGetInputStreamPreservesInsertionOrder() { Config mockConfig = getConfig(); String testStreamId1 = "test-stream-1"; String testStreamId2 = "test-stream-2"; String testStreamId3 = "test-stream-3"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(sd.getInputDescriptor(testStreamId1, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId2, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId3, mock(Serde.class))); }, mockConfig); List<InputOperatorSpec> inputSpecs = new ArrayList<>(streamAppDesc.getInputOperators().values()); assertEquals(inputSpecs.size(), 3); assertEquals(inputSpecs.get(0).getStreamId(), testStreamId1); assertEquals(inputSpecs.get(1).getStreamId(), testStreamId2); assertEquals(inputSpecs.get(2).getStreamId(), testStreamId3); }
@Test public void testGetInputStreamWithRelaxedTypes() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); }
@Test public void testGetInputStreamWithKeyValueSerde() { String streamId = "test-stream-1"; KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockKVSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertEquals(mockKeySerde, inputOpSpec.getKeySerde()); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
@Test public void testGetIntermediateStreamWithDefaultSystemDescriptor() { Config mockConfig = getConfig(); String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig); GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory"); streamAppDesc.withDefaultSystem(sd); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); }
@Test public void testGetInputStreamWithExpandingSystem() { String streamId = "test-stream-1"; String expandedStreamId = "expanded-stream"; AtomicInteger expandCallCount = new AtomicInteger(); StreamExpander expander = (sg, isd) -> { expandCallCount.incrementAndGet(); InputDescriptor expandedISD = new GenericSystemDescriptor("expanded-system", "mockFactoryClass") .getInputDescriptor(expandedStreamId, new IntegerSerde()); return sg.getInputStream(expandedISD); }; MockExpandingSystemDescriptor sd = new MockExpandingSystemDescriptor("mock-system", expander); MockInputDescriptor isd = sd.getInputDescriptor(streamId, new IntegerSerde()); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(expandedStreamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(1, expandCallCount.get()); assertFalse(streamAppDesc.getInputOperators().containsKey(streamId)); assertFalse(streamAppDesc.getInputDescriptors().containsKey(streamId)); assertTrue(streamAppDesc.getInputDescriptors().containsKey(expandedStreamId)); assertEquals(expandedStreamId, inputOpSpec.getStreamId()); }
@Test public void testGetIntermediateStreamWithKeyValueSerde() { String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockKVSerde, false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); assertEquals(mockKeySerde, intermediateStreamImpl.getOutputStream().getKeySerde()); assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde()); assertEquals(mockKeySerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde()); assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde()); }
@Test public void testGetInputStreamWithTransformFunction() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); InputTransformer transformer = ime -> ime; MockTransformingSystemDescriptor sd = new MockTransformingSystemDescriptor("mockSystem", transformer); MockInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertEquals(transformer, inputOpSpec.getTransformer()); }
@Test public void testGetInputStreamWithValueSerde() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
this.outputStrmMap = new LinkedHashMap<>(); outputStrmMap.put(outputStreamId, outputStream1); when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap)); when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap)); this.allOpSpecs = new HashSet<OperatorSpec>() { {
@Test public void testPartitionBy() { MapFunction<Object, String> keyFn = m -> m.toString(); MapFunction<Object, Object> valueFn = m -> m; KVSerde<Object, Object> partitionBySerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(keyFn, valueFn, partitionBySerde, testRepartitionedStreamName); }, getConfig()); assertEquals(2, streamAppDesc.getInputOperators().size()); Map<String, InputOperatorSpec> inputOpSpecs = streamAppDesc.getInputOperators(); assertTrue(inputOpSpecs.keySet().contains(String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName))); InputOperatorSpec inputOpSpec = inputOpSpecs.get(String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName)); assertEquals(String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName), inputOpSpec.getStreamId()); assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde); assertTrue(inputOpSpec.getValueSerde() instanceof NoOpSerde); assertTrue(inputOpSpec.isKeyed()); assertNull(inputOpSpec.getScheduledFn()); assertNull(inputOpSpec.getWatermarkFn()); InputOperatorSpec originInputSpec = inputOpSpecs.get(testInputDescriptor.getStreamId()); assertTrue(originInputSpec.getRegisteredOperatorSpecs().toArray()[0] instanceof PartitionByOperatorSpec); PartitionByOperatorSpec reparOpSpec = (PartitionByOperatorSpec) originInputSpec.getRegisteredOperatorSpecs().toArray()[0]; assertEquals(reparOpSpec.getOpId(), String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName)); assertEquals(reparOpSpec.getKeyFunction(), keyFn); assertEquals(reparOpSpec.getValueFunction(), valueFn); assertEquals(reparOpSpec.getOutputStream().getStreamId(), reparOpSpec.getOpId()); assertNull(reparOpSpec.getScheduledFn()); assertNull(reparOpSpec.getWatermarkFn()); }
@Test public void testPartitionByWithNoSerde() { MapFunction<Object, String> keyFn = m -> m.toString(); MapFunction<Object, Object> valueFn = m -> m; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(keyFn, valueFn, mock(KVSerde.class), testRepartitionedStreamName); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get( String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName)); assertNotNull(inputOpSpec); assertNull(inputOpSpec.getKeySerde()); assertNull(inputOpSpec.getValueSerde()); assertTrue(inputOpSpec.isKeyed()); assertNull(inputOpSpec.getScheduledFn()); assertNull(inputOpSpec.getWatermarkFn()); InputOperatorSpec originInputSpec = streamAppDesc.getInputOperators().get(testInputDescriptor.getStreamId()); assertTrue(originInputSpec.getRegisteredOperatorSpecs().toArray()[0] instanceof PartitionByOperatorSpec); PartitionByOperatorSpec reparOpSpec = (PartitionByOperatorSpec) originInputSpec.getRegisteredOperatorSpecs().toArray()[0]; assertEquals(reparOpSpec.getOpId(), String.format("%s-%s-partition_by-%s", testJobName, testJobId, testRepartitionedStreamName)); assertEquals(reparOpSpec.getKeyFunction(), keyFn); assertEquals(reparOpSpec.getValueFunction(), valueFn); assertEquals(reparOpSpec.getOutputStream().getStreamId(), reparOpSpec.getOpId()); assertNull(reparOpSpec.getScheduledFn()); assertNull(reparOpSpec.getWatermarkFn()); }
@Test public void testGetIntermediateStreamWithValueSerde() { String streamId = "stream-1"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); Serde mockValueSerde = mock(Serde.class); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde()); assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde()); }