/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
@Test public void testGetTaskFactoryWithStreamAppDescriptor() { StreamApplicationDescriptorImpl mockStreamApp = mock(StreamApplicationDescriptorImpl.class); OperatorSpecGraph mockSpecGraph = mock(OperatorSpecGraph.class); when(mockStreamApp.getOperatorSpecGraph()).thenReturn(mockSpecGraph); TaskFactory streamTaskFactory = TaskFactoryUtil.getTaskFactory(mockStreamApp); assertTrue(streamTaskFactory instanceof StreamTaskFactory); StreamTask streamTask = ((StreamTaskFactory) streamTaskFactory).createInstance(); assertTrue(streamTask instanceof StreamOperatorTask); verify(mockSpecGraph).clone(); }
@Test public void testEmptyChain() { StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { }, mock(Config.class)); OperatorImplGraph opGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), context, mock(Clock.class)); assertEquals(0, opGraph.getAllInputOperators().size()); }
@Test public void testCopy() { StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(m -> m.toString(), m -> m, mock(KVSerde.class), testRepartitionedStreamName); }, getConfig()); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); OperatorSpecGraph clonedGraph = specGraph.clone(); OperatorSpecTestUtils.assertClonedGraph(specGraph, clonedGraph); }
@Test public void testNonKeyedTumblingWindowsDiscardingMode() throws Exception { OperatorSpecGraph sgb = this.getTumblingWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(1000))).getOperatorSpecGraph(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); Assert.assertEquals(windowPanes.size(), 0); integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator)); Assert.assertEquals(windowPanes.size(), 0); testClock.advanceTime(Duration.ofSeconds(1)); Assert.assertEquals(windowPanes.size(), 0); task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 1); Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 9); }
@Test public void testTranslateGroupBy() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10); String sql = "Insert into testavro.pageViewCountTopic(jobName, pageKey, `count`)" + " select 'SampleJob' as jobName, pv.pageKey, count(*) as `count`" + " from testavro.PAGEVIEW as pv" + " where pv.pageKey = 'job' or pv.pageKey = 'inbox'" + " group by (pv.pageKey)"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); Assert.assertEquals(1, specGraph.getInputOperators().size()); Assert.assertEquals(1, specGraph.getOutputStreams().size()); assertTrue(specGraph.hasWindowOrJoins()); Collection<OperatorSpec> operatorSpecs = specGraph.getAllOperatorSpecs(); }
@Test public void testTumblingWindowsAccumulatingMode() throws Exception { OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator)); testClock.advanceTime(Duration.ofSeconds(1)); task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 7); Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1)); Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2)); Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1)); Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4); Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2)); Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4); }
@Test public void testTumblingWindowsDiscardingMode() throws Exception { OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator)); testClock.advanceTime(Duration.ofSeconds(1)); task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 5); Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1)); Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2)); Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1)); Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2)); Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(4).getKey().getKey(), new Integer(3)); Assert.assertEquals((windowPanes.get(4).getMessage()).size(), 1); }
Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
@Test public void testTumblingAggregatingWindowsDiscardingMode() throws Exception { when(this.context.getTaskContext().getStore("jobName-jobId-window-w1")) .thenReturn(new TestInMemoryStore<>(new TimeSeriesKeySerde(new IntegerSerde()), new IntegerSerde())); OperatorSpecGraph sgb = this.getAggregateTumblingWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2))).getOperatorSpecGraph(); List<WindowPane<Integer, Integer>> windowPanes = new ArrayList<>(); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Integer>) envelope.getMessage()); integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator)); testClock.advanceTime(Duration.ofSeconds(1)); task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 5); Assert.assertEquals(windowPanes.get(0).getMessage(), new Integer(2)); Assert.assertEquals(windowPanes.get(1).getMessage(), new Integer(2)); Assert.assertEquals(windowPanes.get(2).getMessage(), new Integer(2)); Assert.assertEquals(windowPanes.get(3).getMessage(), new Integer(2)); Assert.assertEquals(windowPanes.get(4).getMessage(), new Integer(1)); }
@Test public void testBroadcastChain() { String inputStreamId = "input"; String inputSystem = "input-system"; String inputPhysicalName = "input-stream"; HashMap<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "test-job"); configMap.put(JobConfig.JOB_ID(), "1"); StreamTestUtils.addStreamConfigs(configMap, inputStreamId, inputSystem, inputPhysicalName); Config config = new MapConfig(configMap); when(this.context.getJobContext().getConfig()).thenReturn(config); StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { GenericSystemDescriptor sd = new GenericSystemDescriptor(inputSystem, "mockFactoryClass"); GenericInputDescriptor inputDescriptor = sd.getInputDescriptor(inputStreamId, mock(Serde.class)); MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor); inputStream.filter(mock(FilterFunction.class)); inputStream.map(mock(MapFunction.class)); }, config); OperatorImplGraph opImplGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class)); InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream(inputSystem, inputPhysicalName)); assertEquals(2, inputOpImpl.registeredOperators.size()); assertTrue(inputOpImpl.registeredOperators.stream() .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER)); assertTrue(inputOpImpl.registeredOperators.stream() .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP)); }
private StreamOperatorTask createStreamOperatorTask(Clock clock, StreamApplicationDescriptorImpl graphSpec) throws Exception { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream2", "insystem", "instream2"); Context context = new MockContext(new MapConfig(mapConfig)); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getSystemStreamPartitions()).thenReturn(ImmutableSet .of(new SystemStreamPartition("insystem", "instream", new Partition(0)), new SystemStreamPartition("insystem", "instream2", new Partition(0)))); when(context.getTaskContext().getTaskModel()).thenReturn(taskModel); when(context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); // need to return different stores for left and right side IntegerSerde integerSerde = new IntegerSerde(); TimestampedValueSerde timestampedValueSerde = new TimestampedValueSerde(new KVSerde(integerSerde, integerSerde)); when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-L"))) .thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde)); when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-R"))) .thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde)); StreamOperatorTask sot = new StreamOperatorTask(graphSpec.getOperatorSpecGraph(), clock); sot.init(context); return sot; }
@Test public void testEndOfStreamFlushesWithNoTriggerFirings() throws Exception { EndOfStreamStates endOfStreamStates = new EndOfStreamStates(ImmutableSet.of(new SystemStreamPartition("kafka", "integers", new Partition(0))), Collections.emptyMap()); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(EndOfStreamStates.class.getName())).thenReturn( endOfStreamStates); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(WatermarkStates.class.getName())).thenReturn( mock(WatermarkStates.class)); OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph(); TestClock testClock = new TestClock(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope( new SystemStreamPartition("kafka", "integers", new Partition(0))); task.process(endOfStream, messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 1); Assert.assertEquals(windowPanes.get(0).getMessage().size(), 4); verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK); verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); }
this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph(); TestClock testClock = new TestClock(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
@Test public void testSessionWindowsAccumulatingMode() throws Exception { OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph(); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); task.init(this.context); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); testClock.advanceTime(Duration.ofSeconds(1)); task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator); testClock.advanceTime(Duration.ofSeconds(1)); task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 2); Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2); Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1)); Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2)); Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2); Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 4); }
@Test public void testCancelationOfRepeatingNestedTriggers() throws Exception { OperatorSpecGraph sgb = this.getKeyedTumblingWindowStreamGraph(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1), Triggers.repeat(Triggers.any(Triggers.count(2), Triggers.timeSinceFirstMessage(Duration.ofMillis(500))))).getOperatorSpecGraph(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); TestClock testClock = new TestClock(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); //assert that the count trigger fired Assert.assertEquals(windowPanes.size(), 1); //advance the timer to enable the potential triggering of the inner timeSinceFirstMessage trigger task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); testClock.advanceTime(Duration.ofMillis(500)); //assert that the triggering of the count trigger cancelled the inner timeSinceFirstMessage trigger task.window(messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 2); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 3); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); //advance timer by 500 more millis to enable the default trigger testClock.advanceTime(Duration.ofMillis(500)); task.window(messageCollector, taskCoordinator); //assert that the default trigger fired Assert.assertEquals(windowPanes.size(), 4); }