MetricsRegistry metricsRegistry = taskContext.getTaskMetricsRegistry(); this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opId + "-messages"); this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-message-ns"); this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-timer-ns"); this.taskName = taskContext.getTaskModel().getTaskName(); this.eosStates = (EndOfStreamStates) taskContext.fetchObject(EndOfStreamStates.class.getName()); this.watermarkStates = (WatermarkStates) taskContext.fetchObject(WatermarkStates.class.getName()); this.controlMessageSender = new ControlMessageSender(taskContext.getStreamMetadataCache()); this.taskModel = taskContext.getTaskModel(); this.callbackScheduler = taskContext.getCallbackScheduler(); handleInit(context);
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
/** * Given a registered object, fetchObject should get it. If an object is not registered at a key, then fetchObject * should return null. */ @Test public void testRegisterAndFetchObject() { String value = "hello world"; taskContext.registerObject("key", value); assertEquals(value, taskContext.fetchObject("key")); assertNull(taskContext.fetchObject("not a key")); } }
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP; taskContext.registerObject(EndOfStreamStates.class.getName(), new EndOfStreamStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts)); taskContext.registerObject(WatermarkStates.class.getName(), new WatermarkStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts, context.getContainerContext().getContainerMetricsRegistry()));
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
"integers", new Partition(0))), Collections.emptyMap()); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(EndOfStreamStates.class.getName())).thenReturn( endOfStreamStates); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(WatermarkStates.class.getName())).thenReturn( mock(WatermarkStates.class));
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP; taskContext.registerObject(EndOfStreamStates.class.getName(), new EndOfStreamStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts)); taskContext.registerObject(WatermarkStates.class.getName(), new WatermarkStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts, context.getContainerContext().getContainerMetricsRegistry()));
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
@Test public void testEndOfStreamFlushesWithNoTriggerFirings() throws Exception { EndOfStreamStates endOfStreamStates = new EndOfStreamStates(ImmutableSet.of(new SystemStreamPartition("kafka", "integers", new Partition(0))), Collections.emptyMap()); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(EndOfStreamStates.class.getName())).thenReturn( endOfStreamStates); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(WatermarkStates.class.getName())).thenReturn( mock(WatermarkStates.class)); OperatorSpecGraph sgb = this.getKeyedSessionWindowStreamGraph(AccumulationMode.DISCARDING, Duration.ofMillis(500)).getOperatorSpecGraph(); TestClock testClock = new TestClock(); List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>(); StreamOperatorTask task = new StreamOperatorTask(sgb, testClock); task.init(this.context); MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage()); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator); final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope( new SystemStreamPartition("kafka", "integers", new Partition(0))); task.process(endOfStream, messageCollector, taskCoordinator); Assert.assertEquals(windowPanes.size(), 1); Assert.assertEquals(windowPanes.get(0).getMessage().size(), 4); verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK); verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); }
MetricsRegistry metricsRegistry = taskContext.getTaskMetricsRegistry(); this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opId + "-messages"); this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-message-ns"); this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-timer-ns"); this.taskName = taskContext.getTaskModel().getTaskName(); this.eosStates = (EndOfStreamStates) taskContext.fetchObject(EndOfStreamStates.class.getName()); this.watermarkStates = (WatermarkStates) taskContext.fetchObject(WatermarkStates.class.getName()); this.controlMessageSender = new ControlMessageSender(taskContext.getStreamMetadataCache()); this.taskModel = taskContext.getTaskModel(); this.callbackScheduler = taskContext.getCallbackScheduler(); handleInit(context);
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP; taskContext.registerObject(EndOfStreamStates.class.getName(), new EndOfStreamStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts)); taskContext.registerObject(WatermarkStates.class.getName(), new WatermarkStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts, context.getContainerContext().getContainerMetricsRegistry()));
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
"integers", new Partition(0))), Collections.emptyMap()); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(EndOfStreamStates.class.getName())).thenReturn( endOfStreamStates); when(((TaskContextImpl) this.context.getTaskContext()).fetchObject(WatermarkStates.class.getName())).thenReturn( mock(WatermarkStates.class));
MetricsRegistry metricsRegistry = taskContext.getTaskMetricsRegistry(); this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opId + "-messages"); this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-message-ns"); this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-timer-ns"); this.taskName = taskContext.getTaskModel().getTaskName(); this.eosStates = (EndOfStreamStates) taskContext.fetchObject(EndOfStreamStates.class.getName()); this.watermarkStates = (WatermarkStates) taskContext.fetchObject(WatermarkStates.class.getName()); this.controlMessageSender = new ControlMessageSender(taskContext.getStreamMetadataCache()); this.taskModel = taskContext.getTaskModel(); this.callbackScheduler = taskContext.getCallbackScheduler(); handleInit(context);
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP; taskContext.registerObject(EndOfStreamStates.class.getName(), new EndOfStreamStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts)); taskContext.registerObject(WatermarkStates.class.getName(), new WatermarkStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts, context.getContainerContext().getContainerMetricsRegistry()));
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
MetricsRegistry metricsRegistry = taskContext.getTaskMetricsRegistry(); this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opId + "-messages"); this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-message-ns"); this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-timer-ns"); this.taskName = taskContext.getTaskModel().getTaskName(); this.eosStates = (EndOfStreamStates) taskContext.fetchObject(EndOfStreamStates.class.getName()); this.watermarkStates = (WatermarkStates) taskContext.fetchObject(WatermarkStates.class.getName()); this.controlMessageSender = new ControlMessageSender(taskContext.getStreamMetadataCache()); this.taskModel = taskContext.getTaskModel(); this.callbackScheduler = taskContext.getCallbackScheduler(); handleInit(context);
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP; taskContext.registerObject(EndOfStreamStates.class.getName(), new EndOfStreamStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts)); taskContext.registerObject(WatermarkStates.class.getName(), new WatermarkStates(taskContext.getTaskModel().getSystemStreamPartitions(), producerTaskCounts, context.getContainerContext().getContainerMetricsRegistry()));