@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
BroadcastOperatorImpl(BroadcastOperatorSpec<M> broadcastOpSpec, SystemStream systemStream, Context context) { this.broadcastOpSpec = broadcastOpSpec; this.systemStream = systemStream; this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); }
BroadcastOperatorImpl(BroadcastOperatorSpec<M> broadcastOpSpec, SystemStream systemStream, Context context) { this.broadcastOpSpec = broadcastOpSpec; this.systemStream = systemStream; this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); }
BroadcastOperatorImpl(BroadcastOperatorSpec<M> broadcastOpSpec, SystemStream systemStream, Context context) { this.broadcastOpSpec = broadcastOpSpec; this.systemStream = systemStream; this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); }
BroadcastOperatorImpl(BroadcastOperatorSpec<M> broadcastOpSpec, SystemStream systemStream, Context context) { this.broadcastOpSpec = broadcastOpSpec; this.systemStream = systemStream; this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); }
BroadcastOperatorImpl(BroadcastOperatorSpec<M> broadcastOpSpec, SystemStream systemStream, Context context) { this.broadcastOpSpec = broadcastOpSpec; this.systemStream = systemStream; this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); }
static void initRateLimiter(RateLimiter rateLimiter) { Map<TaskName, TaskModel> tasks = IntStream.range(0, NUMBER_OF_TASKS) .mapToObj(i -> new TaskName("task-" + i)) .collect(Collectors.toMap(Function.identity(), x -> mock(TaskModel.class))); ContainerModel containerModel = mock(ContainerModel.class); when(containerModel.getTasks()).thenReturn(tasks); JobModel jobModel = mock(JobModel.class); Map<String, ContainerModel> containerModelMap = new HashMap<>(); containerModelMap.put("container-1", containerModel); when(jobModel.getContainers()).thenReturn(containerModelMap); Context context = mock(Context.class); TaskContextImpl taskContext = mock(TaskContextImpl.class); when(context.getTaskContext()).thenReturn(taskContext); when(taskContext.getJobModel()).thenReturn(jobModel); when(context.getTaskContext().getTaskModel()).thenReturn(mock(TaskModel.class)); rateLimiter.init(context); } }
@Override public void init(Context context) { TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); if (perTaskFunctionMap.get(taskName) == null) { perTaskFunctionMap.put(taskName, new HashMap<String, BaseTestFunction>() { { this.put(opId, BaseTestFunction.this); } }); } else { if (perTaskFunctionMap.get(taskName).containsKey(opId)) { throw new IllegalStateException(String.format("Multiple init called for op %s in the same task instance %s", opId, this.taskName.getTaskName())); } perTaskFunctionMap.get(taskName).put(opId, this); } if (perTaskInitList.get(taskName) == null) { perTaskInitList.put(taskName, new ArrayList<String>() { { this.add(opId); } }); } else { perTaskInitList.get(taskName).add(opId); } this.taskName = taskName; this.numInitCalled++; } }
@Before public void setup() { this.context = new MockContext(); // individual tests can override this config if necessary when(this.context.getJobContext().getConfig()).thenReturn(mock(Config.class)); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getTaskName()).thenReturn(new TaskName("task 0")); when(this.context.getTaskContext().getTaskModel()).thenReturn(taskModel); when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); }
@Before public void setup() { this.context = new MockContext(); when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getTaskContext().getTaskModel()).thenReturn(mock(TaskModel.class)); when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, SystemStream systemStream, Context context) { this.partitionByOpSpec = partitionByOpSpec; this.systemStream = systemStream; this.keyFunction = partitionByOpSpec.getKeyFunction(); this.valueFunction = partitionByOpSpec.getValueFunction(); this.taskName = context.getTaskContext().getTaskModel().getTaskName().getTaskName(); StreamMetadataCache streamMetadataCache = ((TaskContextImpl) context.getTaskContext()).getStreamMetadataCache(); this.controlMessageSender = new ControlMessageSender(streamMetadataCache); }
@Before public void setup() { Map<String, String> configMap = new HashMap<>(); configMap.put("job.default.system", "kafka"); configMap.put("job.name", "jobName"); configMap.put("job.id", "jobId"); this.config = new MapConfig(configMap); this.context = new MockContext(); when(this.context.getJobContext().getConfig()).thenReturn(this.config); Serde storeKeySerde = new TimeSeriesKeySerde(new IntegerSerde()); Serde storeValSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getSystemStreamPartitions()).thenReturn(ImmutableSet .of(new SystemStreamPartition("kafka", "integTestExecutionPlannerers", new Partition(0)))); when(taskModel.getTaskName()).thenReturn(new TaskName("task 1")); when(this.context.getTaskContext().getTaskModel()).thenReturn(taskModel); when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getTaskContext().getStore("jobName-jobId-window-w1")) .thenReturn(new TestInMemoryStore<>(storeKeySerde, storeValSerde)); }