@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
@Override public void init(Context context) { this.tagToRateLimiterMap = Collections.unmodifiableMap(tagToTargetRateMap.entrySet().stream() .map(e -> { String tag = e.getKey(); JobModel jobModel = ((TaskContextImpl) context.getTaskContext()).getJobModel(); int numTasks = jobModel.getContainers().values().stream() .mapToInt(cm -> cm.getTasks().size()) .sum(); int effectiveRate = e.getValue() / numTasks; TaskName taskName = context.getTaskContext().getTaskModel().getTaskName(); LOGGER.info(String.format("Effective rate limit for task %s and tag %s is %d", taskName, tag, effectiveRate)); return new ImmutablePair<>(tag, com.google.common.util.concurrent.RateLimiter.create(effectiveRate)); }) .collect(Collectors.toMap(ImmutablePair::getKey, ImmutablePair::getValue)) ); initialized = true; }
static void initRateLimiter(RateLimiter rateLimiter) { Map<TaskName, TaskModel> tasks = IntStream.range(0, NUMBER_OF_TASKS) .mapToObj(i -> new TaskName("task-" + i)) .collect(Collectors.toMap(Function.identity(), x -> mock(TaskModel.class))); ContainerModel containerModel = mock(ContainerModel.class); when(containerModel.getTasks()).thenReturn(tasks); JobModel jobModel = mock(JobModel.class); Map<String, ContainerModel> containerModelMap = new HashMap<>(); containerModelMap.put("container-1", containerModel); when(jobModel.getContainers()).thenReturn(containerModelMap); Context context = mock(Context.class); TaskContextImpl taskContext = mock(TaskContextImpl.class); when(context.getTaskContext()).thenReturn(taskContext); when(taskContext.getJobModel()).thenReturn(jobModel); when(context.getTaskContext().getTaskModel()).thenReturn(mock(TaskModel.class)); rateLimiter.init(context); } }
private Context createMockContext(TableDescriptor tableDescriptor) { Context context = mock(Context.class); ContainerContext containerContext = mock(ContainerContext.class); when(context.getContainerContext()).thenReturn(containerContext); MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); when(metricsRegistry.newTimer(anyString(), anyString())).thenReturn(mock(Timer.class)); when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(mock(Counter.class)); when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry); TaskContextImpl taskContext = mock(TaskContextImpl.class); when(context.getTaskContext()).thenReturn(taskContext); TaskName taskName = new TaskName("MyTask"); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getTaskName()).thenReturn(taskName); when(context.getTaskContext().getTaskModel()).thenReturn(taskModel); ContainerModel containerModel = mock(ContainerModel.class); when(containerModel.getTasks()).thenReturn(ImmutableMap.of(taskName, taskModel)); when(containerContext.getContainerModel()).thenReturn(containerModel); String containerId = "container-1"; JobModel jobModel = mock(JobModel.class); when(taskContext.getJobModel()).thenReturn(jobModel); when(jobModel.getContainers()).thenReturn(ImmutableMap.of(containerId, containerModel)); JobContext jobContext = mock(JobContext.class); when(jobContext.getConfig()).thenReturn(new MapConfig(tableDescriptor.toConfig(new MapConfig()))); when(context.getJobContext()).thenReturn(jobContext); return context; }
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP;
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP;
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP;
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP;
hasIntermediateStreams(specGraph) ? getProducerTaskCountForIntermediateStreams( getStreamToConsumerTasks(taskContext.getJobModel()), getIntermediateToInputStreamsMap(specGraph, streamConfig)) : Collections.EMPTY_MAP;
when(containerModel.getTasks()).thenReturn(Collections.singletonMap(new TaskName("task 0"), taskModel)); when(taskModel.getSystemStreamPartitions()).thenReturn(Collections.emptySet()); when(((TaskContextImpl) this.context.getTaskContext()).getJobModel()).thenReturn(jobModel); OperatorImplGraph opImplGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class));