public boolean refreshBpTaskList() { boolean changed = false; LOG.debug("Running Back Pressure status change check"); for (Entry<Integer, BackpressureState> entry : tasks.entrySet()) { BackpressureState state = entry.getValue(); if (state.backpressure.get() && state.queue.isEmptyOverflow()) { recordNoBackPressure(entry.getKey()); changed = true; } } return changed; }
private void transferLocalBatch(ArrayList<AddressedTuple> tupleBatch) { int lastOverflowCount = 0; // overflowQ size at the time the last BPStatus was sent for (int i = 0; i < tupleBatch.size(); i++) { AddressedTuple tuple = tupleBatch.get(i); JCQueue queue = taskToExecutorQueue.get(tuple.dest); // 1- try adding to main queue if its overflow is not empty if (queue.isEmptyOverflow()) { if (queue.tryPublish(tuple)) { continue; } } // 2- BP detected (i.e MainQ is full). So try adding to overflow int currOverflowCount = queue.getOverflowCount(); if (bpTracker.recordBackPressure(tuple.dest)) { receiver.sendBackPressureStatus(bpTracker.getCurrStatus()); lastOverflowCount = currOverflowCount; } else { if (currOverflowCount - lastOverflowCount > 10000) { // resend BP status, in case prev notification was missed or reordered BackPressureStatus bpStatus = bpTracker.getCurrStatus(); receiver.sendBackPressureStatus(bpStatus); lastOverflowCount = currOverflowCount; LOG.debug("Re-sent BackPressure Status. OverflowCount = {}, BP Status ID = {}. ", currOverflowCount, bpStatus.id); } } if (!queue.tryPublishToOverflow(tuple)) { dropMessage(tuple, queue); } } }
@Test public void testRefreshBackPressureWithNonEmptyOverflow() { int taskId = 1; JCQueue queue = mock(JCQueue.class); when(queue.isEmptyOverflow()).thenReturn(false); BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of( taskId, queue)); tracker.recordBackPressure(taskId); boolean backpressureChanged = tracker.refreshBpTaskList(); BackPressureStatus status = tracker.getCurrStatus(); assertThat(backpressureChanged, is(false)); assertThat(status.workerId, is(WORKER_ID)); assertThat(status.bpTasks, contains(taskId)); }
@Test public void testRefreshBackpressureWithEmptyOverflow() { int taskId = 1; JCQueue queue = mock(JCQueue.class); when(queue.isEmptyOverflow()).thenReturn(true); BackPressureTracker tracker = new BackPressureTracker(WORKER_ID, ImmutableMap.of( taskId, queue)); tracker.recordBackPressure(taskId); boolean backpressureChanged = tracker.refreshBpTaskList(); BackPressureStatus status = tracker.getCurrStatus(); assertThat(backpressureChanged, is(true)); assertThat(status.workerId, is(WORKER_ID)); assertThat(status.nonBpTasks, contains(taskId)); }