private void transferLocalBatch(ArrayList<AddressedTuple> tupleBatch) { int lastOverflowCount = 0; // overflowQ size at the time the last BPStatus was sent for (int i = 0; i < tupleBatch.size(); i++) { AddressedTuple tuple = tupleBatch.get(i); JCQueue queue = taskToExecutorQueue.get(tuple.dest); // 1- try adding to main queue if its overflow is not empty if (queue.isEmptyOverflow()) { if (queue.tryPublish(tuple)) { continue; } } // 2- BP detected (i.e MainQ is full). So try adding to overflow int currOverflowCount = queue.getOverflowCount(); if (bpTracker.recordBackPressure(tuple.dest)) { receiver.sendBackPressureStatus(bpTracker.getCurrStatus()); lastOverflowCount = currOverflowCount; } else { if (currOverflowCount - lastOverflowCount > 10000) { // resend BP status, in case prev notification was missed or reordered BackPressureStatus bpStatus = bpTracker.getCurrStatus(); receiver.sendBackPressureStatus(bpStatus); lastOverflowCount = currOverflowCount; LOG.debug("Re-sent BackPressure Status. OverflowCount = {}, BP Status ID = {}. ", currOverflowCount, bpStatus.id); } } if (!queue.tryPublishToOverflow(tuple)) { dropMessage(tuple, queue); } } }
@Test public void testNoReOrderingUnderBackPressure() throws Exception { final int MESSAGES = 100; final int CAPACITY = 64; final JCQueue queue = createQueue("testBackPressure", CAPACITY); for (int i = 0; i < MESSAGES; i++) { if (!queue.tryPublish(i)) { Assert.assertTrue(queue.tryPublishToOverflow(i)); } } TestConsumer consumer = new TestConsumer(); queue.consume(consumer); Assert.assertEquals(MESSAGES, consumer.lastMsg); queue.close(); }