(firstRetryDelay * Math.pow(retryBackoff, retryCount) * 1000); this.retryCount = retryCount; this.queue = new SinkQueue<MetricsBuffer>(checkArg(queueCapacity, queueCapacity > 0, "queue capacity")); latency = registry.newRate("Sink_"+ name, "Sink end to end latency", false);
while (!stopping) { try { queue.consumeAll(this); refreshQueueSizeGauge(); retryDelay = firstRetryDelay; "suppressing further error messages", e); queue.clear(); refreshQueueSizeGauge(); inError = true; // Don't keep complaining ad infinitum
boolean putMetrics(MetricsBuffer buffer, long logicalTimeMs) { if (logicalTimeMs % periodMs == 0) { LOG.debug("enqueue, logicalTime="+ logicalTimeMs); if (queue.enqueue(buffer)) { refreshQueueSizeGauge(); return true; } dropped.incr(); return false; } return true; // OK }
private synchronized T waitForData() throws InterruptedException { checkConsumer(); while (0 == size) { wait(); } setConsumerLock(); return front(); }
/** * Dequeue one element from head of the queue, will block if queue is empty * @return the first element * @throws InterruptedException */ synchronized T dequeue() throws InterruptedException { checkConsumer(); while (0 == size) { wait(); } return _dequeue(); }
/** * Consume all the elements, will block if queue is empty * @param consumer the consumer callback object * @throws InterruptedException */ void consumeAll(Consumer<T> consumer) throws InterruptedException { waitForData(); try { for (int i = size(); i-- > 0; ) { consumer.consume(front()); // can take forever _dequeue(); } } finally { clearConsumerLock(); } }
/** * Test nonblocking enqueue when queue is full * @throws Exception */ @Test public void testFull() throws Exception { final SinkQueue<Integer> q = new SinkQueue<Integer>(1); q.enqueue(1); assertTrue("should drop", !q.enqueue(2)); assertEquals("element", 1, (int) q.dequeue()); q.enqueue(3); q.consume(new Consumer<Integer>() { @Override public void consume(Integer e) { assertEquals("element", 3, (int) e); } }); assertEquals("queue size", 0, q.size()); }
/** * Test common use case * @throws Exception */ @Test public void testCommon() throws Exception { final SinkQueue<Integer> q = new SinkQueue<Integer>(2); q.enqueue(1); assertEquals("queue front", 1, (int) q.front()); assertEquals("queue back", 1, (int) q.back()); assertEquals("element", 1, (int) q.dequeue()); assertTrue("should enqueue", q.enqueue(2)); q.consume(new Consumer<Integer>() { @Override public void consume(Integer e) { assertEquals("element", 2, (int) e); } }); assertTrue("should enqueue", q.enqueue(3)); assertEquals("element", 3, (int) q.dequeue()); assertEquals("queue size", 0, q.size()); assertEquals("queue front", null, q.front()); assertEquals("queue back", null, q.back()); }
/** * Test consumers that take their time. * @throws Exception */ @Test public void testHangingConsumer() throws Exception { SinkQueue<Integer> q = newSleepingConsumerQueue(2, 1, 2); assertEquals("queue back", 2, (int) q.back()); assertTrue("should drop", !q.enqueue(3)); // should not block assertEquals("queue size", 2, q.size()); assertEquals("queue head", 1, (int) q.front()); assertEquals("queue back", 2, (int) q.back()); }
/** * Consume one element, will block if queue is empty * Only one consumer at a time is allowed * @param consumer the consumer callback object */ void consume(Consumer<T> consumer) throws InterruptedException { T e = waitForData(); try { consumer.consume(e); // can take forever _dequeue(); } finally { clearConsumerLock(); } }
/** * Test the consumeAll method * @throws Exception */ @Test public void testConsumeAll() throws Exception { final int capacity = 64; // arbitrary final SinkQueue<Integer> q = new SinkQueue<Integer>(capacity); for (int i = 0; i < capacity; ++i) { assertTrue("should enqueue", q.enqueue(i)); } assertTrue("should not enqueue", !q.enqueue(capacity)); final Runnable trigger = mock(Runnable.class); q.consumeAll(new Consumer<Integer>() { private int expected = 0; @Override public void consume(Integer e) { assertEquals("element", expected++, (int) e); trigger.run(); } }); verify(trigger, times(capacity)).run(); }
private void testEmptyBlocking(int awhile) throws Exception { final SinkQueue<Integer> q = new SinkQueue<Integer>(2); final Runnable trigger = mock(Runnable.class); // try consuming emtpy equeue and blocking Thread t = new Thread() { @Override public void run() { try { assertEquals("element", 1, (int) q.dequeue()); q.consume(new Consumer<Integer>() { @Override public void consume(Integer e) { assertEquals("element", 2, (int) e); trigger.run(); } }); } catch (InterruptedException e) { LOG.warn("Interrupted", e); } } }; t.start(); // Should work with or without sleep if (awhile > 0) { Thread.sleep(awhile); } q.enqueue(1); q.enqueue(2); t.join(); verify(trigger).run(); }
private void refreshQueueSizeGauge() { qsize.set(queue.size()); }
synchronized void clear() { checkConsumer(); for (int i = data.length; i-- > 0; ) { data[i] = null; } size = 0; }
@Override public void run() throws Exception { q.consumeAll(null); } });
@Override public void run() { q.clear(); } });
/** * Consume all the elements, will block if queue is empty * @param consumer the consumer callback object * @throws InterruptedException */ void consumeAll(Consumer<T> consumer) throws InterruptedException { waitForData(); try { for (int i = size(); i-- > 0; ) { consumer.consume(front()); // can take forever _dequeue(); } } finally { clearConsumerLock(); } }
/** * Test nonblocking enqueue when queue is full * @throws Exception */ @Test public void testFull() throws Exception { final SinkQueue<Integer> q = new SinkQueue<Integer>(1); q.enqueue(1); assertTrue("should drop", !q.enqueue(2)); assertEquals("element", 1, (int) q.dequeue()); q.enqueue(3); q.consume(new Consumer<Integer>() { @Override public void consume(Integer e) { assertEquals("element", 3, (int) e); } }); assertEquals("queue size", 0, q.size()); }
/** * Test common use case * @throws Exception */ @Test public void testCommon() throws Exception { final SinkQueue<Integer> q = new SinkQueue<Integer>(2); q.enqueue(1); assertEquals("queue front", 1, (int) q.front()); assertEquals("queue back", 1, (int) q.back()); assertEquals("element", 1, (int) q.dequeue()); assertTrue("should enqueue", q.enqueue(2)); q.consume(new Consumer<Integer>() { @Override public void consume(Integer e) { assertEquals("element", 2, (int) e); } }); assertTrue("should enqueue", q.enqueue(3)); assertEquals("element", 3, (int) q.dequeue()); assertEquals("queue size", 0, q.size()); assertEquals("queue front", null, q.front()); assertEquals("queue back", null, q.back()); }