@Override public List<String> pop(String queueName, int count, int timeout) { List<Message> msg = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msg.stream() .map(Message::getId) .collect(Collectors.toList()); }
@Override public List<com.netflix.conductor.core.events.queue.Message> pollMessages(String queueName, int count, int timeout) { List<Message> msgs = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msgs.stream() .map(msg -> new com.netflix.conductor.core.events.queue.Message(msg.getId(), msg.getPayload(), null)) .collect(Collectors.toList()); }
@Override public List<String> pop(String queueName, int count, int timeout) { List<Message> msg = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msg.stream() .map(Message::getId) .collect(Collectors.toList()); }
@Override public List<com.netflix.conductor.core.events.queue.Message> pollMessages(String queueName, int count, int timeout) { List<Message> msgs = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msgs.stream() .map(msg -> new com.netflix.conductor.core.events.queue.Message(msg.getId(), msg.getPayload(), null)) .collect(Collectors.toList()); }
@Override public void run() { if (consumed.get() >= count) { return; } List<Message> popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); allMsgs.addAll(popped); consumed.addAndGet(popped.size()); popped.stream().forEach(p -> latch.countDown()); counter.incrementAndGet(); } };
public void consume() { try { Set<String> ids = new HashSet<>(); long s = System.currentTimeMillis(); int loopCount = 100; int batchSize = 3500; int count = 0; for (int i = 0; i < loopCount; i++) { List<Message> popped = queue.pop(batchSize, 1, TimeUnit.MILLISECONDS); queue.ack(popped); Set<String> poppedIds = popped.stream().map(Message::getId).collect(Collectors.toSet()); if (popped.size() != poppedIds.size()) { //We consumed dups throw new RuntimeException("Count does not match. expected: " + popped.size() + ", but actual was : " + poppedIds.size() + ", i: " + i); } ids.addAll(poppedIds); count += popped.size(); } long e = System.currentTimeMillis(); long diff = e - s; long throughput = 1000 * ((count) / diff); if (count != ids.size()) { //We consumed dups throw new RuntimeException("There were duplicate messages consumed... expected messages to be consumed " + count + ", but actual was : " + ids.size()); } System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, messages read: " + count); } catch (Exception e) { e.printStackTrace(); } }
List<Message> more = rdq.pop(1, 1, TimeUnit.SECONDS); assertEquals(0, more.size());
List<Message> popped = queue.pop(1, 1, TimeUnit.SECONDS); popped = queue.pop(2, 1, TimeUnit.SECONDS); elapsedTime = System.currentTimeMillis() - start; assertEquals(2, popped.size()); popped = queue.pop(5, 5, TimeUnit.SECONDS); elapsedTime = System.currentTimeMillis() - start; assertEquals(3, popped.size()); //3 remaining in the current shard
assertEquals(messages, messages2); List<Message> poped = rdq.pop(count, 1, TimeUnit.SECONDS); assertNotNull(poped); assertEquals(count, poped.size()); List<Message> messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); if (messages3.size() < count) { List<Message> messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); messages3.addAll(messages4); messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); assertNotNull(messages3); assertEquals(0, messages3.size());