try { return innerLifecycle.addMaybeStartManagedInstance( new HttpPostEmitter( config.buildHttpEmitterConfig(u.toString()), client, emitter.emit(event);
@Override public void emit(Event event) { emitAndReturnBatch(event); }
@Override public void flush() throws IOException { awaitStarted(); Object batchObj = concurrentBatch.get(); if (batchObj instanceof Batch) { flush((Batch) batchObj); } }
@Override public void run() { for (int i = 0; i < 1000; i++) { emitter.emit(bigEvent); Assert.assertTrue(emitter.getTotalFailedBuffers() <= 10); Assert.assertTrue(emitter.getBuffersToEmit() <= 12); } threadsCompleted.countDown(); } }.start();
@Override public boolean doMonitor(ServiceEmitter emitter) { emitEmittedEvents(emitter); emitDroppedBuffers(emitter); emitAllocatedBuffers(emitter); emitFailedBuffers(emitter); emitTimeCounterMetrics(emitter, httpPostEmitter.getBatchFillingTimeCounter(), "emitter/batchFilling/"); emitTimeCounterMetrics(emitter, httpPostEmitter.getSuccessfulSendingTimeCounter(), "emitter/successfulSending/"); emitTimeCounterMetrics(emitter, httpPostEmitter.getFailedSendingTimeCounter(), "emitter/failedSending/"); emitter.emit(builder.build("emitter/events/emitQueue", httpPostEmitter.getEventsToEmit())); emitter.emit(builder.build("emitter/events/large/emitQueue", httpPostEmitter.getLargeEventsToEmit())); emitter.emit(builder.build("emitter/buffers/emitQueue", httpPostEmitter.getBuffersToEmit())); emitter.emit(builder.build("emitter/buffers/reuseQueue", httpPostEmitter.getBuffersToReuse())); return true; }
@Override public boolean doMonitor(ServiceEmitter emitter) { long newTotalEmittedEvents = httpPostEmitter.getTotalEmittedEvents(); long totalEmittedEventsDiff = newTotalEmittedEvents - lastTotalEmittedEvents; emitter.emit(builder.build("emitter/events/emitted", totalEmittedEventsDiff)); lastTotalEmittedEvents = newTotalEmittedEvents; int newDroppedBuffers = httpPostEmitter.getDroppedBuffers(); int droppedBuffersDiff = newDroppedBuffers - lastDroppedBuffers; emitter.emit(builder.build("emitter/buffers/dropped", droppedBuffersDiff)); lastDroppedBuffers = newDroppedBuffers; emitTimeCounterMetrics(emitter, httpPostEmitter.getBatchFillingTimeCounter(), "emitter/batchFilling/"); emitTimeCounterMetrics(emitter, httpPostEmitter.getSuccessfulSendingTimeCounter(), "emitter/successfulSending/"); emitTimeCounterMetrics(emitter, httpPostEmitter.getFailedSendingTimeCounter(), "emitter/failedSending/"); emitter.emit(builder.build("emitter/events/emitQueue", httpPostEmitter.getEventsToEmit())); emitter.emit(builder.build("emitter/events/large/emitQueue", httpPostEmitter.getLargeEventsToEmit())); emitter.emit(builder.build("emitter/buffers/totalAllocated", httpPostEmitter.getTotalAllocatedBuffers())); emitter.emit(builder.build("emitter/buffers/emitQueue", httpPostEmitter.getBuffersToEmit())); emitter.emit(builder.build("emitter/buffers/failed", httpPostEmitter.getFailedBuffers())); emitter.emit(builder.build("emitter/buffers/reuseQueue", httpPostEmitter.getBuffersToReuse())); return true; }
final UnitEvent event2 = new UnitEvent("test", 2); emitter = sizeBasedEmitter(1); Assert.assertEquals(0, emitter.getTotalEmittedEvents()); Assert.assertEquals(0, emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount()); Assert.assertEquals(0, emitter.getFailedSendingTimeCounter().getTimeSumAndCount()); emitter.emit(event1); emitter.flush(); waitForEmission(emitter, 1); Assert.assertTrue(httpClient.succeeded()); Assert.assertEquals(0, emitter.getTotalEmittedEvents()); Assert.assertEquals(0, emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount()); Assert.assertTrue(emitter.getFailedSendingTimeCounter().getTimeSumAndCount() > 0); ); emitter.emit(event2); emitter.flush(); waitForEmission(emitter, 2); closeNoFlush(emitter); emitter.joinEmitterThread(); Assert.assertEquals(2, emitter.getTotalEmittedEvents()); Assert.assertTrue(emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount() > 0); Assert.assertTrue(emitter.getFailedSendingTimeCounter().getTimeSumAndCount() > 0);
.setBatchQueueSizeLimit(10) .build(); final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, new ObjectMapper()); emitter.start(); emitter.emit(bigEvent); Assert.assertTrue(emitter.getLargeEventsToEmit() <= 11); emitter.flush();
@Test public void timeoutEmptyQueue() throws IOException, InterruptedException { float timeoutAllowanceFactor = 2.0f; final HttpEmitterConfig config = new HttpEmitterConfig.Builder("http://foo.bar") .setBatchingStrategy(BatchingStrategy.ONLY_EVENTS) .setHttpTimeoutAllowanceFactor(timeoutAllowanceFactor) .build(); final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, objectMapper); long startMs = System.currentTimeMillis(); emitter.start(); emitter.emitAndReturnBatch(new IntEvent()); emitter.flush(); long fillTimeMs = System.currentTimeMillis() - startMs; Assert.assertThat((double) timeoutUsed.get(), Matchers.lessThan(fillTimeMs * (timeoutAllowanceFactor + 0.5))); startMs = System.currentTimeMillis(); final Batch batch = emitter.emitAndReturnBatch(new IntEvent()); Thread.sleep(1000); batch.seal(); emitter.flush(); fillTimeMs = System.currentTimeMillis() - startMs; Assert.assertThat((double) timeoutUsed.get(), Matchers.lessThan(fillTimeMs * (timeoutAllowanceFactor + 0.5))); } }
final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, objectMapper); int nThreads = Runtime.getRuntime().availableProcessors() * 2; final List<IntList> eventsPerThread = new ArrayList<>(nThreads); emitter.start(); final CountDownLatch threadsCompleted = new CountDownLatch(nThreads); for (int i = 0; i < nThreads; i++) { emitter.flush(); System.out.println("Allocated buffers: " + emitter.getTotalAllocatedBuffers()); for (int eventIndex = 0; eventIndex < N; eventIndex++) { if (!emittedEvents.get(eventIndex)) {
private HttpPostEmitter timeBasedEmitter(long timeInMillis) { HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) .setFlushMillis(timeInMillis) .setFlushCount(Integer.MAX_VALUE) .build(); HttpPostEmitter emitter = new HttpPostEmitter( config, httpClient, jsonMapper ); emitter.start(); return emitter; }
.setBatchQueueSizeLimit(10) .build(); final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, new ObjectMapper()); emitter.start(); emitter.flush();
@Override public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) { Emitter retVal = new HttpPostEmitter(this, httpClient, objectMapper); lifecycle.addManagedInstance(retVal); return retVal; } }
@Test public void testSizeBasedEmission() throws Exception { emitter = sizeBasedEmitter(3); httpClient.setGoHandler(GoHandlers.failingHandler()); emitter.emit(new UnitEvent("test", 1)); emitter.emit(new UnitEvent("test", 2)); httpClient.setGoHandler(GoHandlers.passingHandler(okResponse()).times(1)); emitter.emit(new UnitEvent("test", 3)); waitForEmission(emitter, 1); httpClient.setGoHandler(GoHandlers.failingHandler()); emitter.emit(new UnitEvent("test", 4)); emitter.emit(new UnitEvent("test", 5)); closeAndExpectFlush(emitter); Assert.assertTrue(httpClient.succeeded()); }
@Override public void flush() { Exception thrown = null; for (HttpPostEmitter httpPostEmitter : emitters.values()) { try { httpPostEmitter.flush(); } catch (Exception e) { // If flush was interrupted, exit the loop if (Thread.currentThread().isInterrupted()) { if (thrown != null) { e.addSuppressed(thrown); } throw Throwables.propagate(e); } if (thrown == null) { thrown = e; } else { if (thrown != e) { thrown.addSuppressed(e); } } } } if (thrown != null) { throw Throwables.propagate(thrown); } }
private void emitAllocatedBuffers(ServiceEmitter emitter) { int newTotalAllocatedBuffers = httpPostEmitter.getTotalAllocatedBuffers(); int allocatedBuffersDelta = newTotalAllocatedBuffers - lastTotalAllocatedBuffers; emitter.emit(builder.build("emitter/buffers/allocated/delta", allocatedBuffersDelta)); lastTotalAllocatedBuffers = newTotalAllocatedBuffers; }
private void emitEmittedEvents(ServiceEmitter emitter) { long newTotalEmittedEvents = httpPostEmitter.getTotalEmittedEvents(); long emittedEventsDelta = newTotalEmittedEvents - lastTotalEmittedEvents; emitter.emit(builder.build("emitter/events/emitted/delta", emittedEventsDelta)); lastTotalEmittedEvents = newTotalEmittedEvents; }
final AtomicInteger counter = new AtomicInteger(); emitter = manualFlushEmitterWithBatchSize(1024 * 1024); Assert.assertEquals(0, emitter.getTotalEmittedEvents()); Assert.assertEquals(0, emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount()); Assert.assertEquals(0, emitter.getFailedSendingTimeCounter().getTimeSumAndCount()); emitter.emit(event); Assert.assertEquals(2, emitter.getTotalEmittedEvents()); Assert.assertTrue(emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount() > 0); Assert.assertEquals(0, emitter.getFailedSendingTimeCounter().getTimeSumAndCount()); emitter.flush(); waitForEmission(emitter, 2); Assert.assertEquals(4, emitter.getTotalEmittedEvents()); Assert.assertTrue(emitter.getSuccessfulSendingTimeCounter().getTimeSumAndCount() > 0); Assert.assertEquals(0, emitter.getFailedSendingTimeCounter().getTimeSumAndCount()); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded());
private HttpPostEmitter sizeBasedEmitter(int size) { HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) .setFlushMillis(Long.MAX_VALUE) .setFlushCount(size) .build(); HttpPostEmitter emitter = new HttpPostEmitter( config, httpClient, jsonMapper ); emitter.start(); return emitter; }