private void flush(Batch batch) throws IOException { if (batch == null) { return; } batch.seal(); try { // This check doesn't always awaits for this exact batch to be emitted, because another batch could be dropped // from the queue ahead of this one, in limitBuffersToEmitSize(). But there is no better way currently to wait for // the exact batch, and it's not that important. emittedBatchCounter.awaitCount(batch.batchNumber, config.getFlushTimeOut(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { String message = StringUtils.format("Timed out after [%d] millis during flushing", config.getFlushTimeOut()); throw new IOException(message, e); } catch (InterruptedException e) { log.debug("Thread Interrupted"); Thread.currentThread().interrupt(); throw new IOException("Thread Interrupted while flushing", e); } }
@Test public void testSettingEverything() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.batchingStrategy", "NEWLINES"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushTimeOut", "1000"); final ObjectMapper objectMapper = new ObjectMapper(); final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); } }
@Test public void testSettingEverythingLegacy() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.http.url", "http://example.com/"); props.setProperty("org.apache.druid.java.util.emitter.http.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.http.batchingStrategy", "newlines"); props.setProperty("org.apache.druid.java.util.emitter.http.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.http.flushTimeOut", "1000"); props.setProperty("org.apache.druid.java.util.emitter.http.batchQueueSizeLimit", "2500"); props.setProperty("org.apache.druid.java.util.emitter.http.httpTimeoutAllowanceFactor", "3.0"); props.setProperty("org.apache.druid.java.util.emitter.http.minHttpTimeoutMillis", "100"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); }
@Test public void testSettingEverything() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); props.setProperty("org.apache.druid.java.util.emitter.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.batchingStrategy", "NEWLINES"); props.setProperty("org.apache.druid.java.util.emitter.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.flushTimeOut", "1000"); props.setProperty("org.apache.druid.java.util.emitter.batchQueueSizeLimit", "2500"); props.setProperty("org.apache.druid.java.util.emitter.httpTimeoutAllowanceFactor", "3.0"); props.setProperty("org.apache.druid.java.util.emitter.minHttpTimeoutMillis", "100"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue( Emitters.makeCustomFactoryMap(props), HttpEmitterConfig.class ); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); }
@Test public void testDefaults() { final Properties props = new Properties(); final ObjectMapper objectMapper = new ObjectMapper(); final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(500, config.getFlushCount()); Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); }
@Test public void testDefaultsLegacy() { final Properties props = new Properties(); props.put("org.apache.druid.java.util.emitter.http.url", "http://example.com/"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(300, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); }
@Test public void testDefaults() { final Properties props = new Properties(); props.put("org.apache.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue( Emitters.makeCustomFactoryMap(props), HttpEmitterConfig.class ); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(500, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); }
private void flush(Batch batch) throws IOException { if (batch == null) { return; } batch.seal(); try { // This check doesn't always awaits for this exact batch to be emitted, because another batch could be dropped // from the queue ahead of this one, in limitBuffersToEmitSize(). But there is no better way currently to wait for // the exact batch, and it's not that important. emittedBatchCounter.awaitCount(batch.batchNumber, config.getFlushTimeOut(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { String message = StringUtils.format("Timed out after [%d] millis during flushing", config.getFlushTimeOut()); throw new IOException(message, e); } catch (InterruptedException e) { log.debug("Thread Interrupted"); Thread.currentThread().interrupt(); throw new IOException("Thread Interrupted while flushing", e); } }