public int getNumThreads() { int numThreadsConfigured = getNumThreadsConfigured(); if (numThreadsConfigured != DEFAULT_NUM_THREADS) { return numThreadsConfigured; } else { return Math.max(JvmUtils.getRuntimeInfo().getAvailableProcessors() - 1, 1); } }
public enum CompressionCodec { IDENTITY { @Override public String getEncodingString() { return "identity"; } }, GZIP { @Override public String getEncodingString() { return "gzip"; } }, @SuppressWarnings("unused") //TODO test this CompressionCodec (it counts as usage) DEFLATE { @Override public String getEncodingString() { return "deflate"; } }; /** * Get the header-ified name of this encoding, which should go in "Accept-Encoding" and * "Content-Encoding" headers. This is not just the lowercasing of the enum name, since * we may one day support x- encodings like LZ4, which would likely be an enum named * "LZ4" that has an encoding string like "x-lz4".
JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getRuntimeInfo().getTotalHeapSizeBytes(), JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes(), JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes() );
public Status(Collection<DruidModule> modules) { this.version = getDruidVersion(); this.modules = getExtensionVersions(modules); this.memory = new Memory(JvmUtils.getRuntimeInfo()); }
public int intermediateComputeSizeBytes() { int sizeBytesConfigured = intermediateComputeSizeBytesConfigured(); if (sizeBytesConfigured != DEFAULT_PROCESSING_BUFFER_SIZE_BYTES) { return sizeBytesConfigured; } else if (computedBufferSizeBytes.get() != null) { return computedBufferSizeBytes.get(); } long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); int numProcessingThreads = getNumThreads(); int numMergeBuffers = getNumMergeBuffers(); int totalNumBuffers = numMergeBuffers + numProcessingThreads; int sizePerBuffer = (int) ((double) directSizeBytes / (double) (totalNumBuffers + 1)); final int computedSizePerBuffer = Math.min(sizePerBuffer, MAX_DEFAULT_PROCESSING_BUFFER_SIZE_BYTES); if (computedBufferSizeBytes.compareAndSet(null, computedSizePerBuffer)) { log.info( "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers " + "out of [%,d] max direct memory", computedSizePerBuffer, numProcessingThreads, numMergeBuffers, directSizeBytes ); } return computedSizePerBuffer; }
public static CaffeineCache create(final CaffeineCacheConfig config, final Executor executor) { Caffeine<Object, Object> builder = Caffeine.newBuilder().recordStats(); if (config.getExpireAfter() >= 0) { builder .expireAfterAccess(config.getExpireAfter(), TimeUnit.MILLISECONDS); } if (config.getSizeInBytes() >= 0) { builder.maximumWeight(config.getSizeInBytes()); } else { builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 10)); } builder .weigher((NamedKey key, byte[] value) -> value.length + key.key.length + key.namespace.length() * Character.BYTES + FIXED_COST) .executor(executor); return new CaffeineCache(builder.build(), config); }
private void verifyDirectMemory(DruidProcessingConfig config) { try { final long maxDirectMemory = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumMergeBuffers() + config.getNumThreads() + 1); if (maxDirectMemory < memoryNeeded) { throw new ProvisionException( StringUtils.format( "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize, druid.processing.buffer.sizeBytes, druid.processing.numThreads, or druid.processing.numMergeBuffers: " + "maxDirectMemory[%,d], memoryNeeded[%,d] = druid.processing.buffer.sizeBytes[%,d] * (druid.processing.numMergeBuffers[%,d] + druid.processing.numThreads[%,d] + 1)", maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumMergeBuffers(), config.getNumThreads() ) ); } } catch (UnsupportedOperationException e) { log.info( "Could not verify that you have enough direct memory, so I hope you do! Error message was: %s", e.getMessage() ); } } }
@Test public void testgetMaxDirectMemory() { try { long maxMemory = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); Assert.assertTrue((maxMemory > 0)); } catch (UnsupportedOperationException expected) { Assert.assertTrue(true); } catch (RuntimeException expected) { Assert.assertTrue(true); } } }