public int getNumThreads() { int numThreadsConfigured = getNumThreadsConfigured(); if (numThreadsConfigured != DEFAULT_NUM_THREADS) { return numThreadsConfigured; } else { return Math.max(JvmUtils.getRuntimeInfo().getAvailableProcessors() - 1, 1); } }
public static long safeGetThreadCpuTime() { if (!isThreadCpuTimeEnabled()) { return 0L; } else { return getCurrentThreadCpuTime(); } }
public static <T> QueryRunner<T> safeBuild( QueryRunner<T> delegate, QueryToolChest<T, ? extends Query<T>> queryToolChest, ServiceEmitter emitter, AtomicLong accumulator, boolean report ) { if (!JvmUtils.isThreadCpuTimeEnabled()) { return delegate; } else { return new CPUTimeMetricQueryRunner<>(delegate, queryToolChest, emitter, accumulator, report); } } }
@Override public <RetType> RetType wrap(Supplier<RetType> sequenceProcessing) { final long start = JvmUtils.getCurrentThreadCpuTime(); try { return sequenceProcessing.get(); } finally { cpuTimeAccumulator.addAndGet(JvmUtils.getCurrentThreadCpuTime() - start); } }
long persistThreadCpuTime = JvmUtils.safeGetThreadCpuTime(); try { for (Pair<FireHydrant, Interval> pair : indexesToPersist) { metrics.incrementPersistCpuTime(JvmUtils.safeGetThreadCpuTime() - persistThreadCpuTime); metrics.incrementNumPersists(); metrics.incrementPersistTimeMillis(persistStopwatch.elapsed(TimeUnit.MILLISECONDS));
private CPUTimeMetricQueryRunner( QueryRunner<T> delegate, QueryToolChest<T, ? extends Query<T>> queryToolChest, ServiceEmitter emitter, AtomicLong cpuTimeAccumulator, boolean report ) { if (!JvmUtils.isThreadCpuTimeEnabled()) { throw new ISE("Cpu time must enabled"); } this.delegate = delegate; this.queryToolChest = queryToolChest; this.emitter = emitter; this.cpuTimeAccumulator = cpuTimeAccumulator == null ? new AtomicLong(0L) : cpuTimeAccumulator; this.report = report; }
final long mergeThreadCpuTime = JvmUtils.safeGetThreadCpuTime(); mergeStopwatch = Stopwatch.createStarted(); metrics.incrementMergeCpuTime(JvmUtils.safeGetThreadCpuTime() - mergeThreadCpuTime); metrics.incrementMergeTimeMillis(mergeStopwatch.elapsed(TimeUnit.MILLISECONDS));
JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getRuntimeInfo().getTotalHeapSizeBytes(), JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes(), JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes() );
public enum CompressionCodec { IDENTITY { @Override public String getEncodingString() { return "identity"; } }, GZIP { @Override public String getEncodingString() { return "gzip"; } }, @SuppressWarnings("unused") //TODO test this CompressionCodec (it counts as usage) DEFLATE { @Override public String getEncodingString() { return "deflate"; } }; /** * Get the header-ified name of this encoding, which should go in "Accept-Encoding" and * "Content-Encoding" headers. This is not just the lowercasing of the enum name, since * we may one day support x- encodings like LZ4, which would likely be an enum named * "LZ4" that has an encoding string like "x-lz4".
public Status(Collection<DruidModule> modules) { this.version = getDruidVersion(); this.modules = getExtensionVersions(modules); this.memory = new Memory(JvmUtils.getRuntimeInfo()); }
public int intermediateComputeSizeBytes() { int sizeBytesConfigured = intermediateComputeSizeBytesConfigured(); if (sizeBytesConfigured != DEFAULT_PROCESSING_BUFFER_SIZE_BYTES) { return sizeBytesConfigured; } else if (computedBufferSizeBytes.get() != null) { return computedBufferSizeBytes.get(); } long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); int numProcessingThreads = getNumThreads(); int numMergeBuffers = getNumMergeBuffers(); int totalNumBuffers = numMergeBuffers + numProcessingThreads; int sizePerBuffer = (int) ((double) directSizeBytes / (double) (totalNumBuffers + 1)); final int computedSizePerBuffer = Math.min(sizePerBuffer, MAX_DEFAULT_PROCESSING_BUFFER_SIZE_BYTES); if (computedBufferSizeBytes.compareAndSet(null, computedSizePerBuffer)) { log.info( "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers " + "out of [%,d] max direct memory", computedSizePerBuffer, numProcessingThreads, numMergeBuffers, directSizeBytes ); } return computedSizePerBuffer; }
public static CaffeineCache create(final CaffeineCacheConfig config, final Executor executor) { Caffeine<Object, Object> builder = Caffeine.newBuilder().recordStats(); if (config.getExpireAfter() >= 0) { builder .expireAfterAccess(config.getExpireAfter(), TimeUnit.MILLISECONDS); } if (config.getSizeInBytes() >= 0) { builder.maximumWeight(config.getSizeInBytes()); } else { builder.maximumWeight(Math.min(MAX_DEFAULT_BYTES, JvmUtils.getRuntimeInfo().getMaxHeapSizeBytes() / 10)); } builder .weigher((NamedKey key, byte[] value) -> value.length + key.key.length + key.namespace.length() * Character.BYTES + FIXED_COST) .executor(executor); return new CaffeineCache(builder.build(), config); }
private void verifyDirectMemory(DruidProcessingConfig config) { try { final long maxDirectMemory = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumMergeBuffers() + config.getNumThreads() + 1); if (maxDirectMemory < memoryNeeded) { throw new ProvisionException( StringUtils.format( "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize, druid.processing.buffer.sizeBytes, druid.processing.numThreads, or druid.processing.numMergeBuffers: " + "maxDirectMemory[%,d], memoryNeeded[%,d] = druid.processing.buffer.sizeBytes[%,d] * (druid.processing.numMergeBuffers[%,d] + druid.processing.numThreads[%,d] + 1)", maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumMergeBuffers(), config.getNumThreads() ) ); } } catch (UnsupportedOperationException e) { log.info( "Could not verify that you have enough direct memory, so I hope you do! Error message was: %s", e.getMessage() ); } } }
@Test public void testgetMaxDirectMemory() { try { long maxMemory = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes(); Assert.assertTrue((maxMemory > 0)); } catch (UnsupportedOperationException expected) { Assert.assertTrue(true); } catch (RuntimeException expected) { Assert.assertTrue(true); } } }