@Override public R visitQueryContext(QueryContext queryContext, C visitContext) { return mergeResults(queryContext.acceptChildren(this, visitContext)); }
@Override public synchronized void setResourceOvercommit() { // Allow the query to use the entire pool. This way the worker will kill the query, if it uses the entire local general pool. // The coordinator will kill the query if the cluster runs out of memory. maxUserMemory = memoryPool.getMaxBytes(); maxTotalMemory = memoryPool.getMaxBytes(); }
@Override public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitQueryContext(this, context); }
private synchronized ListenableFuture<?> updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { enforceUserMemoryLimit(queryMemoryContext.getUserMemory(), delta, maxMemory); return memoryPool.reserve(queryId, allocationTag, delta); } memoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
/** * Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory * calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. * * @see this##updateSystemMemory(long) for details. */ private synchronized ListenableFuture<?> updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { enforceUserMemoryLimit(queryMemoryContext.getUserMemory(), delta, maxUserMemory); return memoryPool.reserve(queryId, allocationTag, delta); } memoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
private boolean memoryRevokingNeeded(MemoryPool memoryPool) { return memoryPool.getReservedRevocableBytes() > 0 && memoryPool.getFreeBytes() <= memoryPool.getMaxBytes() * (1.0 - memoryRevokingThreshold); }
private synchronized ListenableFuture<?> updateSystemMemory(String allocationTag, long delta) { if (delta >= 0) { systemMemoryPool.reserve(queryId, allocationTag, delta); // Since various operators and the output buffers now support blocking when the system pool is full // we return NOT_BLOCKED to prevent them from blocking, which is the legacy behavior. return NOT_BLOCKED; } systemMemoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
private QueryMemoryInfo createQueryMemoryInfo(QueryExecution query) { // when the legacy system pool is enabled we use the user memory instead of the total memory if (isLegacySystemPoolEnabled) { return new QueryMemoryInfo(query.getQueryId(), query.getMemoryPool().getId(), query.getUserMemoryReservation().toBytes()); } return new QueryMemoryInfo(query.getQueryId(), query.getMemoryPool().getId(), query.getTotalMemoryReservation().toBytes()); }
@Test(expectedExceptions = IllegalArgumentException.class) public void testInvalidValues() { NodeMemoryConfig config = new NodeMemoryConfig(); config.setMaxQueryTotalMemoryPerNode(new DataSize(1, GIGABYTE)); config.setHeapHeadroom(new DataSize(3.1, GIGABYTE)); // In this case we have 4GB - 1GB = 3GB available memory for the general pool // and the heap headroom and the config is more than that. validateHeapHeadroom(config, new DataSize(4, GIGABYTE).toBytes()); } }
private synchronized ListenableFuture<?> updateRevocableMemory(String allocationTag, long delta) { if (delta >= 0) { return memoryPool.reserveRevocable(queryId, delta); } memoryPool.freeRevocable(queryId, -delta); return NOT_BLOCKED; }
@Override public VersionedMemoryPoolId getMemoryPool() { return new VersionedMemoryPoolId(GENERAL_POOL, 0); }
public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitOperatorContext(this, context); }
public static <V> NonCancellableMemoryFuture<V> create() { return new NonCancellableMemoryFuture<>(); }
public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitPipelineContext(this, context); }
public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitTaskContext(this, context); }
public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitDriverContext(this, context); }
@Override public synchronized void setResourceOvercommit() { // Allow the query to use the entire pool. This way the worker will kill the query, if it uses the entire local general pool. // The coordinator will kill the query if the cluster runs out of memory. maxMemory = memoryPool.getMaxBytes(); }
private synchronized ListenableFuture<?> updateRevocableMemory(String allocationTag, long delta) { if (delta >= 0) { return memoryPool.reserveRevocable(queryId, delta); } memoryPool.freeRevocable(queryId, -delta); return NOT_BLOCKED; }
@Override public <C, R> R accept(QueryContextVisitor<C, R> visitor, C context) { return visitor.visitQueryContext(this, context); }