public long getMemoryUsage() { return driverMemoryContext.getUserMemory(); }
public long getMemoryUsage() { return driverMemoryContext.getUserMemory(); }
public DataSize getMemoryReservation() { return new DataSize(taskMemoryContext.getUserMemory(), BYTE); }
public DataSize getMemoryReservation() { return new DataSize(taskMemoryContext.getUserMemory(), BYTE); }
private synchronized ListenableFuture<?> updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { enforceUserMemoryLimit(queryMemoryContext.getUserMemory(), delta, maxMemory); return memoryPool.reserve(queryId, allocationTag, delta); } memoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
/** * Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory * calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. * * @see this##updateSystemMemory(long) for details. */ private synchronized ListenableFuture<?> updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { enforceUserMemoryLimit(queryMemoryContext.getUserMemory(), delta, maxUserMemory); return memoryPool.reserve(queryId, allocationTag, delta); } memoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
public void destroy() { // reset memory revocation listener so that OperatorContext doesn't hold any references to Driver instance synchronized (this) { memoryRevocationRequestListener = null; } operatorMemoryContext.close(); if (operatorMemoryContext.getSystemMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero system memory (%d bytes) after destroy()", this, operatorMemoryContext.getSystemMemory())); } if (operatorMemoryContext.getUserMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero user memory (%d bytes) after destroy()", this, operatorMemoryContext.getUserMemory())); } if (operatorMemoryContext.getRevocableMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero revocable memory (%d bytes) after destroy()", this, operatorMemoryContext.getRevocableMemory())); } }
public void destroy() { // reset memory revocation listener so that OperatorContext doesn't hold any references to Driver instance synchronized (this) { memoryRevocationRequestListener = null; } operatorMemoryContext.close(); if (operatorMemoryContext.getSystemMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero system memory (%d bytes) after destroy()", this, operatorMemoryContext.getSystemMemory())); } if (operatorMemoryContext.getUserMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero user memory (%d bytes) after destroy()", this, operatorMemoryContext.getUserMemory())); } if (operatorMemoryContext.getRevocableMemory() != 0) { throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Operator %s has non-zero revocable memory (%d bytes) after destroy()", this, operatorMemoryContext.getRevocableMemory())); } }
/** * Deadlock is possible for concurrent user and system allocations when updateSystemMemory()/updateUserMemory * calls queryMemoryContext.getUserMemory()/queryMemoryContext.getSystemMemory(), respectively. * * @see this##updateSystemMemory(long) for details. */ private synchronized ListenableFuture<?> updateUserMemory(String allocationTag, long delta) { if (delta >= 0) { enforceUserMemoryLimit(queryMemoryContext.getUserMemory(), delta, maxUserMemory); return memoryPool.reserve(queryId, allocationTag, delta); } memoryPool.free(queryId, allocationTag, -delta); return NOT_BLOCKED; }
private void updatePeakMemoryReservations() { long userMemory = operatorMemoryContext.getUserMemory(); long systemMemory = operatorMemoryContext.getSystemMemory(); long totalMemory = userMemory + systemMemory; peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); peakSystemMemoryReservation.accumulateAndGet(systemMemory, Math::max); peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); }
private void assertLocalMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedPoolMemory, long expectedContextUserMemory, long expectedContextSystemMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedContextUserMemory, "User memory verification failed"); assertEquals(memoryPool.getReservedBytes(), expectedPoolMemory, "Memory pool verification failed"); assertEquals(memoryTrackingContext.localSystemMemoryContext().getBytes(), expectedContextSystemMemory, "Local system memory verification failed"); } }
private void updatePeakMemoryReservations() { long userMemory = operatorMemoryContext.getUserMemory(); long systemMemory = operatorMemoryContext.getSystemMemory(); long totalMemory = userMemory + systemMemory; peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); peakSystemMemoryReservation.accumulateAndGet(systemMemory, Math::max); peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); }
private void assertOperatorMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedUserMemory, long expectedSystemMemory, long expectedRevocableMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedUserMemory, "User memory verification failed"); // both user and system memory are allocated from the same memoryPool assertEquals(memoryPool.getReservedBytes(), expectedUserMemory + expectedSystemMemory, "Memory pool verification failed"); assertEquals(memoryTrackingContext.getSystemMemory(), expectedSystemMemory, "System memory verification failed"); assertEquals(memoryTrackingContext.getRevocableMemory(), expectedRevocableMemory, "Revocable memory verification failed"); }
private void assertOperatorMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedUserMemory, long expectedSystemMemory, long expectedRevocableMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedUserMemory, "User memory verification failed"); // both user and system memory are allocated from the same memoryPool assertEquals(memoryPool.getReservedBytes(), expectedUserMemory + expectedSystemMemory, "Memory pool verification failed"); assertEquals(memoryTrackingContext.getSystemMemory(), expectedSystemMemory, "System memory verification failed"); assertEquals(memoryTrackingContext.getRevocableMemory(), expectedRevocableMemory, "Revocable memory verification failed"); }
private synchronized boolean tryUpdateUserMemory(String allocationTag, long delta) { if (delta <= 0) { ListenableFuture<?> future = updateUserMemory(allocationTag, delta); // When delta == 0 and the pool is full the future can still not be done, // but, for negative deltas it must always be done. if (delta < 0) { verify(future.isDone(), "future should be done"); } return true; } if (queryMemoryContext.getUserMemory() + delta > maxUserMemory) { return false; } return memoryPool.tryReserve(queryId, allocationTag, delta); }
private void assertLocalMemoryAllocations( MemoryTrackingContext memoryTrackingContext, long expectedPoolMemory, long expectedContextUserMemory, long expectedContextSystemMemory) { assertEquals(memoryTrackingContext.getUserMemory(), expectedContextUserMemory, "User memory verification failed"); assertEquals(memoryPool.getReservedBytes(), expectedPoolMemory, "Memory pool verification failed"); assertEquals(memoryTrackingContext.localSystemMemoryContext().getBytes(), expectedContextSystemMemory, "Local system memory verification failed"); } }
private synchronized boolean tryUpdateUserMemory(String allocationTag, long delta) { if (delta <= 0) { ListenableFuture<?> future = updateUserMemory(allocationTag, delta); // When delta == 0 and the pool is full the future can still not be done, // but, for negative deltas it must always be done. if (delta < 0) { verify(future.isDone(), "future should be done"); } return true; } if (queryMemoryContext.getUserMemory() + delta > maxMemory) { return false; } return memoryPool.tryReserve(queryId, allocationTag, delta); }
private synchronized boolean tryUpdateUserMemory(String allocationTag, long delta) { if (delta <= 0) { ListenableFuture<?> future = updateUserMemory(allocationTag, delta); // When delta == 0 and the pool is full the future can still not be done, // but, for negative deltas it must always be done. if (delta < 0) { verify(future.isDone(), "future should be done"); } return true; } if (queryMemoryContext.getUserMemory() + delta > maxUserMemory) { return false; } return memoryPool.tryReserve(queryId, allocationTag, delta); }
private void assertMemoryIsReleased(TableWriterOperator tableWriterOperator) { OperatorContext tableWriterOperatorOperatorContext = tableWriterOperator.getOperatorContext(); MemoryTrackingContext tableWriterMemoryContext = tableWriterOperatorOperatorContext.getOperatorMemoryContext(); assertEquals(tableWriterMemoryContext.getSystemMemory(), 0); assertEquals(tableWriterMemoryContext.getUserMemory(), 0); assertEquals(tableWriterMemoryContext.getRevocableMemory(), 0); Operator statisticAggregationOperator = tableWriterOperator.getStatisticAggregationOperator(); assertTrue(statisticAggregationOperator instanceof AggregationOperator); AggregationOperator aggregationOperator = (AggregationOperator) statisticAggregationOperator; OperatorContext aggregationOperatorOperatorContext = aggregationOperator.getOperatorContext(); MemoryTrackingContext aggregationOperatorMemoryContext = aggregationOperatorOperatorContext.getOperatorMemoryContext(); assertEquals(aggregationOperatorMemoryContext.getSystemMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getUserMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getRevocableMemory(), 0); }
private void assertMemoryIsReleased(TableWriterOperator tableWriterOperator) { OperatorContext tableWriterOperatorOperatorContext = tableWriterOperator.getOperatorContext(); MemoryTrackingContext tableWriterMemoryContext = tableWriterOperatorOperatorContext.getOperatorMemoryContext(); assertEquals(tableWriterMemoryContext.getSystemMemory(), 0); assertEquals(tableWriterMemoryContext.getUserMemory(), 0); assertEquals(tableWriterMemoryContext.getRevocableMemory(), 0); Operator statisticAggregationOperator = tableWriterOperator.getStatisticAggregationOperator(); assertTrue(statisticAggregationOperator instanceof AggregationOperator); AggregationOperator aggregationOperator = (AggregationOperator) statisticAggregationOperator; OperatorContext aggregationOperatorOperatorContext = aggregationOperator.getOperatorContext(); MemoryTrackingContext aggregationOperatorMemoryContext = aggregationOperatorOperatorContext.getOperatorMemoryContext(); assertEquals(aggregationOperatorMemoryContext.getSystemMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getUserMemory(), 0); assertEquals(aggregationOperatorMemoryContext.getRevocableMemory(), 0); }