/** * Update memory usage. * * @return true if the reservation is within the limit */ // TODO: update in the interface after the new memory tracking framework is landed (#9049) // Essentially we would love to have clean interfaces to support both pushing and pulling memory usage // The following implementation is a hybrid model, where the push model is going to call the pull model causing reentrancy private boolean updateMemoryReservation() { // Operator/driver will be blocked on memory after we call localUserMemoryContext.setBytes(). // If memory is not available, once we return, this operator will be blocked until memory is available. long memorySizeInBytes = groupByHash.map(GroupByHash::getEstimatedSize).orElse(0L) + partitionRowCount.sizeOf(); localUserMemoryContext.setBytes(memorySizeInBytes); // If memory is not available, inform the caller that we cannot proceed for allocation. return operatorContext.isWaitingForMemory().isDone(); }
public boolean updateMemoryReservation() { // If memory is not available, once we return, this operator will be blocked until memory is available. localMemoryContext.setBytes(hash.getEstimatedSize()); // If memory is not available, inform the caller that we cannot proceed for allocation. return operatorContext.isWaitingForMemory().isDone(); }
private boolean updateMemoryReservation() { // TODO: may need to use trySetMemoryReservation with a compaction to free memory (but that may cause GC pressure) localUserMemoryContext.setBytes(groupedTopNBuilder.getEstimatedSizeInBytes()); return operatorContext.isWaitingForMemory().isDone(); }
private Predicate<OperatorContext> waitingForUserMemory() { return (OperatorContext operatorContext) -> !operatorContext.isWaitingForMemory().isDone(); }
/** * Update memory usage. * * @return true to if the reservation is within the limit */ // TODO: update in the interface after the new memory tracking framework is landed (#9049) // Essentially we would love to have clean interfaces to support both pushing and pulling memory usage // The following implementation is a hybrid model, where the push model is going to call the pull model causing reentrancy private boolean updateMemoryReservation() { // Operator/driver will be blocked on memory after we call localUserMemoryContext.setBytes(). // If memory is not available, once we return, this operator will be blocked until memory is available. localUserMemoryContext.setBytes(markDistinctHash.getEstimatedSize()); // If memory is not available, inform the caller that we cannot proceed for allocation. return operatorContext.isWaitingForMemory().isDone(); }
/** * Update memory usage. * * @return true if the reservation is within the limit */ // TODO: update in the interface after the new memory tracking framework is landed (#9049) // Essentially we would love to have clean interfaces to support both pushing and pulling memory usage // The following implementation is a hybrid model, where the push model is going to call the pull model causing reentrancy private boolean updateMemoryReservation() { // Operator/driver will be blocked on memory after we call localUserMemoryContext.setBytes(). // If memory is not available, once we return, this operator will be blocked until memory is available. localUserMemoryContext.setBytes(groupByHash.getEstimatedSize()); // If memory is not available, inform the caller that we cannot proceed for allocation. return operatorContext.isWaitingForMemory().isDone(); }
private Optional<ListenableFuture<?>> getBlockedFuture(Operator operator) { ListenableFuture<?> blocked = revokingOperators.get(operator); if (blocked != null) { // We mark operator as blocked regardless of blocked.isDone(), because finishMemoryRevoke has not been called yet. return Optional.of(blocked); } blocked = operator.isBlocked(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } blocked = operator.getOperatorContext().isWaitingForRevocableMemory(); if (!blocked.isDone()) { return Optional.of(blocked); } return Optional.empty(); }
/** * Update memory usage with extra memory needed. * * @return true to if the reservation is within the limit */ // TODO: update in the interface after the new memory tracking framework is landed // Essentially we would love to have clean interfaces to support both pushing and pulling memory usage // The following implementation is a hybrid model, where the push model is going to call the pull model causing reentrancy private boolean updateMemoryWithYieldInfo() { long memorySize = getSizeInMemory(); if (partial && maxPartialMemory.isPresent()) { updateMemory(memorySize); full = (memorySize > maxPartialMemory.getAsLong()); return true; } // Operator/driver will be blocked on memory after we call setBytes. // If memory is not available, once we return, this operator will be blocked until memory is available. updateMemory(memorySize); // If memory is not available, inform the caller that we cannot proceed for allocation. return operatorContext.isWaitingForMemory().isDone(); }
assertFalse(source.getOperatorContext().isWaitingForMemory().isDone()); blocked = driver.processFor(new Duration(1, NANOSECONDS)); assertFalse(blocked.isDone()); assertFalse(source.getOperatorContext().isWaitingForMemory().isDone()); assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
assertTrue(operator.getOperatorContext().isWaitingForMemory().isDone()); assertFalse(operator.getOperatorContext().isWaitingForMemory().isDone());
private static ListenableFuture<?> isBlocked(Operator operator) { ListenableFuture<?> blocked = operator.isBlocked(); if (blocked.isDone()) { blocked = operator.getOperatorContext().isWaitingForMemory(); } return blocked; }