public RevocableMemoryOperator(OperatorContext operatorContext, DataSize reservedPerPage, long numberOfPages) { this.operatorContext = operatorContext; this.reservedPerPage = reservedPerPage; this.numberOfPages = numberOfPages; this.revocableMemoryContext = operatorContext.localRevocableMemoryContext(); }
private void closeAggregationBuilder() { outputPages = null; if (aggregationBuilder != null) { aggregationBuilder.recordHashCollisions(hashCollisionsCounter); aggregationBuilder.close(); // aggregationBuilder.close() will release all memory reserved in memory accounting. // The reference must be set to null afterwards to avoid unaccounted memory. aggregationBuilder = null; } operatorContext.localUserMemoryContext().setBytes(0); operatorContext.localRevocableMemoryContext().setBytes(0); }
@Test public void testRevocationAlreadyRequested() { AtomicInteger counter = new AtomicInteger(); OperatorContext operatorContext = TestingOperatorContext.create(scheduledExecutor); LocalMemoryContext revocableMemoryContext = operatorContext.localRevocableMemoryContext(); revocableMemoryContext.setBytes(1000); // when memory revocation is already requested setting a listener should immediately execute it operatorContext.requestMemoryRevoking(); operatorContext.setMemoryRevocationRequestListener(() -> counter.incrementAndGet()); assertTrue(operatorContext.isMemoryRevokingRequested()); assertEquals(counter.get(), 1); }
this.searchFunctionFactories = searchFunctionFactories; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext();
public SpillableHashAggregationBuilder( List<AccumulatorFactory> accumulatorFactories, AggregationNode.Step step, int expectedGroups, List<Type> groupByTypes, List<Integer> groupByChannels, Optional<Integer> hashChannel, OperatorContext operatorContext, DataSize memoryLimitForMerge, DataSize memoryLimitForMergeWithMemory, SpillerFactory spillerFactory, JoinCompiler joinCompiler) { this.accumulatorFactories = accumulatorFactories; this.step = step; this.expectedGroups = expectedGroups; this.groupByTypes = groupByTypes; this.groupByChannels = groupByChannels; this.hashChannel = hashChannel; this.operatorContext = operatorContext; this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.localRevocableMemoryContext = operatorContext.localRevocableMemoryContext(); this.memoryLimitForMerge = memoryLimitForMerge.toBytes(); this.memoryLimitForMergeWithMemory = memoryLimitForMergeWithMemory.toBytes(); this.spillerFactory = spillerFactory; this.joinCompiler = joinCompiler; rebuildHashAggregationBuilder(); }
@Test public void testOperatorMemoryRevocation() { AtomicInteger counter = new AtomicInteger(); OperatorContext operatorContext = TestingOperatorContext.create(scheduledExecutor); LocalMemoryContext revocableMemoryContext = operatorContext.localRevocableMemoryContext(); revocableMemoryContext.setBytes(1000); operatorContext.setMemoryRevocationRequestListener(() -> counter.incrementAndGet()); operatorContext.requestMemoryRevoking(); assertTrue(operatorContext.isMemoryRevokingRequested()); assertEquals(counter.get(), 1); // calling resetMemoryRevokingRequested() should clear the memory revoking requested flag operatorContext.resetMemoryRevokingRequested(); assertFalse(operatorContext.isMemoryRevokingRequested()); operatorContext.requestMemoryRevoking(); assertEquals(counter.get(), 2); assertTrue(operatorContext.isMemoryRevokingRequested()); }
@Override public ListenableFuture<?> isBlocked() { // this operator is always blocked and when queried by the driver // it triggers memory revocation so that the driver gets unblocked LocalMemoryContext revocableMemoryContext = getOperatorContext().localRevocableMemoryContext(); revocableMemoryContext.setBytes(100); getOperatorContext().requestMemoryRevoking(); return SettableFuture.create(); } }
/** * Test that when a {@link MemoryPool} is over-allocated, revocable memory is revoked without delay (although asynchronously). */ @Test public void testImmediateMemoryRevoking() throws Exception { // Given SqlTask sqlTask = newSqlTask(); OperatorContext operatorContext = createContexts(sqlTask); allOperatorContexts = ImmutableSet.of(operatorContext); List<SqlTask> tasks = ImmutableList.of(sqlTask); MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, executor, 1.0, 1.0); scheduler.registerPoolListeners(); // no periodic check initiated // When operatorContext.localRevocableMemoryContext().setBytes(12); awaitAsynchronousCallbacksRun(); // Then assertMemoryRevokingRequestedFor(operatorContext); }
@Test public void testCountAlreadyRevokedMemoryWithinAPool() throws Exception { // Given SqlTask sqlTask1 = newSqlTask(); MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(10, BYTE)); sqlTask1.getQueryContext().setMemoryPool(anotherMemoryPool); OperatorContext operatorContext1 = createContexts(sqlTask1); SqlTask sqlTask2 = newSqlTask(); OperatorContext operatorContext2 = createContexts(sqlTask2); List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2); MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, executor, 1.0, 1.0); allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2); /* * sqlTask1 fills its pool */ operatorContext1.localRevocableMemoryContext().setBytes(12); requestMemoryRevoking(scheduler); assertMemoryRevokingRequestedFor(operatorContext1); /* * When sqlTask2 fills its pool */ operatorContext2.localRevocableMemoryContext().setBytes(12); requestMemoryRevoking(scheduler); /* * Then sqlTask2 should be asked to revoke its memory too */ assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2); }
assertMemoryRevokingNotRequested(); LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext(); LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext(); LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext(); LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();