private void finishLookupSourceUnspilling() { checkState(state == State.INPUT_UNSPILLING); if (!unspillInProgress.get().isDone()) { // Pages have not be unspilled yet. return; } // Use Queue so that Pages already consumed by Index are not retained by us. Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get())); long memoryRetainedByRemainingPages = pages.stream() .mapToLong(Page::getRetainedSizeInBytes) .sum(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); while (!pages.isEmpty()) { Page next = pages.remove(); index.addPage(next); // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size. memoryRetainedByRemainingPages -= next.getRetainedSizeInBytes(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); } LookupSourceSupplier partition = buildLookupSource(); lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one")); localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); spilledLookupSourceHandle.setLookupSource(partition); state = State.INPUT_UNSPILLED_AND_BUILT; }
private void finishInput() { checkState(state == State.CONSUMING_INPUT); if (lookupSourceFactoryDestroyed.isDone()) { close(); return; } LookupSourceSupplier partition = buildLookupSource(); if (spillEnabled) { localRevocableMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); } else { localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); } lookupSourceNotNeeded = Optional.of(lookupSourceFactory.lendPartitionLookupSource(partitionIndex, partition)); state = State.LOOKUP_SOURCE_BUILT; }