close(); return; finishInput(); return; disposeLookupSourceIfRequested(); return; finishSpilledInput(); return; close(); unspillLookupSourceIfRequested(); finishLookupSourceUnspilling(); return; disposeUnspilledLookupSourceIfRequested(); return;
private static void revokeMemory(HashBuilderOperator operator) { getFutureValue(operator.startMemoryRevoke()); operator.finishMemoryRevoke(); checkState(operator.getState() == HashBuilderOperator.State.SPILLING_INPUT || operator.getState() == HashBuilderOperator.State.INPUT_SPILLED); }
@Override public void addInput(Page page) { requireNonNull(page, "page is null"); if (lookupSourceFactoryDestroyed.isDone()) { close(); return; } if (state == State.SPILLING_INPUT) { spillInput(page); return; } checkState(state == State.CONSUMING_INPUT); updateIndex(page); }
@Override public HashBuilderOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName()); PartitionedLookupSourceFactory lookupSourceFactory = this.lookupSourceFactoryManager.getJoinBridge(driverContext.getLifespan()); int partitionIndex = getAndIncrementPartitionIndex(driverContext.getLifespan()); verify(partitionIndex < lookupSourceFactory.partitions()); return new HashBuilderOperator( operatorContext, lookupSourceFactory, partitionIndex, outputChannels, hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, expectedPositions, pagesIndexFactory, spillEnabled, singleStreamSpillerFactory); }
private static void triggerMemoryRevokingAndWait(HashBuilderOperator operator, TaskStateMachine taskStateMachine) throws Exception { // When there is background thread running Driver, we must delegate memory revoking to that thread operator.getOperatorContext().requestMemoryRevoking(); while (operator.getOperatorContext().isMemoryRevokingRequested()) { checkErrors(taskStateMachine); Thread.sleep(10); } checkErrors(taskStateMachine); checkState(operator.getState() == HashBuilderOperator.State.SPILLING_INPUT || operator.getState() == HashBuilderOperator.State.INPUT_SPILLED); }
private void finishInput() { checkState(state == State.CONSUMING_INPUT); if (lookupSourceFactoryDestroyed.isDone()) { close(); return; } LookupSourceSupplier partition = buildLookupSource(); if (spillEnabled) { localRevocableMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); } else { localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); } lookupSourceNotNeeded = Optional.of(lookupSourceFactory.lendPartitionLookupSource(partitionIndex, partition)); state = State.LOOKUP_SOURCE_BUILT; }
@Override public boolean isFinished() { if (lookupSourceFactoryDestroyed.isDone()) { // Finish early when the probe side is empty close(); return true; } return state == State.CLOSED; }
private void finishLookupSourceUnspilling() { checkState(state == State.INPUT_UNSPILLING); if (!unspillInProgress.get().isDone()) { // Pages have not be unspilled yet. return; } // Use Queue so that Pages already consumed by Index are not retained by us. Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get())); long memoryRetainedByRemainingPages = pages.stream() .mapToLong(Page::getRetainedSizeInBytes) .sum(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); while (!pages.isEmpty()) { Page next = pages.remove(); index.addPage(next); // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size. memoryRetainedByRemainingPages -= next.getRetainedSizeInBytes(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); } LookupSourceSupplier partition = buildLookupSource(); lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one")); localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); spilledLookupSourceHandle.setLookupSource(partition); state = State.INPUT_UNSPILLED_AND_BUILT; }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName()); return new HashBuilderOperator( operatorContext, lookupSourceSupplier, hashChannels, hashChannel, expectedPositions); }
private void disposeLookupSourceIfRequested() { checkState(state == State.LOOKUP_SOURCE_BUILT); verify(lookupSourceNotNeeded.isPresent()); if (!lookupSourceNotNeeded.get().isDone()) { return; } index.clear(); localRevocableMemoryContext.setBytes(0); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); lookupSourceSupplier = null; close(); }
private void disposeUnspilledLookupSourceIfRequested() { checkState(state == State.INPUT_UNSPILLED_AND_BUILT); if (!spilledLookupSourceHandle.getDisposeRequested().isDone()) { return; } index.clear(); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); close(); }