public LocalMergeSourceOperator(OperatorContext operatorContext, List<LocalExchangeSource> sources, List<Type> types, PageWithPositionComparator comparator) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.sources = requireNonNull(sources, "sources is null"); List<WorkProcessor<Page>> pageProducers = sources.stream() .map(LocalExchangeSource::pages) .collect(toImmutableList()); mergedPages = mergeSortedPages( pageProducers, requireNonNull(comparator, "comparator is null"), types, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); }
private boolean isFinalStageInfo(StageInfo stageInfo) { List<StageInfo> subStages = getSubStagesOf(operatorContext.getDriverContext().getTaskId().getStageId(), stageInfo); return subStages.stream().allMatch(StageInfo::isFinalStageInfo); }
private WorkProcessor<Page> mergeFromDisk() { checkState(spiller.isPresent()); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList()), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, memoryLimitForMerge); }
private WorkProcessor<Page> mergeFromDiskAndMemory() { checkState(spiller.isPresent()); hashAggregationBuilder.setOutputPartial(); mergeHashSort = Optional.of(new MergeHashSort(operatorContext.newAggregateSystemMemoryContext())); WorkProcessor<Page> mergedSpilledPages = mergeHashSort.get().merge( groupByTypes, hashAggregationBuilder.buildIntermediateTypes(), ImmutableList.<WorkProcessor<Page>>builder() .addAll(spiller.get().getSpills().stream() .map(WorkProcessor::fromIterator) .collect(toImmutableList())) .add(hashAggregationBuilder.buildHashSortedResult()) .build(), operatorContext.getDriverContext().getYieldSignal()); return mergeSortedPages(mergedSpilledPages, max(memoryLimitForMerge - memoryLimitForMergeWithMemory, 1L)); }
public AssignUniqueIdOperator(OperatorContext operatorContext, AtomicLong rowIdPool) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.rowIdPool = requireNonNull(rowIdPool, "rowIdPool is null"); TaskId fullTaskId = operatorContext.getDriverContext().getTaskId(); uniqueValueMask = (((long) fullTaskId.getStageId().getId()) << 54) | (((long) fullTaskId.getId()) << 40); requestValues(); }
@Override public void noMoreSplits() { mergedPages = mergeSortedPages( pageProducers, comparator, outputChannels, outputTypes, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, operatorContext.aggregateUserMemoryContext(), operatorContext.getDriverContext().getYieldSignal()); blockedOnSplits.set(null); }
@Override public final void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); requireNonNull(page, "page is null"); checkState(mergingOutput.needsInput(), "Page buffer is full"); mergingOutput.addInput(processor.process( operatorContext.getSession().toConnectorSession(), operatorContext.getDriverContext().getYieldSignal(), pageProcessorMemoryContext, page)); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); }
@Override public Page getOutput() { if (!finishing) { return null; } QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing"); checkState(queryInfo.getOutputStage().get().getSubStages().size() == 1, "Expected one sub stage of explain node"); if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionRegistry, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); outputConsumed = true; return new Page(builder.build()); }
if (operator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats().getSystemMemoryReservation().toBytes() > 0) { greaterThanZero = true; break;
private void processProbe(LookupSource lookupSource) { verify(probe != null); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (!yieldSignal.isSet()) { if (probe.getPosition() >= 0) { if (!joinCurrentPosition(lookupSource, yieldSignal)) { break; } if (!currentProbePositionProducedRow) { currentProbePositionProducedRow = true; if (!outerJoinCurrentPosition()) { break; } } } currentProbePositionProducedRow = false; if (!advanceProbePosition(lookupSource)) { break; } statisticsCounter.recordProbe(joinSourcePositions); joinSourcePositions = 0; } }
DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (probePosition < probe.getPositionCount()) { if (joinPositions == null) {
private Page processPageSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && mergingOutput.needsInput() && !yieldSignal.isSet()) { Page page = pageSource.getNextPage(); finishing = pageSource.isFinished(); pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); if (page != null) { page = recordProcessedInput(page); // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; Iterator<Optional<Page>> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); mergingOutput.addInput(output); } if (finishing) { mergingOutput.finish(); } } Page result = mergingOutput.getOutput(); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); return result; }
memoryPool.reserve(queryId, "test", reservedMemoryInBytes); long oldMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); int oldCapacity = getHashCapacity.apply(operator); long newMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); long rehashedMemoryUsage = operator.getOperatorContext().getDriverContext().getMemoryUsage(); assertBetweenInclusive(rehashedMemoryUsage * 1.0 / newMemoryUsage, 0.99, 1.01);
private Page processColumnSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && !yieldSignal.isSet()) { CursorProcessorOutput output = cursorProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, cursor, pageBuilder); pageSourceMemoryContext.setBytes(cursor.getSystemMemoryUsage()); long bytesProcessed = cursor.getCompletedBytes() - completedBytes; long elapsedNanos = cursor.getReadTimeNanos() - readTimeNanos; operatorContext.recordRawInputWithTiming(bytesProcessed, elapsedNanos); // TODO: derive better values for cursors operatorContext.recordProcessedInput(bytesProcessed, output.getProcessedRows()); completedBytes = cursor.getCompletedBytes(); readTimeNanos = cursor.getReadTimeNanos(); if (output.isNoMoreRows()) { finishing = true; mergingOutput.finish(); } } // only return a page if buffer is full or we are finishing Page page = null; if (!pageBuilder.isEmpty() && (finishing || pageBuilder.isFull())) { page = pageBuilder.build(); pageBuilder.reset(); } outputMemoryContext.setBytes(pageBuilder.getRetainedSizeInBytes()); return page; }
NestedLoopJoinPages(List<Page> pages, DataSize estimatedSize, OperatorContext operatorContext) { requireNonNull(pages, "pages is null"); requireNonNull(operatorContext, "operatorContext is null"); this.pages = ImmutableList.copyOf(pages); this.taskContext = operatorContext.getDriverContext().getPipelineContext().getTaskContext(); this.estimatedSize = requireNonNull(estimatedSize, "estimatedSize is null"); operatorContext.transferMemoryToTaskContext(estimatedSize.toBytes()); }
public SharedLookupSource(LookupSource lookupSource, OperatorContext operatorContext) { requireNonNull(operatorContext, "operatorContext is null"); this.lookupSource = requireNonNull(lookupSource, "lookupSource is null"); this.taskContext = operatorContext.getDriverContext().getPipelineContext().getTaskContext(); operatorContext.transferMemoryToTaskContext(lookupSource.getInMemorySizeInBytes()); }
@Override public ListenableFuture<LookupSource> getLookupSource(OperatorContext operatorContext) { IndexLoader indexLoader = indexLoaderSupplier.get(); indexLoader.setContext(operatorContext.getDriverContext().getPipelineContext().getTaskContext()); return Futures.immediateFuture(new IndexLookupSource(indexLoader)); }