public MarkDistinctOperator(OperatorContext operatorContext, List<Type> types, List<Integer> markDistinctChannels, Optional<Integer> hashChannel, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); requireNonNull(hashChannel, "hashChannel is null"); requireNonNull(markDistinctChannels, "markDistinctChannels is null"); ImmutableList.Builder<Type> distinctTypes = ImmutableList.builder(); for (int channel : markDistinctChannels) { distinctTypes.add(types.get(channel)); } this.markDistinctHash = new MarkDistinctHash(operatorContext.getSession(), distinctTypes.build(), Ints.toArray(markDistinctChannels), hashChannel, joinCompiler, this::updateMemoryReservation); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); }
private LookupSourceSupplier buildLookupSource() { LookupSourceSupplier partition = index.createLookupSourceSupplier(operatorContext.getSession(), hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, Optional.of(outputChannels)); hashCollisionsCounter.recordHashCollision(partition.getHashCollisions(), partition.getExpectedHashCollisions()); checkState(lookupSourceSupplier == null, "lookupSourceSupplier is already set"); this.lookupSourceSupplier = partition; return partition; }
this.groupByHash = Optional.of(createGroupByHash(partitionTypes, channels, hashChannel, expectedPositions, isDictionaryAggregationEnabled(operatorContext.getSession()), joinCompiler, this::updateMemoryReservation));
hashChannel, expectedPositions, isDictionaryAggregationEnabled(operatorContext.getSession()), joinCompiler, this::updateMemoryReservation);
@Override public final void addInput(Page page) { checkState(!finishing, "Operator is already finishing"); requireNonNull(page, "page is null"); checkState(mergingOutput.needsInput(), "Page buffer is full"); mergingOutput.addInput(processor.process( operatorContext.getSession().toConnectorSession(), operatorContext.getDriverContext().getYieldSignal(), pageProcessorMemoryContext, page)); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); }
@Override public Page getOutput() { if (!finishing) { return null; } QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing"); checkState(queryInfo.getOutputStage().get().getSubStages().size() == 1, "Expected one sub stage of explain node"); if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionRegistry, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); outputConsumed = true; return new Page(builder.build()); }
public ChannelSetBuilder(Type type, Optional<Integer> hashChannel, int expectedPositions, OperatorContext operatorContext, JoinCompiler joinCompiler) { List<Type> types = ImmutableList.of(type); this.hash = createGroupByHash( types, HASH_CHANNELS, hashChannel, expectedPositions, isDictionaryAggregationEnabled(operatorContext.getSession()), joinCompiler, this::updateMemoryReservation); this.nullBlockPage = new Page(type.createBlockBuilder(null, 1, UNKNOWN.getFixedSize()).appendNull().build()); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.localMemoryContext = operatorContext.localUserMemoryContext(); }
@Override public Page getOutput() { if (split == null) { return null; } if (!finishing && pageSource == null && cursor == null) { ConnectorPageSource source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); if (source instanceof RecordPageSource) { cursor = ((RecordPageSource) source).getCursor(); } else { pageSource = source; } } if (pageSource != null) { return processPageSource(); } else { return processColumnSource(); } }
hashChannel, expectedGroups, isDictionaryAggregationEnabled(operatorContext.getSession()), joinCompiler, updateMemory);
private static MaterializedResult getProperColumns(Operator joinOperator, List<Type> types, RowPagesBuilder probePages, List<Page> actualPages) { if (probePages.getHashChannel().isPresent()) { List<Integer> hashChannels = ImmutableList.of(probePages.getHashChannel().get()); actualPages = dropChannel(actualPages, hashChannels); types = without(types, hashChannels); } return OperatorAssertion.toMaterializedResult(joinOperator.getOperatorContext().getSession(), types, actualPages); }
@Override public void finish() { if (finishing) { return; } finishing = true; PagesSpatialIndexSupplier spatialIndex = index.createPagesSpatialIndex(operatorContext.getSession(), indexChannel, radiusChannel, partitionChannel, spatialRelationshipTest, filterFunctionFactory, outputChannels, partitions); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes() + spatialIndex.getEstimatedSize().toBytes()); indexNotNeeded = pagesSpatialIndexFactory.lendPagesSpatialIndex(spatialIndex); }
@Override public Page getOutput() { if (split == null) { return null; } if (source == null) { source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); } Page page = source.getNextPage(); if (page != null) { // assure the page is in memory before handing to another operator page = page.getLoadedPage(); // update operator stats long endCompletedBytes = source.getCompletedBytes(); long endReadTimeNanos = source.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; } // updating system memory usage should happen after page is loaded. systemMemoryContext.setBytes(source.getSystemMemoryUsage()); return page; } }
private Page processPageSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && mergingOutput.needsInput() && !yieldSignal.isSet()) { Page page = pageSource.getNextPage(); finishing = pageSource.isFinished(); pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); if (page != null) { page = recordProcessedInput(page); // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; Iterator<Optional<Page>> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); mergingOutput.addInput(output); } if (finishing) { mergingOutput.finish(); } } Page result = mergingOutput.getOutput(); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); return result; }
private Page processColumnSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && !yieldSignal.isSet()) { CursorProcessorOutput output = cursorProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, cursor, pageBuilder); pageSourceMemoryContext.setBytes(cursor.getSystemMemoryUsage()); long bytesProcessed = cursor.getCompletedBytes() - completedBytes; long elapsedNanos = cursor.getReadTimeNanos() - readTimeNanos; operatorContext.recordRawInputWithTiming(bytesProcessed, elapsedNanos); // TODO: derive better values for cursors operatorContext.recordProcessedInput(bytesProcessed, output.getProcessedRows()); completedBytes = cursor.getCompletedBytes(); readTimeNanos = cursor.getReadTimeNanos(); if (output.isNoMoreRows()) { finishing = true; mergingOutput.finish(); } } // only return a page if buffer is full or we are finishing Page page = null; if (!pageBuilder.isEmpty() && (finishing || pageBuilder.isFull())) { page = pageBuilder.build(); pageBuilder.reset(); } outputMemoryContext.setBytes(pageBuilder.getRetainedSizeInBytes()); return page; }
private void createSourceIfNecessary() { if ((split != null) && (source == null)) { source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); } } }
public MarkDistinctOperator(OperatorContext operatorContext, List<Type> types, List<Integer> markDistinctChannels, Optional<Integer> hashChannel) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); requireNonNull(hashChannel, "hashChannel is null"); requireNonNull(markDistinctChannels, "markDistinctChannels is null"); ImmutableList.Builder<Type> distinctTypes = ImmutableList.builder(); for (int channel : markDistinctChannels) { distinctTypes.add(types.get(channel)); } this.markDistinctHash = new MarkDistinctHash(operatorContext.getSession(), distinctTypes.build(), Ints.toArray(markDistinctChannels), hashChannel); }
public FilterAndProjectOperator(OperatorContext operatorContext, Iterable<? extends Type> types, PageProcessor processor) { this.processor = requireNonNull(processor, "processor is null"); this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); this.columnarProcessingEnabled = isColumnarProcessingEnabled(operatorContext.getSession()); this.columnarProcessingDictionaryEnabled = isColumnarProcessingDictionaryEnabled(operatorContext.getSession()); this.pageBuilder = new PageBuilder(getTypes()); }
private void createSourceIfNecessary() { if ((split != null) && (pageSource == null) && (cursor == null)) { ConnectorPageSource source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); if (source instanceof RecordPageSource) { cursor = ((RecordPageSource) source).getCursor(); } else { pageSource = source; } } }
public ChannelSetBuilder(Type type, Optional<Integer> hashChannel, int expectedPositions, OperatorContext operatorContext) { List<Type> types = ImmutableList.of(type); this.hash = createGroupByHash(operatorContext.getSession(), types, HASH_CHANNELS, Optional.empty(), hashChannel, expectedPositions); this.operatorContext = operatorContext; this.nullBlockPage = new Page(type.createBlockBuilder(new BlockBuilderStatus(), 1, UNKNOWN.getFixedSize()).appendNull().build()); }
public static void assertOperatorEquals(Operator operator, MaterializedResult expected) { List<Page> pages = toPages(operator); MaterializedResult actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), pages); assertEquals(actual, expected); }