private static Long toBytes(DataSize dataSize) { if (dataSize == null) { return null; } return dataSize.toBytes(); }
public MemoryPool(MemoryPoolId id, DataSize size) { this.id = requireNonNull(id, "name is null"); requireNonNull(size, "size is null"); maxBytes = size.toBytes(); }
public static long retainedSizeOfPages(int count) { return RETAINED_PAGE_SIZE.toBytes() * count; } }
@NotNull public DataSize getMaxQueryTotalMemory() { if (maxQueryTotalMemory == null) { return succinctBytes(maxQueryMemory.toBytes() * 2); } return maxQueryTotalMemory; }
public static DataSize getAggregationOperatorUnspillMemoryLimit(Session session) { DataSize memoryLimitForMerge = session.getSystemProperty(AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT, DataSize.class); checkArgument(memoryLimitForMerge.toBytes() >= 0, "%s must be positive", AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT); return memoryLimitForMerge; }
@Inject public MemoryPagesStore(MemoryConfig config) { this.maxBytes = config.getMaxDataPerNode().toBytes(); }
@Inject public ThriftIndexProvider(DriftClient<PrestoThriftService> client, ThriftHeaderProvider thriftHeaderProvider, ThriftConnectorStats stats, ThriftConnectorConfig config) { this.client = requireNonNull(client, "client is null"); this.thriftHeaderProvider = requireNonNull(thriftHeaderProvider, "thriftHeaderProvider is null"); this.stats = requireNonNull(stats, "stats is null"); requireNonNull(config, "config is null"); this.maxBytesPerResponse = config.getMaxResponseSize().toBytes(); this.lookupRequestsConcurrency = config.getLookupRequestsConcurrency(); }
@Override public ListenableFuture<BufferResult> get(OutputBufferId bufferId, long startingSequenceId, DataSize maxSize) { checkState(!Thread.holdsLock(this), "Can not get pages while holding a lock on this"); requireNonNull(bufferId, "bufferId is null"); checkArgument(maxSize.toBytes() > 0, "maxSize must be at least 1 byte"); return getBuffer(bufferId).getPages(startingSequenceId, maxSize, Optional.of(masterBuffer)); }
@Override public ListenableFuture<BufferResult> get(OutputBufferId outputBufferId, long startingSequenceId, DataSize maxSize) { checkState(!Thread.holdsLock(this), "Can not get pages while holding a lock on this"); requireNonNull(outputBufferId, "outputBufferId is null"); checkArgument(maxSize.toBytes() > 0, "maxSize must be at least 1 byte"); return getBuffer(outputBufferId).getPages(startingSequenceId, maxSize); }
@Override public ListenableFuture<BufferResult> get(OutputBufferId outputBufferId, long startingSequenceId, DataSize maxSize) { requireNonNull(outputBufferId, "outputBufferId is null"); checkArgument(maxSize.toBytes() > 0, "maxSize must be at least 1 byte"); return partitions.get(outputBufferId.getId()).getPages(startingSequenceId, maxSize); }
@JsonProperty public DataSize getLogicalWrittenDataSize() { return succinctBytes( operatorSummaries.stream() .filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())) .mapToLong(stats -> stats.getInputDataSize().toBytes()) .sum()); }
@Override public long getEstimatedSize() { return pagesIndex.getEstimatedSize().toBytes() + accumulator.getEstimatedSize(); }
@Override public long getEstimatedSize() { return pagesIndex.getEstimatedSize().toBytes() + accumulator.getEstimatedSize(); }
@Override public Page getOutput() { if (state == State.NEEDS_INPUT || state == State.FINISHED) { return null; } Page page = extractOutput(); localUserMemoryContext.setBytes(pagesIndex.getEstimatedSize().toBytes()); return page; }
@Override public void close() { index.clear(); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); } }
private void disposeUnspilledLookupSourceIfRequested() { checkState(state == State.INPUT_UNSPILLED_AND_BUILT); if (!spilledLookupSourceHandle.getDisposeRequested().isDone()) { return; } index.clear(); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); close(); }
private boolean canReplicate(SemiJoinNode node, Context context) { Optional<DataSize> joinMaxBroadcastTableSize = getJoinMaxBroadcastTableSize(context.getSession()); if (!joinMaxBroadcastTableSize.isPresent()) { return true; } PlanNode buildSide = node.getFilteringSource(); PlanNodeStatsEstimate buildSideStatsEstimate = context.getStatsProvider().getStats(buildSide); double buildSideSizeInBytes = buildSideStatsEstimate.getOutputSizeInBytes(buildSide.getOutputSymbols(), context.getSymbolAllocator().getTypes()); return buildSideSizeInBytes <= joinMaxBroadcastTableSize.get().toBytes(); }
@Test public void testBlockingOnUserMemory() { setUpCountStarFromOrdersWithJoin(); assertTrue(userPool.tryReserve(fakeQueryId, "test", TEN_MEGABYTES.toBytes())); runDriversUntilBlocked(waitingForUserMemory()); assertTrue(userPool.getFreeBytes() <= 0, String.format("Expected empty pool but got [%d]", userPool.getFreeBytes())); userPool.free(fakeQueryId, "test", TEN_MEGABYTES.toBytes()); assertDriversProgress(waitingForUserMemory()); }
private SourceOperator createExchangeOperator() { ExchangeOperatorFactory operatorFactory = new ExchangeOperatorFactory(0, new PlanNodeId("test"), exchangeClientSupplier, SERDE_FACTORY); DriverContext driverContext = createTaskContext(scheduler, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) .addDriverContext(); SourceOperator operator = operatorFactory.createOperator(driverContext); assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0); return operator; }