private static boolean isHashEnabled(Session session) { return SystemSessionProperties.isOptimizeHashGenerationEnabled(session); }
public SessionPropertyManager() { this(new SystemSessionProperties()); }
WarningCollector warningCollector) String executionPolicyName = SystemSessionProperties.getExecutionPolicy(session); ExecutionPolicy executionPolicy = executionPolicies.get(executionPolicyName); checkArgument(executionPolicy != null, "No execution policy %s", executionPolicy);
protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) { ImmutableMap.Builder<Symbol, Type> symbolTypes = ImmutableMap.builder(); ImmutableMap.Builder<Symbol, Integer> symbolToInputMapping = ImmutableMap.builder(); ImmutableList.Builder<PageProjection> projections = ImmutableList.builder(); for (int channel = 0; channel < types.size(); channel++) { Symbol symbol = new Symbol("h" + channel); symbolTypes.put(symbol, types.get(channel)); symbolToInputMapping.put(symbol, channel); projections.add(new InputPageProjection(channel, types.get(channel))); } Optional<Expression> hashExpression = HashGenerationOptimizer.getHashExpression(ImmutableList.copyOf(symbolTypes.build().keySet())); verify(hashExpression.isPresent()); projections.add(new InterpretedPageProjection( hashExpression.get(), TypeProvider.copyOf(symbolTypes.build()), symbolToInputMapping.build(), localQueryRunner.getMetadata(), localQueryRunner.getSqlParser(), session)); return new FilterAndProjectOperator.FilterAndProjectOperatorFactory( operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session)); }
private static TaskHandle createTaskHandle( TaskStateMachine taskStateMachine, TaskContext taskContext, OutputBuffer outputBuffer, LocalExecutionPlan localExecutionPlan, TaskExecutor taskExecutor) { TaskHandle taskHandle = taskExecutor.addTask( taskStateMachine.getTaskId(), outputBuffer::getUtilization, getInitialSplitsPerNode(taskContext.getSession()), getSplitConcurrencyAdjustmentInterval(taskContext.getSession()), getMaxDriversPerTask(taskContext.getSession())); taskStateMachine.addStateChangeListener(state -> { if (state.isDone()) { taskExecutor.removeTask(taskHandle); for (DriverFactory factory : localExecutionPlan.getDriverFactories()) { factory.noMoreDrivers(); } } }); return taskHandle; }
long totalMemoryBytes = 0L; for (QueryExecution query : runningQueries) { boolean resourceOvercommit = resourceOvercommit(query.getSession()); long userMemoryReservation = query.getUserMemoryReservation().toBytes(); long totalMemoryReservation = query.getTotalMemoryReservation().toBytes(); long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes()); if (userMemoryReservation > userMemoryLimit) { query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit))); long totalMemoryLimit = min(maxQueryTotalMemory.toBytes(), getQueryMaxTotalMemory(query.getSession()).toBytes()); if (!isLegacySystemPoolEnabled && totalMemoryReservation > totalMemoryLimit) { query.fail(exceededGlobalTotalLimit(succinctBytes(totalMemoryLimit)));
/** * Enforce query max runtime/execution time limits */ private void enforceTimeLimits() { for (T query : queries.values()) { if (query.isDone()) { continue; } Duration queryMaxRunTime = getQueryMaxRunTime(query.getSession()); Duration queryMaxExecutionTime = getQueryMaxExecutionTime(query.getSession()); Optional<DateTime> executionStartTime = query.getExecutionStartTime(); DateTime createTime = query.getCreateTime(); if (executionStartTime.isPresent() && executionStartTime.get().plus(queryMaxExecutionTime.toMillis()).isBeforeNow()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded the maximum execution time limit of " + queryMaxExecutionTime)); } if (createTime.plus(queryMaxRunTime.toMillis()).isBeforeNow()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded maximum time limit of " + queryMaxRunTime)); } } }
taskHandle = taskExecutor.addTask(taskId, sharedBuffer::getUtilization, getInitialSplitsPerNode(taskContext.getSession()), getSplitConcurrencyAdjustmentInterval(taskContext.getSession())); taskStateMachine.addStateChangeListener(new RemoveTaskHandleWhenDone(taskExecutor, taskHandle)); taskStateMachine.addStateChangeListener(state -> {
for (QueryExecution query : queries) { long bytes = query.getTotalMemoryReservation(); DataSize sessionMaxQueryMemory = getQueryMaxMemory(query.getSession()); long queryMemoryLimit = Math.min(maxQueryMemory.toBytes(), sessionMaxQueryMemory.toBytes()); totalBytes += bytes; if (bytes > queryMemoryLimit) { if (resourceOvercommit(query.getSession())) {
public NodePartitionMap getNodePartitionMap(Session session, NodeScheduler nodeScheduler) { NodeSelector nodeSelector = nodeScheduler.createNodeSelector(null); List<Node> nodes; if (partitioning == SystemPartitioning.COORDINATOR_ONLY) { nodes = ImmutableList.of(nodeSelector.selectCurrentNode()); } else if (partitioning == SystemPartitioning.SINGLE) { nodes = nodeSelector.selectRandomNodes(1); } else if (partitioning == SystemPartitioning.FIXED) { nodes = nodeSelector.selectRandomNodes(getHashPartitionCount(session)); } else { throw new IllegalArgumentException("Unsupported plan distribution " + partitioning); } checkCondition(!nodes.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); return new NodePartitionMap(nodes, split -> { throw new UnsupportedOperationException("System distribution does not support source splits"); }); }
@Inject public SessionPropertyManager(SystemSessionProperties systemSessionProperties) { this(systemSessionProperties.getSessionProperties()); }
/** * Enforce timeout at the query level */ public void enforceQueryMaxRunTimeLimits() { for (QueryExecution query : queries.values()) { if (query.getState().isDone()) { continue; } Duration queryMaxRunTime = SystemSessionProperties.getQueryMaxRunTime(query.getSession()); DateTime executionStartTime = query.getQueryInfo().getQueryStats().getCreateTime(); if (executionStartTime.plus(queryMaxRunTime.toMillis()).isBeforeNow()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded maximum time limit of " + queryMaxRunTime)); } } }
columns, getTypes(rewrittenProjections, expressionTypes), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session)); pageProcessor, getTypes(rewrittenProjections, expressionTypes), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session));
Assignments.builder() .putIdentities(node.getSource().getOutputSymbols()) .put(partitionCountSymbol, new LongLiteral(Integer.toString(getHashPartitionCount(context.getSession())))) .putAll(envelopeAssignments.build()) .build()),
@Inject public SessionPropertyManager(SystemSessionProperties systemSessionProperties) { this(systemSessionProperties.getSessionProperties()); }
@Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { requireNonNull(plan, "plan is null"); requireNonNull(session, "session is null"); requireNonNull(types, "types is null"); requireNonNull(symbolAllocator, "symbolAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); if (SystemSessionProperties.isOptimizeHashGenerationEnabled(session)) { PlanWithProperties result = plan.accept(new Rewriter(idAllocator, symbolAllocator, types), new HashComputationSet()); return result.getNode(); } return plan; }
protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) { ImmutableMap.Builder<Symbol, Type> symbolTypes = ImmutableMap.builder(); ImmutableMap.Builder<Symbol, Integer> symbolToInputMapping = ImmutableMap.builder(); ImmutableList.Builder<PageProjection> projections = ImmutableList.builder(); for (int channel = 0; channel < types.size(); channel++) { Symbol symbol = new Symbol("h" + channel); symbolTypes.put(symbol, types.get(channel)); symbolToInputMapping.put(symbol, channel); projections.add(new InputPageProjection(channel, types.get(channel))); } Optional<Expression> hashExpression = HashGenerationOptimizer.getHashExpression(ImmutableList.copyOf(symbolTypes.build().keySet())); verify(hashExpression.isPresent()); projections.add(new InterpretedPageProjection( hashExpression.get(), TypeProvider.copyOf(symbolTypes.build()), symbolToInputMapping.build(), localQueryRunner.getMetadata(), localQueryRunner.getSqlParser(), session)); return new FilterAndProjectOperator.FilterAndProjectOperatorFactory( operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session)); }
@Test public void testTooManyGroupingElements() { Session session = testSessionBuilder(new SessionPropertyManager(new SystemSessionProperties( new QueryManagerConfig(), new TaskManagerConfig(), new MemoryManagerConfig(), new FeaturesConfig().setMaxGroupingSets(2048)))).build(); analyze(session, "SELECT a, b, c, d, e, f, g, h, i, j, k, SUM(l)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l)\n" + "GROUP BY CUBE (a, b, c, d, e, f), CUBE (g, h, i, j, k)"); assertFails(session, TOO_MANY_GROUPING_SETS, "line 3:10: GROUP BY has 4096 grouping sets but can contain at most 2048", "SELECT a, b, c, d, e, f, g, h, i, j, k, l, SUM(m)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l, m)\n" + "GROUP BY CUBE (a, b, c, d, e, f), CUBE (g, h, i, j, k, l)"); assertFails(session, TOO_MANY_GROUPING_SETS, format("line 3:10: GROUP BY has more than %s grouping sets but can contain at most 2048", Integer.MAX_VALUE), "SELECT a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae, SUM(af)" + "FROM (VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, " + "17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32))\n" + "t (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae, af)\n" + "GROUP BY CUBE (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, " + "q, r, s, t, u, v, x, w, y, z, aa, ab, ac, ad, ae)"); }
public NodePartitionMap getNodePartitionMap(Session session, NodeScheduler nodeScheduler) { NodeSelector nodeSelector = nodeScheduler.createNodeSelector(null); List<Node> nodes; if (partitioning == SystemPartitioning.COORDINATOR_ONLY) { nodes = ImmutableList.of(nodeSelector.selectCurrentNode()); } else if (partitioning == SystemPartitioning.SINGLE) { nodes = nodeSelector.selectRandomNodes(1); } else if (partitioning == SystemPartitioning.FIXED) { nodes = nodeSelector.selectRandomNodes(getHashPartitionCount(session)); } else { throw new IllegalArgumentException("Unsupported plan distribution " + partitioning); } checkCondition(!nodes.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); ImmutableMap.Builder<Integer, Node> partitionToNode = ImmutableMap.builder(); for (int i = 0; i < nodes.size(); i++) { Node node = nodes.get(i); partitionToNode.put(i, node); } return new NodePartitionMap(partitionToNode.build(), split -> { throw new UnsupportedOperationException("System distribution does not support source splits"); }); }
@Override public SqlQueryExecution createQueryExecution(QueryId queryId, String query, Session session, Statement statement) { String executionPolicyName = SystemSessionProperties.getExecutionPolicy(session); ExecutionPolicy executionPolicy = executionPolicies.get(executionPolicyName); checkArgument(executionPolicy != null, "No execution policy %s", executionPolicy); return new SqlQueryExecution( queryId, query, session, locationFactory.createQueryLocation(queryId), statement, transactionManager, metadata, accessControl, sqlParser, splitManager, nodePartitioningManager, nodeScheduler, planOptimizers, remoteTaskFactory, locationFactory, scheduleSplitBatchSize, experimentalSyntaxEnabled, executor, nodeTaskMap, queryExplainer, executionPolicy); } }