/** * Attempts to rewrite an Expression in terms of the symbols allowed by the symbol scope * given the known equalities. Returns null if unsuccessful. * This method checks if rewritten expression is non-deterministic. */ public Expression rewriteExpression(Expression expression, Predicate<Symbol> symbolScope) { checkArgument(isDeterministic(expression), "Only deterministic expressions may be considered for rewrite"); return rewriteExpression(expression, symbolScope, true); }
public Builder addEquality(Expression expression1, Expression expression2) { checkArgument(!expression1.equals(expression2), "Need to provide equality between different expressions"); checkArgument(isDeterministic(expression1), "Expression must be deterministic: " + expression1); checkArgument(isDeterministic(expression2), "Expression must be deterministic: " + expression2); equalities.findAndUnion(expression1, expression2); return this; }
/** * Removes duplicate deterministic expressions. Preserves the relative order * of the expressions in the list. */ private static List<Expression> removeDuplicates(List<Expression> expressions) { Set<Expression> seen = new HashSet<>(); ImmutableList.Builder<Expression> result = ImmutableList.builder(); for (Expression expression : expressions) { if (!DeterminismEvaluator.isDeterministic(expression)) { result.add(expression); } else if (!seen.contains(expression)) { result.add(expression); seen.add(expression); } } return result.build(); }
private boolean canConvertOuterToInner(List<Symbol> innerSymbolsForOuterJoin, Expression inheritedPredicate) { Set<Symbol> innerSymbols = ImmutableSet.copyOf(innerSymbolsForOuterJoin); for (Expression conjunct : extractConjuncts(inheritedPredicate)) { if (DeterminismEvaluator.isDeterministic(conjunct)) { // Ignore a conjunct for this test if we can not deterministically get responses from it Object response = nullInputEvaluator(innerSymbols, conjunct); if (response == null || response instanceof NullLiteral || Boolean.FALSE.equals(response)) { // If there is a single conjunct that returns FALSE or NULL given all NULL inputs for the inner side symbols of an outer join // then this conjunct removes all effects of the outer join, and effectively turns this into an equivalent of an inner join. // So, let's just rewrite this join as an INNER join return true; } } } return false; }
private static Expression pullExpressionThroughSymbols(Expression expression, Collection<Symbol> symbols) { EqualityInference equalityInference = createEqualityInference(expression); ImmutableList.Builder<Expression> effectiveConjuncts = ImmutableList.builder(); for (Expression conjunct : EqualityInference.nonInferrableConjuncts(expression)) { if (DeterminismEvaluator.isDeterministic(conjunct)) { Expression rewritten = equalityInference.rewriteExpression(conjunct, in(symbols)); if (rewritten != null) { effectiveConjuncts.add(rewritten); } } } effectiveConjuncts.addAll(equalityInference.generateEqualitiesPartitionedBy(in(symbols)).getScopeEqualities()); return combineConjuncts(effectiveConjuncts.build()); } }
private static Predicate<Expression> joinEqualityExpression(final Collection<Symbol> leftSymbols) { return expression -> { // At this point in time, our join predicates need to be deterministic if (isDeterministic(expression) && expression instanceof ComparisonExpression) { ComparisonExpression comparison = (ComparisonExpression) expression; if (comparison.getOperator() == ComparisonExpression.Operator.EQUAL) { Set<Symbol> symbols1 = SymbolsExtractor.extractUnique(comparison.getLeft()); Set<Symbol> symbols2 = SymbolsExtractor.extractUnique(comparison.getRight()); if (symbols1.isEmpty() || symbols2.isEmpty()) { return false; } return (Iterables.all(symbols1, in(leftSymbols)) && Iterables.all(symbols2, not(in(leftSymbols)))) || (Iterables.all(symbols2, in(leftSymbols)) && Iterables.all(symbols1, not(in(leftSymbols)))); } } return false; }; }
public InterpretedPageFilter( Expression expression, TypeProvider symbolTypes, Map<Symbol, Integer> symbolToInputMappings, Metadata metadata, SqlParser sqlParser, Session session) { SymbolToInputParameterRewriter rewriter = new SymbolToInputParameterRewriter(symbolTypes, symbolToInputMappings); Expression rewritten = rewriter.rewrite(expression); this.inputChannels = new InputChannels(rewriter.getInputChannels()); this.deterministic = DeterminismEvaluator.isDeterministic(expression); // analyze rewritten expression so we can know the type of every expression in the tree List<Type> inputTypes = rewriter.getInputTypes(); ImmutableMap.Builder<Integer, Type> parameterTypes = ImmutableMap.builder(); for (int parameter = 0; parameter < inputTypes.size(); parameter++) { Type type = inputTypes.get(parameter); parameterTypes.put(parameter, type); } Map<NodeRef<Expression>, Type> expressionTypes = getExpressionTypesFromInput(session, metadata, sqlParser, parameterTypes.build(), rewritten, emptyList(), WarningCollector.NOOP); this.evaluator = ExpressionInterpreter.expressionInterpreter(rewritten, metadata, session, expressionTypes); }
private void flattenNode(PlanNode node, int limit) { PlanNode resolved = lookup.resolve(node); // (limit - 2) because you need to account for adding left and right side if (!(resolved instanceof JoinNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } JoinNode joinNode = (JoinNode) resolved; if (joinNode.getType() != INNER || !isDeterministic(joinNode.getFilter().orElse(TRUE_LITERAL)) || joinNode.getDistributionType().isPresent()) { sources.add(node); return; } // we set the left limit to limit - 1 to account for the node on the right flattenNode(joinNode.getLeft(), limit - 1); flattenNode(joinNode.getRight(), limit); joinNode.getCriteria().stream() .map(EquiJoinClause::toExpression) .forEach(filters::add); joinNode.getFilter().ifPresent(filters::add); }
@Override public PlanNode visitProject(ProjectNode node, RewriteContext<Expression> context) { Set<Symbol> deterministicSymbols = node.getAssignments().entrySet().stream() .filter(entry -> DeterminismEvaluator.isDeterministic(entry.getValue())) .map(Map.Entry::getKey) .collect(Collectors.toSet()); Predicate<Expression> deterministic = conjunct -> SymbolsExtractor.extractUnique(conjunct).stream() .allMatch(deterministicSymbols::contains); Map<Boolean, List<Expression>> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(deterministic)); // Push down conjuncts from the inherited predicate that only depend on deterministic assignments with // certain limitations. List<Expression> deterministicConjuncts = conjuncts.get(true); // We partition the expressions in the deterministicConjuncts into two lists, and only inline the // expressions that are in the inlining targets list. Map<Boolean, List<Expression>> inlineConjuncts = deterministicConjuncts.stream() .collect(Collectors.partitioningBy(expression -> isInliningCandidate(expression, node))); List<Expression> inlinedDeterministicConjuncts = inlineConjuncts.get(true).stream() .map(entry -> inlineSymbols(node.getAssignments().getMap(), entry)) .collect(Collectors.toList()); PlanNode rewrittenNode = context.defaultRewrite(node, combineConjuncts(inlinedDeterministicConjuncts)); // All deterministic conjuncts that contains non-inlining targets, and non-deterministic conjuncts, // if any, will be in the filter node. List<Expression> nonInliningConjuncts = inlineConjuncts.get(false); nonInliningConjuncts.addAll(conjuncts.get(false)); if (!nonInliningConjuncts.isEmpty()) { rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, combineConjuncts(nonInliningConjuncts)); } return rewrittenNode; }
@Test public void testSanity() { assertFalse(DeterminismEvaluator.isDeterministic(function("rand"))); assertFalse(DeterminismEvaluator.isDeterministic(function("random"))); assertFalse(DeterminismEvaluator.isDeterministic(function("shuffle"))); assertTrue(DeterminismEvaluator.isDeterministic(function("abs", input("symbol")))); assertFalse(DeterminismEvaluator.isDeterministic(function("abs", function("rand")))); assertTrue(DeterminismEvaluator.isDeterministic(function("abs", function("abs", input("symbol"))))); }
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Expression> context) { List<Symbol> partitionSymbols = node.getPartitionBy(); // TODO: This could be broader. We can push down conjucts if they are constant for all rows in a window partition. // The simplest way to guarantee this is if the conjucts are deterministic functions of the partitioning symbols. // This can leave out cases where they're both functions of some set of common expressions and the partitioning // function is injective, but that's a rare case. The majority of window nodes are expected to be partitioned by // pre-projected symbols. Predicate<Expression> isSupported = conjunct -> DeterminismEvaluator.isDeterministic(conjunct) && SymbolsExtractor.extractUnique(conjunct).stream() .allMatch(partitionSymbols::contains); Map<Boolean, List<Expression>> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(isSupported)); PlanNode rewrittenNode = context.defaultRewrite(node, combineConjuncts(conjuncts.get(true))); if (!conjuncts.get(false).isEmpty()) { rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, combineConjuncts(conjuncts.get(false))); } return rewrittenNode; }
public InterpretedPageProjection( Expression expression, TypeProvider symbolTypes, Map<Symbol, Integer> symbolToInputMappings, Metadata metadata, SqlParser sqlParser, Session session) { SymbolToInputParameterRewriter rewriter = new SymbolToInputParameterRewriter(symbolTypes, symbolToInputMappings); Expression rewritten = rewriter.rewrite(expression); this.inputChannels = new InputChannels(rewriter.getInputChannels()); this.deterministic = DeterminismEvaluator.isDeterministic(expression); // analyze rewritten expression so we can know the type of every expression in the tree List<Type> inputTypes = rewriter.getInputTypes(); ImmutableMap.Builder<Integer, Type> parameterTypes = ImmutableMap.builder(); for (int parameter = 0; parameter < inputTypes.size(); parameter++) { Type type = inputTypes.get(parameter); parameterTypes.put(parameter, type); } Map<NodeRef<Expression>, Type> expressionTypes = getExpressionTypesFromInput(session, metadata, sqlParser, parameterTypes.build(), rewritten, emptyList(), WarningCollector.NOOP); this.evaluator = ExpressionInterpreter.expressionInterpreter(rewritten, metadata, session, expressionTypes); blockBuilder = evaluator.getType().createBlockBuilder(null, 1); }
if (!DeterminismEvaluator.isDeterministic(expression)) {
else if (DeterminismEvaluator.isDeterministic(expression) && !(expression instanceof NullLiteral)) {
.distinct(), expressionValues.stream() .filter((expression -> !DeterminismEvaluator.isDeterministic(expression)))) .collect(toImmutableList()); return new InPredicate(toExpression(value, type), new InListExpression(simplifiedExpressionValues));
private void verifySelectDistinct(QuerySpecification node, List<Expression> outputExpressions) { for (SortItem item : node.getOrderBy().get().getSortItems()) { Expression expression = item.getSortKey(); if (expression instanceof LongLiteral) { continue; } Expression rewrittenOrderByExpression = ExpressionTreeRewriter.rewriteWith(new OrderByExpressionRewriter(extractNamedOutputExpressions(node.getSelect())), expression); int index = outputExpressions.indexOf(rewrittenOrderByExpression); if (index == -1) { throw new SemanticException(ORDER_BY_MUST_BE_IN_SELECT, node.getSelect(), "For SELECT DISTINCT, ORDER BY expressions must appear in select list"); } if (!isDeterministic(expression)) { throw new SemanticException(NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT, expression, "Non deterministic ORDER BY expression is not supported with SELECT DISTINCT"); } } }
/** * Determines whether an Expression may be successfully applied to the equality inference */ public static Predicate<Expression> isInferenceCandidate() { return expression -> { expression = normalizeInPredicateToEquality(expression); if (expression instanceof ComparisonExpression && isDeterministic(expression) && !mayReturnNullOnNonNullInput(expression)) { ComparisonExpression comparison = (ComparisonExpression) expression; if (comparison.getOperator() == ComparisonExpression.Operator.EQUAL) { // We should only consider equalities that have distinct left and right components return !comparison.getLeft().equals(comparison.getRight()); } } return false; }; }
DeterminismEvaluator.isDeterministic(leftResult.getRemainingExpression())) {
/** * Attempts to rewrite an Expression in terms of the symbols allowed by the symbol scope * given the known equalities. Returns null if unsuccessful. */ public Expression rewriteExpression(Expression expression, Predicate<Symbol> symbolScope) { checkArgument(DeterminismEvaluator.isDeterministic(expression), "Only deterministic expressions may be considered for rewrite"); return rewriteExpression(expression, symbolScope, true); }
public Builder addEquality(Expression expression1, Expression expression2) { checkArgument(!expression1.equals(expression2), "Need to provide equality between different expressions"); checkArgument(DeterminismEvaluator.isDeterministic(expression1), "Expression must be deterministic: " + expression1); checkArgument(DeterminismEvaluator.isDeterministic(expression2), "Expression must be deterministic: " + expression2); equalities.findAndUnion(expression1, expression2); return this; }