/** * Attempts to rewrite an Expression in terms of the symbols allowed by the symbol scope * given the known equalities. Returns null if unsuccessful. * This method allows rewriting non-deterministic expressions. */ public Expression rewriteExpressionAllowNonDeterministic(Expression expression, Predicate<Symbol> symbolScope) { return rewriteExpression(expression, symbolScope, true); }
Expression rewrittenConjunct = allInference.rewriteExpression(conjunct, in(filteringSourceSymbols)); Expression rewritten = allInference.rewriteExpression(conjunct, in(sourceSymbols)); if (rewritten != null) { sourceConjuncts.add(rewritten); Expression rewritten = allInference.rewriteExpression(conjunct, in(filteringSourceSymbols)); if (rewritten != null) { filteringSourceConjuncts.add(rewritten);
/** * Attempts to rewrite an Expression in terms of the symbols allowed by the symbol scope * given the known equalities. Returns null if unsuccessful. * This method checks if rewritten expression is non-deterministic. */ public Expression rewriteExpression(Expression expression, Predicate<Symbol> symbolScope) { checkArgument(isDeterministic(expression), "Only deterministic expressions may be considered for rewrite"); return rewriteExpression(expression, symbolScope, true); }
private List<Expression> getJoinPredicates(Set<Symbol> leftSymbols, Set<Symbol> rightSymbols) { ImmutableList.Builder<Expression> joinPredicatesBuilder = ImmutableList.builder(); // This takes all conjuncts that were part of allFilters that // could not be used for equality inference. // If they use both the left and right symbols, we add them to the list of joinPredicates stream(nonInferrableConjuncts(allFilter)) .map(conjunct -> allFilterInference.rewriteExpression(conjunct, symbol -> leftSymbols.contains(symbol) || rightSymbols.contains(symbol))) .filter(Objects::nonNull) // filter expressions that contain only left or right symbols .filter(conjunct -> allFilterInference.rewriteExpression(conjunct, leftSymbols::contains) == null) .filter(conjunct -> allFilterInference.rewriteExpression(conjunct, rightSymbols::contains) == null) .forEach(joinPredicatesBuilder::add); // create equality inference on available symbols // TODO: make generateEqualitiesPartitionedBy take left and right scope List<Expression> joinEqualities = allFilterInference.generateEqualitiesPartitionedBy(symbol -> leftSymbols.contains(symbol) || rightSymbols.contains(symbol)).getScopeEqualities(); EqualityInference joinInference = createEqualityInference(joinEqualities.toArray(new Expression[0])); joinPredicatesBuilder.addAll(joinInference.generateEqualitiesPartitionedBy(in(leftSymbols)).getScopeStraddlingEqualities()); return joinPredicatesBuilder.build(); }
private static Expression pullExpressionThroughSymbols(Expression expression, Collection<Symbol> symbols) { EqualityInference equalityInference = createEqualityInference(expression); ImmutableList.Builder<Expression> effectiveConjuncts = ImmutableList.builder(); for (Expression conjunct : EqualityInference.nonInferrableConjuncts(expression)) { if (DeterminismEvaluator.isDeterministic(conjunct)) { Expression rewritten = equalityInference.rewriteExpression(conjunct, in(symbols)); if (rewritten != null) { effectiveConjuncts.add(rewritten); } } } effectiveConjuncts.addAll(equalityInference.generateEqualitiesPartitionedBy(in(symbols)).getScopeEqualities()); return combineConjuncts(effectiveConjuncts.build()); } }
Expression scopeRewritten = rewriteExpression(expression, symbolScope, false); if (scopeRewritten != null) { scopeExpressions.add(scopeRewritten); Expression scopeComplementRewritten = rewriteExpression(expression, not(symbolScope), false); if (scopeComplementRewritten != null) { scopeComplementExpressions.add(scopeComplementRewritten);
private JoinEnumerationResult getJoinSource(LinkedHashSet<PlanNode> nodes, List<Symbol> outputSymbols) { if (nodes.size() == 1) { PlanNode planNode = getOnlyElement(nodes); ImmutableList.Builder<Expression> predicates = ImmutableList.builder(); predicates.addAll(allFilterInference.generateEqualitiesPartitionedBy(outputSymbols::contains).getScopeEqualities()); stream(nonInferrableConjuncts(allFilter)) .map(conjunct -> allFilterInference.rewriteExpression(conjunct, outputSymbols::contains)) .filter(Objects::nonNull) .forEach(predicates::add); Expression filter = combineConjuncts(predicates.build()); if (!TRUE_LITERAL.equals(filter)) { planNode = new FilterNode(idAllocator.getNextId(), planNode, filter); } return createJoinEnumerationResult(planNode); } return chooseJoinOrder(nodes, outputSymbols); }
Expression leftRewrittenConjunct = allInference.rewriteExpression(conjunct, in(leftSymbols)); if (leftRewrittenConjunct != null) { leftPushDownConjuncts.add(leftRewrittenConjunct); Expression rightRewrittenConjunct = allInference.rewriteExpression(conjunct, not(in(leftSymbols))); if (rightRewrittenConjunct != null) { rightPushDownConjuncts.add(rightRewrittenConjunct); Expression rewritten = allInference.rewriteExpression(conjunct, in(leftSymbols)); if (rewritten != null) { leftPushDownConjuncts.add(rewritten); Expression rewritten = allInference.rewriteExpression(conjunct, not(in(leftSymbols))); if (rewritten != null) { rightPushDownConjuncts.add(rewritten); Expression leftRewritten = allInference.rewriteExpression(conjunct, in(leftSymbols)); if (leftRewritten != null) { leftPushDownConjuncts.add(leftRewritten); Expression rightRewritten = allInference.rewriteExpression(conjunct, not(in(leftSymbols))); if (rightRewritten != null) { rightPushDownConjuncts.add(rightRewritten);
Expression outerRewritten = outerInference.rewriteExpression(conjunct, in(outerSymbols)); if (outerRewritten != null) { outerPushdownConjuncts.add(outerRewritten); Expression innerRewritten = potentialNullSymbolInference.rewriteExpression(outerRewritten, not(in(outerSymbols))); if (innerRewritten != null) { innerPushdownConjuncts.add(innerRewritten); Expression rewritten = potentialNullSymbolInference.rewriteExpression(conjunct, not(in(outerSymbols))); if (rewritten != null) { innerPushdownConjuncts.add(rewritten); Expression innerRewritten = potentialNullSymbolInference.rewriteExpression(conjunct, not(in(outerSymbols))); if (innerRewritten != null) { innerPushdownConjuncts.add(innerRewritten);
private Set<Expression> normalizeConjuncts(Expression predicate) { // Normalize the predicate by identity so that the EqualityInference will produce stable rewrites in this test // and thereby produce comparable Sets of conjuncts from this method. predicate = expressionNormalizer.normalize(predicate); // Equality inference rewrites and equality generation will always be stable across multiple runs in the same JVM EqualityInference inference = EqualityInference.createEqualityInference(predicate); Set<Expression> rewrittenSet = new HashSet<>(); for (Expression expression : EqualityInference.nonInferrableConjuncts(predicate)) { Expression rewritten = inference.rewriteExpression(expression, Predicates.alwaysTrue()); Preconditions.checkState(rewritten != null, "Rewrite with full symbol scope should always be possible"); rewrittenSet.add(rewritten); } rewrittenSet.addAll(inference.generateEqualitiesPartitionedBy(Predicates.alwaysTrue()).getScopeEqualities()); return rewrittenSet; }
Expression rewrittenConjunct = equalityInference.rewriteExpression(conjunct, in(node.getReplicateSymbols())); if (rewrittenConjunct != null) { pushdownConjuncts.add(rewrittenConjunct);
@Test public void testUnrewritable() { EqualityInference.Builder builder = new EqualityInference.Builder(); addEquality("a1", "b1", builder); addEquality("a2", "b2", builder); EqualityInference inference = builder.build(); assertNull(inference.rewriteExpression(someExpression("a1", "a2"), matchesSymbols("b1", "c1"))); assertNull(inference.rewriteExpression(someExpression("c1", "c2"), matchesSymbols("a1", "a2"))); }
@Test public void testTriviallyRewritable() { EqualityInference.Builder builder = new EqualityInference.Builder(); Expression expression = builder.build() .rewriteExpression(someExpression("a1", "a2"), matchesSymbols("a1", "a2")); assertEquals(expression, someExpression("a1", "a2")); }
Expression rewrittenConjunct = equalityInference.rewriteExpression(conjunct, in(node.getGroupingKeys())); if (rewrittenConjunct != null) { pushdownConjuncts.add(rewrittenConjunct);
@Test public void testExtractInferrableEqualities() { EqualityInference inference = new EqualityInference.Builder() .extractInferenceCandidates(ExpressionUtils.and(equals("a1", "b1"), equals("b1", "c1"), someExpression("c1", "d1"))) .build(); // Able to rewrite to c1 due to equalities assertEquals(nameReference("c1"), inference.rewriteExpression(nameReference("a1"), matchesSymbols("c1"))); // But not be able to rewrite to d1 which is not connected via equality assertNull(inference.rewriteExpression(nameReference("a1"), matchesSymbols("d1"))); }
@Test public void testParseEqualityExpression() { EqualityInference inference = new EqualityInference.Builder() .addEquality(equals("a1", "b1")) .addEquality(equals("a1", "c1")) .addEquality(equals("c1", "a1")) .build(); Expression expression = inference.rewriteExpression(someExpression("a1", "b1"), matchesSymbols("c1")); assertEquals(expression, someExpression("c1", "c1")); }
@Test public void testSubExpressionRewrites() { EqualityInference.Builder builder = new EqualityInference.Builder(); builder.addEquality(nameReference("a1"), add("b", "c")); // a1 = b + c builder.addEquality(nameReference("a2"), multiply(nameReference("b"), add("b", "c"))); // a2 = b * (b + c) builder.addEquality(nameReference("a3"), multiply(nameReference("a1"), add("b", "c"))); // a3 = a1 * (b + c) EqualityInference inference = builder.build(); // Expression (b + c) should get entirely rewritten as a1 assertEquals(inference.rewriteExpression(add("b", "c"), symbolBeginsWith("a")), nameReference("a1")); // Only the sub-expression (b + c) should get rewritten in terms of a* assertEquals(inference.rewriteExpression(multiply(nameReference("ax"), add("b", "c")), symbolBeginsWith("a")), multiply(nameReference("ax"), nameReference("a1"))); // To be compliant, could rewrite either the whole expression, or just the sub-expression. Rewriting larger expressions are preferred assertEquals(inference.rewriteExpression(multiply(nameReference("a1"), add("b", "c")), symbolBeginsWith("a")), nameReference("a3")); }
inference.rewriteExpression(someExpression("a1", "a2"), matchesSymbols("d1", "d2")), someExpression("d1", "d2")); inference.rewriteExpression(someExpression("a1", "c1"), matchesSymbols("b1")), someExpression("b1", "b1")); inference.rewriteExpression(someExpression("a1", "a2"), matchesSymbols("b1", "d2", "c3")), someExpression("b1", "d2")); inference.rewriteExpression(someExpression("a2", "b2"), matchesSymbols("c2", "d2")), someExpression(canonical, canonical));
@Test public void testConstantEqualities() { EqualityInference.Builder builder = new EqualityInference.Builder(); addEquality("a1", "b1", builder); addEquality("b1", "c1", builder); builder.addEquality(nameReference("c1"), number(1)); EqualityInference inference = builder.build(); // Should always prefer a constant if available (constant is part of all scopes) assertEquals(inference.rewriteExpression(nameReference("a1"), matchesSymbols("a1", "b1")), number(1)); // All scope equalities should utilize the constant if possible EqualityInference.EqualityPartition equalityPartition = inference.generateEqualitiesPartitionedBy(matchesSymbols("a1", "b1")); assertEquals(equalitiesAsSets(equalityPartition.getScopeEqualities()), set(set(nameReference("a1"), number(1)), set(nameReference("b1"), number(1)))); assertEquals(equalitiesAsSets(equalityPartition.getScopeComplementEqualities()), set(set(nameReference("c1"), number(1)))); // There should be no scope straddling equalities as the full set of equalities should be already represented by the scope and inverse scope assertTrue(equalityPartition.getScopeStraddlingEqualities().isEmpty()); }
@Test public void testExtractInferrableEqualities() throws Exception { EqualityInference inference = new EqualityInference.Builder() .extractInferenceCandidates(ExpressionUtils.and(equals("a1", "b1"), equals("b1", "c1"), someExpression("c1", "d1"))) .build(); // Able to rewrite to c1 due to equalities assertEquals(nameReference("c1"), inference.rewriteExpression(nameReference("a1"), matchesSymbols("c1"))); // But not be able to rewrite to d1 which is not connected via equality assertNull(inference.rewriteExpression(nameReference("a1"), matchesSymbols("d1"))); }