private List<JoinNode.EquiJoinClause> canonicalizeJoinCriteria(List<JoinNode.EquiJoinClause> criteria) { ImmutableList.Builder<JoinNode.EquiJoinClause> builder = ImmutableList.builder(); for (JoinNode.EquiJoinClause clause : criteria) { builder.add(new JoinNode.EquiJoinClause(canonicalize(clause.getLeft()), canonicalize(clause.getRight()))); } return builder.build(); }
private JoinNode joinNode(PlanNode left, PlanNode right, String... symbols) { checkArgument(symbols.length % 2 == 0); ImmutableList.Builder<JoinNode.EquiJoinClause> criteria = ImmutableList.builder(); for (int i = 0; i < symbols.length; i += 2) { criteria.add(new JoinNode.EquiJoinClause(new Symbol(symbols[i]), new Symbol(symbols[i + 1]))); } return new JoinNode( idAllocator.getNextId(), JoinNode.Type.INNER, left, right, criteria.build(), ImmutableList.<Symbol>builder() .addAll(left.getOutputSymbols()) .addAll(right.getOutputSymbols()) .build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); }
@Test public void testRetainDistributionType() { assertDetermineJoinDistributionType() .on(p -> p.join( INNER, p.values(ImmutableList.of(p.symbol("A1")), ImmutableList.of(expressions("10"), expressions("11"))), p.values(ImmutableList.of(p.symbol("B1")), ImmutableList.of(expressions("50"), expressions("11"))), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(DistributionType.REPLICATED))) .doesNotFire(); }
@Test public void testReplicateScalar() { assertDetermineJoinDistributionType() .on(p -> p.join( INNER, p.values(ImmutableList.of(p.symbol("A1")), ImmutableList.of(expressions("10"), expressions("11"))), p.enforceSingleRow( p.values(ImmutableList.of(p.symbol("B1")), ImmutableList.of(expressions("50"), expressions("11")))), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.PARTITIONED.name()) .matches(join( INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(DistributionType.REPLICATED), values(ImmutableMap.of("A1", 0)), enforceSingleRow(values(ImmutableMap.of("B1", 0))))); }
@Test public void testDoesNotFireWhenAggregationDoesNotHaveSymbols() { tester().assertThat(new PushAggregationThroughOuterJoin()) .on(p -> p.aggregation(ab -> ab .source(p.join( JoinNode.Type.LEFT, p.values(ImmutableList.of(p.symbol("COL1")), ImmutableList.of(expressions("10"))), p.values(ImmutableList.of(p.symbol("COL2")), ImmutableList.of(expressions("20"))), ImmutableList.of(new JoinNode.EquiJoinClause(new Symbol("COL1"), new Symbol("COL2"))), ImmutableList.of(new Symbol("COL1"), new Symbol("COL2")), Optional.empty(), Optional.empty(), Optional.empty())) .addAggregation(new Symbol("SUM"), PlanBuilder.expression("sum(COL1)"), ImmutableList.of(DOUBLE)) .singleGroupingSet(new Symbol("COL1")))) .doesNotFire(); } }
private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJoin, Type joinType) { assertDetermineJoinDistributionType() .on(p -> p.join( joinType, p.values(ImmutableList.of(p.symbol("A1")), ImmutableList.of(expressions("10"), expressions("11"))), p.values(ImmutableList.of(p.symbol("B1")), ImmutableList.of(expressions("50"), expressions("11"))), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, sessionDistributedJoin.name()) .matches(join( joinType, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(DistributionType.PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0)))); }
private void testDetermineDistributionType(JoinDistributionType sessionDistributedJoin, Type joinType, DistributionType expectedDistribution) { assertDetermineJoinDistributionType() .on(p -> p.join( joinType, p.values(ImmutableList.of(p.symbol("A1")), ImmutableList.of(expressions("10"), expressions("11"))), p.values(ImmutableList.of(p.symbol("B1")), ImmutableList.of(expressions("50"), expressions("11"))), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())) .setSystemProperty(JOIN_DISTRIBUTION_TYPE, sessionDistributedJoin.name()) .matches(join( joinType, ImmutableList.of(equiJoinClause("B1", "A1")), Optional.empty(), Optional.of(expectedDistribution), values(ImmutableMap.of("B1", 0)), values(ImmutableMap.of("A1", 0)))); }
@Test public void testRightJoinComplementStats() { PlanNodeStatsEstimate expected = NORMALIZER.normalize( planNodeStats( RIGHT_ROWS_COUNT * RIGHT_JOIN_COLUMN_NULLS, symbolStatistics(RIGHT_JOIN_COLUMN, NaN, NaN, 1.0, 0), RIGHT_OTHER_COLUMN_STATS), TYPES); PlanNodeStatsEstimate actual = JOIN_STATS_RULE.calculateJoinComplementStats( Optional.empty(), ImmutableList.of(new EquiJoinClause(new Symbol(RIGHT_JOIN_COLUMN), new Symbol(LEFT_JOIN_COLUMN))), RIGHT_STATS, LEFT_STATS, TYPES); assertEquals(actual, expected); }
@Test public void testDoesNotFireWhenGroupingOnInner() { tester().assertThat(new PushAggregationThroughOuterJoin()) .on(p -> p.aggregation(ab -> ab .source(p.join(JoinNode.Type.LEFT, p.values(ImmutableList.of(p.symbol("COL1")), ImmutableList.of(expressions("10"))), p.values(new Symbol("COL2"), new Symbol("COL3")), ImmutableList.of(new JoinNode.EquiJoinClause(new Symbol("COL1"), new Symbol("COL2"))), ImmutableList.of(new Symbol("COL1"), new Symbol("COL2")), Optional.empty(), Optional.empty(), Optional.empty())) .addAggregation(new Symbol("AVG"), PlanBuilder.expression("avg(COL2)"), ImmutableList.of(DOUBLE)) .singleGroupingSet(new Symbol("COL1"), new Symbol("COL3")))) .doesNotFire(); }
private Function<PlanBuilder, PlanNode> crossJoinAndJoin(JoinNode.Type secondJoinType) { return p -> { Symbol axSymbol = p.symbol("axSymbol"); Symbol bySymbol = p.symbol("bySymbol"); Symbol cxSymbol = p.symbol("cxSymbol"); Symbol cySymbol = p.symbol("cySymbol"); // (a inner join b) inner join c on c.x = a.x and c.y = b.y return p.join(INNER, p.join(secondJoinType, p.values(axSymbol), p.values(bySymbol)), p.values(cxSymbol, cySymbol), new EquiJoinClause(cxSymbol, axSymbol), new EquiJoinClause(cySymbol, bySymbol)); }; }
@Test public void testDoesNotFireWithNoStats() { assertReorderJoins() .on(p -> p.join( INNER, p.values(new PlanNodeId("valuesA"), ImmutableList.of(p.symbol("A1")), TWO_ROWS), p.values(new PlanNodeId("valuesB"), p.symbol("B1")), ImmutableList.of(new EquiJoinClause(p.symbol("A1"), p.symbol("B1"))), ImmutableList.of(p.symbol("A1")), Optional.empty())) .overrideStats("valuesA", PlanNodeStatsEstimate.unknown()) .doesNotFire(); }
@Test public void testLeftJoinComplementStatsWithMultipleClauses() { PlanNodeStatsEstimate expected = planNodeStats( LEFT_ROWS_COUNT * (LEFT_JOIN_COLUMN_NULLS + LEFT_JOIN_COLUMN_NON_NULLS / 4), symbolStatistics(LEFT_JOIN_COLUMN, 0.0, 20.0, LEFT_JOIN_COLUMN_NULLS / (LEFT_JOIN_COLUMN_NULLS + LEFT_JOIN_COLUMN_NON_NULLS / 4), 5), LEFT_OTHER_COLUMN_STATS) .mapOutputRowCount(rowCount -> rowCount / UNKNOWN_FILTER_COEFFICIENT); PlanNodeStatsEstimate actual = JOIN_STATS_RULE.calculateJoinComplementStats( Optional.empty(), ImmutableList.of(new EquiJoinClause(new Symbol(LEFT_JOIN_COLUMN), new Symbol(RIGHT_JOIN_COLUMN)), new EquiJoinClause(new Symbol(LEFT_OTHER_COLUMN), new Symbol(RIGHT_OTHER_COLUMN))), LEFT_STATS, RIGHT_STATS, TYPES); assertEquals(actual, expected); }
private void assertJoinStats(JoinNode.Type joinType, String leftJoinColumn, String leftOtherColumn, String rightJoinColumn, String rightOtherColumn, PlanNodeStatsEstimate leftStats, PlanNodeStatsEstimate rightStats, PlanNodeStatsEstimate resultStats) { tester().assertStatsFor(pb -> { Symbol leftJoinColumnSymbol = pb.symbol(leftJoinColumn, BIGINT); Symbol rightJoinColumnSymbol = pb.symbol(rightJoinColumn, DOUBLE); Symbol leftOtherColumnSymbol = pb.symbol(leftOtherColumn, BIGINT); Symbol rightOtherColumnSymbol = pb.symbol(rightOtherColumn, DOUBLE); return pb .join(joinType, pb.values(leftJoinColumnSymbol, leftOtherColumnSymbol), pb.values(rightJoinColumnSymbol, rightOtherColumnSymbol), new EquiJoinClause(leftJoinColumnSymbol, rightJoinColumnSymbol)); }).withSourceStats(0, leftStats) .withSourceStats(1, rightStats) .check(JOIN_STATS_RULE, stats -> stats.equalTo(resultStats)); }
@Test public void testEliminateCrossJoin() { tester().assertThat(new EliminateCrossJoins()) .setSystemProperty(JOIN_REORDERING_STRATEGY, "ELIMINATE_CROSS_JOINS") .on(crossJoinAndJoin(INNER)) .matches( join(INNER, ImmutableList.of(aliases -> new EquiJoinClause(new Symbol("cySymbol"), new Symbol("bySymbol"))), join(INNER, ImmutableList.of(aliases -> new EquiJoinClause(new Symbol("axSymbol"), new Symbol("cxSymbol"))), any(), any()), any())); }
@Test public void testJoinComplementStats() { PlanNodeStatsEstimate expected = planNodeStats(LEFT_ROWS_COUNT * (LEFT_JOIN_COLUMN_NULLS + LEFT_JOIN_COLUMN_NON_NULLS / 4), symbolStatistics(LEFT_JOIN_COLUMN, 0.0, 20.0, LEFT_JOIN_COLUMN_NULLS / (LEFT_JOIN_COLUMN_NULLS + LEFT_JOIN_COLUMN_NON_NULLS / 4), 5), LEFT_OTHER_COLUMN_STATS); PlanNodeStatsEstimate actual = JOIN_STATS_RULE.calculateJoinComplementStats( Optional.empty(), ImmutableList.of(new EquiJoinClause(new Symbol(LEFT_JOIN_COLUMN), new Symbol(RIGHT_JOIN_COLUMN))), LEFT_STATS, RIGHT_STATS, TYPES); assertEquals(actual, expected); }
@Test public void testDoesNotFireForNonDeterministicFilter() { assertReorderJoins() .on(p -> p.join( INNER, p.values(new PlanNodeId("valuesA"), p.symbol("A1")), p.values(new PlanNodeId("valuesB"), p.symbol("B1")), ImmutableList.of(new EquiJoinClause(p.symbol("A1"), p.symbol("B1"))), ImmutableList.of(p.symbol("A1"), p.symbol("B1")), Optional.of(new ComparisonExpression(LESS_THAN, p.symbol("A1").toSymbolReference(), new FunctionCall(QualifiedName.of("random"), ImmutableList.of()))))) .doesNotFire(); }
public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { return new JoinNode.EquiJoinClause(left.toSymbol(aliases), right.toSymbol(aliases)); }
public EquiJoinClause flip() { return new EquiJoinClause(right, left); }
private EquiJoinClause equiJoinClause(Symbol symbol1, Symbol symbol2) { return new EquiJoinClause(symbol1, symbol2); }
private static EquiJoinClause toEquiJoinClause(ComparisonExpression equality, Set<Symbol> leftSymbols) { Symbol leftSymbol = Symbol.from(equality.getLeft()); Symbol rightSymbol = Symbol.from(equality.getRight()); EquiJoinClause equiJoinClause = new EquiJoinClause(leftSymbol, rightSymbol); return leftSymbols.contains(leftSymbol) ? equiJoinClause : equiJoinClause.flip(); }